Bring F3 up to its release branch, with both sides' store tests (part 5 of 14)
mesh/merge-gate head 14c1e3bb9e7d merged into trunk fcdecb436117 — fail: head 14c1e3bb9e7d does not merge cleanly into trunk fcdecb436117: in conflict cmd/…
mesh/repo-check head 14c1e3bb9e7d merged into trunk fcdecb436117 — fail: head 14c1e3bb9e7d does not merge cleanly into trunk fcdecb436117: in conflict cmd/…
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head 14c1e3bb9e7d merged into trunk fcdecb436117 — fail: head 14c1e3bb9e7d does not merge cleanly into trunk fcdecb436117: in conflict cmd/…
mesh/repo-check head 14c1e3bb9e7d merged into trunk fcdecb436117 — fail: head 14c1e3bb9e7d does not merge cleanly into trunk fcdecb436117: in conflict cmd/…
mesh/delivery superseded: a newer head of the same pull request
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,194 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed
|
||||||
|
// branch or commit, with no pull request.
|
||||||
|
|
||||||
|
// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own
|
||||||
|
// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image.
|
||||||
|
func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) {
|
||||||
|
request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a",
|
||||||
|
Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"},
|
||||||
|
Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf,
|
||||||
|
Plan: &link.ChangePlan{}}}
|
||||||
|
tried := asATry(request, "^TestStore", true)
|
||||||
|
c := tried.Check
|
||||||
|
if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly {
|
||||||
|
t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c)
|
||||||
|
}
|
||||||
|
if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil {
|
||||||
|
t.Fatalf("a try runs the gate or builds for a merge: %+v", c)
|
||||||
|
}
|
||||||
|
if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" {
|
||||||
|
t.Fatalf("a try lost what it merges into what: %+v", tried)
|
||||||
|
}
|
||||||
|
if link.KindOf(tried) != link.KindCheck {
|
||||||
|
t.Fatal("a try does not count as a check in the build queue")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and
|
||||||
|
// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one).
|
||||||
|
func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) {
|
||||||
|
said := &saidOnTheBus{}
|
||||||
|
checkEvents = said
|
||||||
|
defer func() { checkEvents = nil }()
|
||||||
|
for name, result := range map[string]link.BuildResult{
|
||||||
|
"a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||||
|
Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}},
|
||||||
|
"a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
|
||||||
|
Failed: "cannot clone"},
|
||||||
|
// A build seat from before the mark echoes the ask without it: a check of no pull request and no
|
||||||
|
// group is still nobody's status.
|
||||||
|
"a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"},
|
||||||
|
Check: &link.CheckOutcome{Verdict: "pass"}},
|
||||||
|
} {
|
||||||
|
if err := (builds{}).Built(context.Background(), result); err != nil {
|
||||||
|
t.Fatalf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(said.events) != 0 {
|
||||||
|
t.Fatalf("a try's outcome was said for the forge: %v", said.events)
|
||||||
|
}
|
||||||
|
// A pull request's check is still said.
|
||||||
|
_ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{
|
||||||
|
Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}})
|
||||||
|
if len(said.events) != 1 {
|
||||||
|
t.Fatalf("a pull request's check was said %d time(s)", len(said.events))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// saidOnTheBus keeps the seat events said, and nothing else.
|
||||||
|
type saidOnTheBus struct {
|
||||||
|
link.Bus
|
||||||
|
events []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error {
|
||||||
|
s.events = append(s.events, seat+" "+event)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// A ref the forge does not hold is refused before anything is asked, naming it.
|
||||||
|
func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) {
|
||||||
|
was := forgeCommit
|
||||||
|
defer func() { forgeCommit = was }()
|
||||||
|
forgeCommit = func(_ context.Context, owner, repo, ref string) (string, string, error) {
|
||||||
|
return "", "", errors.New("404 object does not exist [id: " + ref + "]")
|
||||||
|
}
|
||||||
|
err := tryCommand(context.Background(), []string{"novox/mesh-controller", "fix/no-such-branch"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a try of a ref the forge does not hold was asked")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"fix/no-such-branch", "novox/mesh-controller", "nothing was asked", "404"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, args := range [][]string{{"mesh-controller", "main"}, {"novox/mesh-controller"}, {"novox/a", "--upload-pack=x"},
|
||||||
|
{"novox/a", "main", "--tests", "x; rm -rf /"}, {"novox/a", "main", "--id", "build-1"}} {
|
||||||
|
if err := tryCommand(context.Background(), args); err == nil {
|
||||||
|
t.Errorf("try %v was taken", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took.
|
||||||
|
func TestATrysAnswerCarriesTheFailingLines(t *testing.T) {
|
||||||
|
tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s",
|
||||||
|
Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2",
|
||||||
|
"FAIL\tgithub.com/novox/a/cmd\t3.1s"}})
|
||||||
|
lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"},
|
||||||
|
{Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"},
|
||||||
|
{Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}}
|
||||||
|
a := tryAnswerOf("build-1", lines)
|
||||||
|
if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" ||
|
||||||
|
a.Trunk != "ca3f43c" || a.ID != "build-1" {
|
||||||
|
t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a)
|
||||||
|
}
|
||||||
|
// Not answered yet: said as running, with how far it is — never as a pass.
|
||||||
|
if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" {
|
||||||
|
t.Fatalf("a try still running: %+v", a)
|
||||||
|
}
|
||||||
|
if a := tryAnswerOf("build-1", nil); a.State != "waiting" {
|
||||||
|
t.Fatalf("a try nothing was said of: %+v", a)
|
||||||
|
}
|
||||||
|
// One that could not run is an error with the build seat's word, never a pass.
|
||||||
|
if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed",
|
||||||
|
Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") {
|
||||||
|
t.Fatalf("a try that could not run: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id,
|
||||||
|
// a fifth is refused; a try that ended, and a pull request's check, count for nothing.
|
||||||
|
func TestTriesAreBoundedInNumber(t *testing.T) {
|
||||||
|
ask := func(id, repo, ref, state string, try bool) link.QueuedAsk {
|
||||||
|
return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref,
|
||||||
|
Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}}
|
||||||
|
}
|
||||||
|
q := link.Queue{Asks: []link.QueuedAsk{
|
||||||
|
ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true),
|
||||||
|
ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}}
|
||||||
|
if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room {
|
||||||
|
t.Fatalf("a second try of a commit: %q %v", same, room)
|
||||||
|
}
|
||||||
|
if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room {
|
||||||
|
t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room)
|
||||||
|
}
|
||||||
|
if same, _ := triesIn(q, "novox", "a", "c9"); same != "" {
|
||||||
|
t.Fatalf("a pull request's check was taken for a try: %q", same)
|
||||||
|
}
|
||||||
|
q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true))
|
||||||
|
if _, room := triesIn(q, "novox", "e", "c5"); room {
|
||||||
|
t.Fatal("a fifth try was let in")
|
||||||
|
}
|
||||||
|
// An id is checked before it is a token of a bus subject.
|
||||||
|
for _, id := range []string{"x.>", "*", "a b"} {
|
||||||
|
if err := tryCommand(context.Background(), []string{"--id", id}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "not a try's id") {
|
||||||
|
t.Errorf("try --id %q: %v", id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The verb runs its command, and is declared with the shell habit it replaces (novox/hq ADR 0245).
|
||||||
|
func TestTheTryVerbRunsItsCommand(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
args map[string]any
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{map[string]any{"repository": "novox/a", "ref": "fix/x"}, []string{"try", "novox/a", "fix/x"}},
|
||||||
|
{map[string]any{"repository": "novox/a", "ref": "fix/x", "base": "main", "tests": "^TestX$", "store-only": "true"},
|
||||||
|
[]string{"try", "novox/a", "fix/x", "--base", "main", "--tests", "^TestX$", "--store-only"}},
|
||||||
|
{map[string]any{"id": "build-1"}, []string{"try", "--id", "build-1"}},
|
||||||
|
} {
|
||||||
|
got, err := argvFor("try", c.args)
|
||||||
|
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("try %v: %v %v, want %v", c.args, got, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := argvFor("try", map[string]any{"repository": "novox/a"}); err == nil {
|
||||||
|
t.Error("a try of no ref was taken")
|
||||||
|
}
|
||||||
|
for _, v := range catalogue.ControllerVerbs {
|
||||||
|
if v.Name == "try" {
|
||||||
|
if len(v.Replaces) == 0 {
|
||||||
|
t.Error("try names no shell habit it replaces")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Error("try is not a verb of the controller seat")
|
||||||
|
}
|
||||||
@@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
|||||||
// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
|
// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
|
||||||
// now waits for a gate like any other build, and the release plan walks it through the machines, one
|
// now waits for a gate like any other build, and the release plan walks it through the machines, one
|
||||||
// at a time, each judged.
|
// at a time, each judged.
|
||||||
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+
|
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+
|
||||||
"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
|
"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s
|
|||||||
reached = appendNew(reached, r.Gate.Machines...)
|
reached = appendNew(reached, r.Gate.Machines...)
|
||||||
carried(r.Gate)
|
carried(r.Gate)
|
||||||
}
|
}
|
||||||
|
for _, g := range r.Gates {
|
||||||
|
if g != nil && !g.Pending {
|
||||||
|
reached = appendNew(reached, g.Machines...)
|
||||||
|
carried(g)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, tier := range p.Tiers {
|
for _, tier := range p.Tiers {
|
||||||
for _, m := range tier {
|
for _, m := range tier {
|
||||||
for _, n := range running(m) {
|
for _, n := range running(m) {
|
||||||
|
|||||||
@@ -268,7 +268,14 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
}
|
}
|
||||||
// What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is
|
// What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is
|
||||||
// where a changed definition is first read, so it is refused here before anything is made of it.
|
// where a changed definition is first read, so it is refused here before anything is made of it.
|
||||||
if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 {
|
// Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the
|
||||||
|
// module's directory lacks is refused here, before anything is built.
|
||||||
|
judged, err := catalogue.WithTemplatesFrom(manifest, within)
|
||||||
|
if err != nil {
|
||||||
|
say("manifest", "INVALID: %v", err)
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
if problems := catalogue.WrittenSettingsProblems(judged); len(problems) > 0 {
|
||||||
err := fmt.Errorf("%s", strings.Join(problems, "; "))
|
err := fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||||
say("manifest", "INVALID: %v", err)
|
say("manifest", "INVALID: %v", err)
|
||||||
return Result{}, err
|
return Result{}, err
|
||||||
@@ -410,6 +417,11 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Each template's text, read back from the archive the build made or took (novox/hq ADR 0349 §2).
|
||||||
|
if built, err = withTemplateTexts(manifest, within, built); err != nil {
|
||||||
|
say("templates", "FAILED: %v", err)
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
resolved, err := manifest.Resolve(built)
|
resolved, err := manifest.Resolve(built)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
say("resolve", "FAILED: %v", err)
|
say("resolve", "FAILED: %v", err)
|
||||||
@@ -1076,6 +1088,16 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
|||||||
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
||||||
}
|
}
|
||||||
say("image", "built, publishing")
|
say("image", "built, publishing")
|
||||||
|
// Its bases linked into its repository first, so the push needs none of their blobs from this
|
||||||
|
// machine's store, which may not hold them (novox/hq issue 694). Said and not fatal: a push
|
||||||
|
// from a store that holds them works without, and one that fails says which blob it lacked.
|
||||||
|
if lender, can := publish.(BaseLender); can {
|
||||||
|
if n, err := lender.LendBases(ctx, module+"/"+a.Name, buildArgValues(args)); err != nil {
|
||||||
|
say("image", "its bases could not all be linked into %s/%s before the push: %v", module, a.Name, err)
|
||||||
|
} else if n > 0 {
|
||||||
|
say("image", "%d blob(s) of its bases linked into %s/%s before the push", n, module, a.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name)
|
reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Built{}, err
|
return catalogue.Built{}, err
|
||||||
@@ -1427,6 +1449,19 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
|
|||||||
return args, resolved, nil
|
return args, resolved, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildArgValues is what each --build-arg of a build was handed: the bases, as this mesh holds them.
|
||||||
|
func buildArgValues(args []string) []string {
|
||||||
|
var values []string
|
||||||
|
for i := 0; i+1 < len(args); i += 2 {
|
||||||
|
if args[i] == "--build-arg" {
|
||||||
|
if _, value, has := strings.Cut(args[i+1], "="); has {
|
||||||
|
values = append(values, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
// compile runs a module's own code through its toolchain, and says where the result is.
|
// compile runs a module's own code through its toolchain, and says where the result is.
|
||||||
//
|
//
|
||||||
// **In the module's own directory, under the path the toolchain expects.** A module is compiled
|
// **In the module's own directory, under the path the toolchain expects.** A module is compiled
|
||||||
|
|||||||
@@ -103,6 +103,23 @@ type CheckSpec struct {
|
|||||||
// for the walk to take. None for a group's composed check, whose members' own checks build.
|
// for the walk to take. None for a group's composed check, whose members' own checks build.
|
||||||
Builds []CheckBuildSpec
|
Builds []CheckBuildSpec
|
||||||
BuildWith *CheckBuilding
|
BuildWith *CheckBuilding
|
||||||
|
// Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull
|
||||||
|
// request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read.
|
||||||
|
Tests string
|
||||||
|
StoreOnly bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing
|
||||||
|
// for a pull request's check, which always runs whole.
|
||||||
|
func selection(spec CheckSpec) []string {
|
||||||
|
var env []string
|
||||||
|
if spec.Tests != "" {
|
||||||
|
env = append(env, EnvTests+"="+spec.Tests)
|
||||||
|
}
|
||||||
|
if spec.StoreOnly {
|
||||||
|
env = append(env, EnvStoreOnly+"=1")
|
||||||
|
}
|
||||||
|
return env
|
||||||
}
|
}
|
||||||
|
|
||||||
// GroupHead is one other head of a delivery group's composed check.
|
// GroupHead is one other head of a delivery group's composed check.
|
||||||
@@ -193,6 +210,11 @@ const (
|
|||||||
// EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a
|
// EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a
|
||||||
// pull request checked alone.
|
// pull request checked alone.
|
||||||
EnvGroup = "MESH_CHECK_GROUP"
|
EnvGroup = "MESH_CHECK_GROUP"
|
||||||
|
// EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request
|
||||||
|
// asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's
|
||||||
|
// check. A script that reads neither runs whole.
|
||||||
|
EnvTests = "MESH_CHECK_TESTS"
|
||||||
|
EnvStoreOnly = "MESH_CHECK_STORE_ONLY"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CheckTimeout bounds one check; a check that runs past it is an error, not a pass.
|
// CheckTimeout bounds one check; a check that runs past it is an error, not a pass.
|
||||||
@@ -515,6 +537,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
|
|||||||
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
|
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
|
||||||
EnvGroup + "=" + groupFile,
|
EnvGroup + "=" + groupFile,
|
||||||
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
|
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
|
||||||
|
env = append(env, selection(spec)...)
|
||||||
|
|
||||||
v := CheckVerdict{}
|
v := CheckVerdict{}
|
||||||
modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...)
|
modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...)
|
||||||
|
|||||||
@@ -590,3 +590,15 @@ func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) {
|
|||||||
t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
|
t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a
|
||||||
|
// pull request's check is handed nothing, and runs whole.
|
||||||
|
func TestOnlyATrysSelectionReachesTheScript(t *testing.T) {
|
||||||
|
if env := selection(CheckSpec{Number: 7}); len(env) != 0 {
|
||||||
|
t.Fatalf("a pull request's check was handed a selection: %v", env)
|
||||||
|
}
|
||||||
|
env := selection(CheckSpec{Tests: "^TestX$", StoreOnly: true})
|
||||||
|
if len(env) != 2 || env[0] != "MESH_CHECK_TESTS=^TestX$" || env[1] != "MESH_CHECK_STORE_ONLY=1" {
|
||||||
|
t.Fatalf("a try's selection: %v", env)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
package builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A built image's bases are linked into its repository before it is pushed** (novox/hq issue 694).
|
||||||
|
//
|
||||||
|
// A push reads every blob of the image out of this machine's store, unless the repository it goes to
|
||||||
|
// already holds that blob: the runtime asks the registry first, blob by blob, and reads locally only
|
||||||
|
// what the registry lacks. A runtime keeping its images in containerd's store — the default for a
|
||||||
|
// fresh install from docker 29 on — does not always hold the compressed layers of the base an image
|
||||||
|
// was built FROM: a layer whose unpacked form it already had is never fetched. The push of such an
|
||||||
|
// image then fails on a layer this machine never had ("not all of them are available locally"), while
|
||||||
|
// the very bytes sit in this registry, under the repository the base was mirrored into.
|
||||||
|
//
|
||||||
|
// So the base's blobs are mounted into the image's repository first. A mount moves no bytes, and once
|
||||||
|
// the repository holds a blob the push asks this machine for nothing of it — whichever store the
|
||||||
|
// runtime keeps. A runtime that holds every layer (the classic store) is not changed by this: its push
|
||||||
|
// finds the same blobs already there.
|
||||||
|
|
||||||
|
// BaseLender links what a build's bases hold into the repository its image is about to be pushed to.
|
||||||
|
type BaseLender interface {
|
||||||
|
// LendBases mounts every blob of each base this registry holds — for the platform this machine
|
||||||
|
// builds — into `repository`, and answers how many it mounted.
|
||||||
|
LendBases(ctx context.Context, repository string, bases []string) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// platformed is an index's entry: a manifest and the platform it is for.
|
||||||
|
type platformed struct {
|
||||||
|
descriptor
|
||||||
|
Platform *struct {
|
||||||
|
OS string `json:"os"`
|
||||||
|
Architecture string `json:"architecture"`
|
||||||
|
} `json:"platform"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LendBases mounts each base's blobs into the repository. A base this registry does not hold under its
|
||||||
|
// repository — named by tag, or held somewhere else — lends nothing and is no error: the push reads it
|
||||||
|
// from this machine's store as before.
|
||||||
|
func (r Registry) LendBases(ctx context.Context, repository string, bases []string) (int, error) {
|
||||||
|
mounted := 0
|
||||||
|
for _, base := range bases {
|
||||||
|
name, digest, pinned := strings.Cut(base, "@")
|
||||||
|
if !pinned || !strings.HasPrefix(digest, "sha256:") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
where, err := parseReference(name + "@" + digest)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
from := where.repository
|
||||||
|
if from == repository {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
blobs, held, err := r.blobsOf(ctx, from, digest)
|
||||||
|
if err != nil {
|
||||||
|
return mounted, fmt.Errorf("reading %s: %w", base, err)
|
||||||
|
}
|
||||||
|
if !held {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, blob := range blobs {
|
||||||
|
did, err := r.mount(ctx, repository, from, blob)
|
||||||
|
if err != nil {
|
||||||
|
return mounted, fmt.Errorf("linking %s of %s into %s: %w", blob, base, repository, err)
|
||||||
|
}
|
||||||
|
if did {
|
||||||
|
mounted++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mounted, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// blobsOf is the config and layers of the manifest under repository@digest — for an index, of each
|
||||||
|
// manifest it names for this machine's platform. False when the registry holds no such manifest there.
|
||||||
|
func (r Registry) blobsOf(ctx context.Context, repository, digest string) ([]string, bool, error) {
|
||||||
|
body, held, err := r.manifest(ctx, repository, digest)
|
||||||
|
if err != nil || !held {
|
||||||
|
return nil, held, err
|
||||||
|
}
|
||||||
|
var document struct {
|
||||||
|
Manifests []platformed `json:"manifests"`
|
||||||
|
Config *descriptor `json:"config"`
|
||||||
|
Layers []descriptor `json:"layers"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &document); err != nil {
|
||||||
|
return nil, false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
|
||||||
|
}
|
||||||
|
var blobs []string
|
||||||
|
if document.Config != nil {
|
||||||
|
blobs = append(blobs, document.Config.Digest)
|
||||||
|
}
|
||||||
|
for _, l := range document.Layers {
|
||||||
|
blobs = append(blobs, l.Digest)
|
||||||
|
}
|
||||||
|
for _, m := range document.Manifests {
|
||||||
|
if m.Platform == nil || m.Platform.OS != runtime.GOOS || m.Platform.Architecture != runtime.GOARCH {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
more, held, err := r.blobsOf(ctx, repository, m.Digest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if held {
|
||||||
|
blobs = append(blobs, more...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return blobs, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// manifest reads one manifest document from this registry; false when it is not there.
|
||||||
|
func (r Registry) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||||
|
"http://"+r.Address+"/v2/"+repository+"/manifests/"+digest, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
request.Header.Set("Accept", manifestAccept)
|
||||||
|
response, err := r.client().Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
if response.StatusCode == http.StatusNotFound {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
if response.StatusCode != http.StatusOK {
|
||||||
|
return nil, false, fmt.Errorf("%s/%s@%s: %s", r.Address, repository, digest, response.Status)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
|
||||||
|
return body, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// mount links one blob from one repository into another; false when the repository already held it.
|
||||||
|
func (r Registry) mount(ctx context.Context, repository, from, digest string) (bool, error) {
|
||||||
|
base := "http://" + r.Address + "/v2/" + repository
|
||||||
|
if there, err := r.has(ctx, base+"/blobs/"+digest); err != nil {
|
||||||
|
return false, err
|
||||||
|
} else if there {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
base+"/blobs/uploads/?mount="+digest+"&from="+from, nil)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
begun, err := r.client().Do(start)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
|
||||||
|
}
|
||||||
|
begun.Body.Close()
|
||||||
|
if begun.StatusCode == http.StatusCreated {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if begun.StatusCode == http.StatusAccepted {
|
||||||
|
// Not mounted, and an upload begun in its place: let go of it, so nothing is left half-open.
|
||||||
|
if location := begun.Header.Get("Location"); location != "" {
|
||||||
|
if strings.HasPrefix(location, "/") {
|
||||||
|
location = "http://" + r.Address + location
|
||||||
|
}
|
||||||
|
if cancel, err := http.NewRequestWithContext(ctx, http.MethodDelete, location, nil); err == nil {
|
||||||
|
if done, err := r.client().Do(cancel); err == nil {
|
||||||
|
done.Body.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("%s did not mount it from %s", r.Address, from)
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("%s answered %s when asked to mount it from %s", r.Address, begun.Status, from)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user