diff --git a/cmd/mesh-controller/try_test.go b/cmd/mesh-controller/try_test.go new file mode 100644 index 00000000..5f72bcda --- /dev/null +++ b/cmd/mesh-controller/try_test.go @@ -0,0 +1,194 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed +// branch or commit, with no pull request. + +// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own +// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image. +func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) { + request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a", + Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"}, + Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf, + Plan: &link.ChangePlan{}}} + tried := asATry(request, "^TestStore", true) + c := tried.Check + if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly { + t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c) + } + if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil { + t.Fatalf("a try runs the gate or builds for a merge: %+v", c) + } + if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" { + t.Fatalf("a try lost what it merges into what: %+v", tried) + } + if link.KindOf(tried) != link.KindCheck { + t.Fatal("a try does not count as a check in the build queue") + } +} + +// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and +// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one). +func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) { + said := &saidOnTheBus{} + checkEvents = said + defer func() { checkEvents = nil }() + for name, result := range map[string]link.BuildResult{ + "a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}}, + "a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Failed: "cannot clone"}, + // A build seat from before the mark echoes the ask without it: a check of no pull request and no + // group is still nobody's status. + "a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"}, + Check: &link.CheckOutcome{Verdict: "pass"}}, + } { + if err := (builds{}).Built(context.Background(), result); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + if len(said.events) != 0 { + t.Fatalf("a try's outcome was said for the forge: %v", said.events) + } + // A pull request's check is still said. + _ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{ + Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}}) + if len(said.events) != 1 { + t.Fatalf("a pull request's check was said %d time(s)", len(said.events)) + } +} + +// saidOnTheBus keeps the seat events said, and nothing else. +type saidOnTheBus struct { + link.Bus + events []string +} + +func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error { + s.events = append(s.events, seat+" "+event) + return nil +} + +// A ref the forge does not hold is refused before anything is asked, naming it. +func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) { + was := forgeCommit + defer func() { forgeCommit = was }() + forgeCommit = func(_ context.Context, owner, repo, ref string) (string, string, error) { + return "", "", errors.New("404 object does not exist [id: " + ref + "]") + } + err := tryCommand(context.Background(), []string{"novox/mesh-controller", "fix/no-such-branch"}) + if err == nil { + t.Fatal("a try of a ref the forge does not hold was asked") + } + for _, want := range []string{"fix/no-such-branch", "novox/mesh-controller", "nothing was asked", "404"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + for _, args := range [][]string{{"mesh-controller", "main"}, {"novox/mesh-controller"}, {"novox/a", "--upload-pack=x"}, + {"novox/a", "main", "--tests", "x; rm -rf /"}, {"novox/a", "main", "--id", "build-1"}} { + if err := tryCommand(context.Background(), args); err == nil { + t.Errorf("try %v was taken", args) + } + } +} + +// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took. +func TestATrysAnswerCarriesTheFailingLines(t *testing.T) { + tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s", + Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2", + "FAIL\tgithub.com/novox/a/cmd\t3.1s"}}) + lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"}, + {Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"}, + {Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}} + a := tryAnswerOf("build-1", lines) + if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" || + a.Trunk != "ca3f43c" || a.ID != "build-1" { + t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a) + } + // Not answered yet: said as running, with how far it is — never as a pass. + if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" { + t.Fatalf("a try still running: %+v", a) + } + if a := tryAnswerOf("build-1", nil); a.State != "waiting" { + t.Fatalf("a try nothing was said of: %+v", a) + } + // One that could not run is an error with the build seat's word, never a pass. + if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed", + Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") { + t.Fatalf("a try that could not run: %+v", a) + } +} + +// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id, +// a fifth is refused; a try that ended, and a pull request's check, count for nothing. +func TestTriesAreBoundedInNumber(t *testing.T) { + ask := func(id, repo, ref, state string, try bool) link.QueuedAsk { + return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref, + Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}} + } + q := link.Queue{Asks: []link.QueuedAsk{ + ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true), + ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}} + if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room { + t.Fatalf("a second try of a commit: %q %v", same, room) + } + if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room { + t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room) + } + if same, _ := triesIn(q, "novox", "a", "c9"); same != "" { + t.Fatalf("a pull request's check was taken for a try: %q", same) + } + q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true)) + if _, room := triesIn(q, "novox", "e", "c5"); room { + t.Fatal("a fifth try was let in") + } + // An id is checked before it is a token of a bus subject. + for _, id := range []string{"x.>", "*", "a b"} { + if err := tryCommand(context.Background(), []string{"--id", id}); err == nil || + !strings.Contains(err.Error(), "not a try's id") { + t.Errorf("try --id %q: %v", id, err) + } + } +} + +// The verb runs its command, and is declared with the shell habit it replaces (novox/hq ADR 0245). +func TestTheTryVerbRunsItsCommand(t *testing.T) { + for _, c := range []struct { + args map[string]any + want []string + }{ + {map[string]any{"repository": "novox/a", "ref": "fix/x"}, []string{"try", "novox/a", "fix/x"}}, + {map[string]any{"repository": "novox/a", "ref": "fix/x", "base": "main", "tests": "^TestX$", "store-only": "true"}, + []string{"try", "novox/a", "fix/x", "--base", "main", "--tests", "^TestX$", "--store-only"}}, + {map[string]any{"id": "build-1"}, []string{"try", "--id", "build-1"}}, + } { + got, err := argvFor("try", c.args) + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("try %v: %v %v, want %v", c.args, got, err, c.want) + } + } + if _, err := argvFor("try", map[string]any{"repository": "novox/a"}); err == nil { + t.Error("a try of no ref was taken") + } + for _, v := range catalogue.ControllerVerbs { + if v.Name == "try" { + if len(v.Replaces) == 0 { + t.Error("try names no shell habit it replaces") + } + return + } + } + t.Error("try is not a verb of the controller seat") +} diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 7f8bc8e2..7570fb79 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { // `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it // now waits for a gate like any other build, and the release plan walks it through the machines, one // at a time, each judged. - fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+ + fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+ "(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on)) return nil } diff --git a/cmd/mesh-controller/walk_said.go b/cmd/mesh-controller/walk_said.go index ddb724e4..3e5d9bad 100644 --- a/cmd/mesh-controller/walk_said.go +++ b/cmd/mesh-controller/walk_said.go @@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s reached = appendNew(reached, r.Gate.Machines...) carried(r.Gate) } + for _, g := range r.Gates { + if g != nil && !g.Pending { + reached = appendNew(reached, g.Machines...) + carried(g) + } + } for _, tier := range p.Tiers { for _, m := range tier { for _, n := range running(m) { diff --git a/internal/builder/builder.go b/internal/builder/builder.go index e819ea41..728ce83e 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -268,7 +268,14 @@ func build(ctx context.Context, run Runner, publish Publisher, } // What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is // where a changed definition is first read, so it is refused here before anything is made of it. - if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 { + // Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the + // module's directory lacks is refused here, before anything is built. + judged, err := catalogue.WithTemplatesFrom(manifest, within) + if err != nil { + say("manifest", "INVALID: %v", err) + return Result{}, err + } + if problems := catalogue.WrittenSettingsProblems(judged); len(problems) > 0 { err := fmt.Errorf("%s", strings.Join(problems, "; ")) say("manifest", "INVALID: %v", err) return Result{}, err @@ -410,6 +417,11 @@ func build(ctx context.Context, run Runner, publish Publisher, } } + // Each template's text, read back from the archive the build made or took (novox/hq ADR 0349 §2). + if built, err = withTemplateTexts(manifest, within, built); err != nil { + say("templates", "FAILED: %v", err) + return Result{}, err + } resolved, err := manifest.Resolve(built) if err != nil { say("resolve", "FAILED: %v", err) @@ -1076,6 +1088,16 @@ func one(ctx context.Context, run Runner, publish Publisher, return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err) } say("image", "built, publishing") + // Its bases linked into its repository first, so the push needs none of their blobs from this + // machine's store, which may not hold them (novox/hq issue 694). Said and not fatal: a push + // from a store that holds them works without, and one that fails says which blob it lacked. + if lender, can := publish.(BaseLender); can { + if n, err := lender.LendBases(ctx, module+"/"+a.Name, buildArgValues(args)); err != nil { + say("image", "its bases could not all be linked into %s/%s before the push: %v", module, a.Name, err) + } else if n > 0 { + say("image", "%d blob(s) of its bases linked into %s/%s before the push", n, module, a.Name) + } + } reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name) if err != nil { return catalogue.Built{}, err @@ -1427,6 +1449,19 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin return args, resolved, nil } +// buildArgValues is what each --build-arg of a build was handed: the bases, as this mesh holds them. +func buildArgValues(args []string) []string { + var values []string + for i := 0; i+1 < len(args); i += 2 { + if args[i] == "--build-arg" { + if _, value, has := strings.Cut(args[i+1], "="); has { + values = append(values, value) + } + } + } + return values +} + // compile runs a module's own code through its toolchain, and says where the result is. // // **In the module's own directory, under the path the toolchain expects.** A module is compiled diff --git a/internal/builder/check.go b/internal/builder/check.go index bf0ce5bc..a2107499 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -103,6 +103,23 @@ type CheckSpec struct { // for the walk to take. None for a group's composed check, whose members' own checks build. Builds []CheckBuildSpec BuildWith *CheckBuilding + // Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull + // request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read. + Tests string + StoreOnly bool +} + +// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing +// for a pull request's check, which always runs whole. +func selection(spec CheckSpec) []string { + var env []string + if spec.Tests != "" { + env = append(env, EnvTests+"="+spec.Tests) + } + if spec.StoreOnly { + env = append(env, EnvStoreOnly+"=1") + } + return env } // GroupHead is one other head of a delivery group's composed check. @@ -193,6 +210,11 @@ const ( // EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a // pull request checked alone. EnvGroup = "MESH_CHECK_GROUP" + // EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request + // asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's + // check. A script that reads neither runs whole. + EnvTests = "MESH_CHECK_TESTS" + EnvStoreOnly = "MESH_CHECK_STORE_ONLY" ) // CheckTimeout bounds one check; a check that runs past it is an error, not a pass. @@ -515,6 +537,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","), EnvGroup + "=" + groupFile, "GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")} + env = append(env, selection(spec)...) v := CheckVerdict{} modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...) diff --git a/internal/builder/check_test.go b/internal/builder/check_test.go index 92583f5a..1eb5890a 100644 --- a/internal/builder/check_test.go +++ b/internal/builder/check_test.go @@ -590,3 +590,15 @@ func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) { t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer) } } + +// A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a +// pull request's check is handed nothing, and runs whole. +func TestOnlyATrysSelectionReachesTheScript(t *testing.T) { + if env := selection(CheckSpec{Number: 7}); len(env) != 0 { + t.Fatalf("a pull request's check was handed a selection: %v", env) + } + env := selection(CheckSpec{Tests: "^TestX$", StoreOnly: true}) + if len(env) != 2 || env[0] != "MESH_CHECK_TESTS=^TestX$" || env[1] != "MESH_CHECK_STORE_ONLY=1" { + t.Fatalf("a try's selection: %v", env) + } +} diff --git a/internal/builder/lend.go b/internal/builder/lend.go new file mode 100644 index 00000000..d3984647 --- /dev/null +++ b/internal/builder/lend.go @@ -0,0 +1,178 @@ +package builder + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "runtime" + "strings" +) + +// **A built image's bases are linked into its repository before it is pushed** (novox/hq issue 694). +// +// A push reads every blob of the image out of this machine's store, unless the repository it goes to +// already holds that blob: the runtime asks the registry first, blob by blob, and reads locally only +// what the registry lacks. A runtime keeping its images in containerd's store — the default for a +// fresh install from docker 29 on — does not always hold the compressed layers of the base an image +// was built FROM: a layer whose unpacked form it already had is never fetched. The push of such an +// image then fails on a layer this machine never had ("not all of them are available locally"), while +// the very bytes sit in this registry, under the repository the base was mirrored into. +// +// So the base's blobs are mounted into the image's repository first. A mount moves no bytes, and once +// the repository holds a blob the push asks this machine for nothing of it — whichever store the +// runtime keeps. A runtime that holds every layer (the classic store) is not changed by this: its push +// finds the same blobs already there. + +// BaseLender links what a build's bases hold into the repository its image is about to be pushed to. +type BaseLender interface { + // LendBases mounts every blob of each base this registry holds — for the platform this machine + // builds — into `repository`, and answers how many it mounted. + LendBases(ctx context.Context, repository string, bases []string) (int, error) +} + +// platformed is an index's entry: a manifest and the platform it is for. +type platformed struct { + descriptor + Platform *struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } `json:"platform"` +} + +// LendBases mounts each base's blobs into the repository. A base this registry does not hold under its +// repository — named by tag, or held somewhere else — lends nothing and is no error: the push reads it +// from this machine's store as before. +func (r Registry) LendBases(ctx context.Context, repository string, bases []string) (int, error) { + mounted := 0 + for _, base := range bases { + name, digest, pinned := strings.Cut(base, "@") + if !pinned || !strings.HasPrefix(digest, "sha256:") { + continue + } + where, err := parseReference(name + "@" + digest) + if err != nil { + continue + } + from := where.repository + if from == repository { + continue + } + blobs, held, err := r.blobsOf(ctx, from, digest) + if err != nil { + return mounted, fmt.Errorf("reading %s: %w", base, err) + } + if !held { + continue + } + for _, blob := range blobs { + did, err := r.mount(ctx, repository, from, blob) + if err != nil { + return mounted, fmt.Errorf("linking %s of %s into %s: %w", blob, base, repository, err) + } + if did { + mounted++ + } + } + } + return mounted, nil +} + +// blobsOf is the config and layers of the manifest under repository@digest — for an index, of each +// manifest it names for this machine's platform. False when the registry holds no such manifest there. +func (r Registry) blobsOf(ctx context.Context, repository, digest string) ([]string, bool, error) { + body, held, err := r.manifest(ctx, repository, digest) + if err != nil || !held { + return nil, held, err + } + var document struct { + Manifests []platformed `json:"manifests"` + Config *descriptor `json:"config"` + Layers []descriptor `json:"layers"` + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err) + } + var blobs []string + if document.Config != nil { + blobs = append(blobs, document.Config.Digest) + } + for _, l := range document.Layers { + blobs = append(blobs, l.Digest) + } + for _, m := range document.Manifests { + if m.Platform == nil || m.Platform.OS != runtime.GOOS || m.Platform.Architecture != runtime.GOARCH { + continue + } + more, held, err := r.blobsOf(ctx, repository, m.Digest) + if err != nil { + return nil, false, err + } + if held { + blobs = append(blobs, more...) + } + } + return blobs, true, nil +} + +// manifest reads one manifest document from this registry; false when it is not there. +func (r Registry) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, + "http://"+r.Address+"/v2/"+repository+"/manifests/"+digest, nil) + if err != nil { + return nil, false, err + } + request.Header.Set("Accept", manifestAccept) + response, err := r.client().Do(request) + if err != nil { + return nil, false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err) + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { + return nil, false, nil + } + if response.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("%s/%s@%s: %s", r.Address, repository, digest, response.Status) + } + body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + return body, err == nil, err +} + +// mount links one blob from one repository into another; false when the repository already held it. +func (r Registry) mount(ctx context.Context, repository, from, digest string) (bool, error) { + base := "http://" + r.Address + "/v2/" + repository + if there, err := r.has(ctx, base+"/blobs/"+digest); err != nil { + return false, err + } else if there { + return false, nil + } + start, err := http.NewRequestWithContext(ctx, http.MethodPost, + base+"/blobs/uploads/?mount="+digest+"&from="+from, nil) + if err != nil { + return false, err + } + begun, err := r.client().Do(start) + if err != nil { + return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err) + } + begun.Body.Close() + if begun.StatusCode == http.StatusCreated { + return true, nil + } + if begun.StatusCode == http.StatusAccepted { + // Not mounted, and an upload begun in its place: let go of it, so nothing is left half-open. + if location := begun.Header.Get("Location"); location != "" { + if strings.HasPrefix(location, "/") { + location = "http://" + r.Address + location + } + if cancel, err := http.NewRequestWithContext(ctx, http.MethodDelete, location, nil); err == nil { + if done, err := r.client().Do(cancel); err == nil { + done.Body.Close() + } + } + } + return false, fmt.Errorf("%s did not mount it from %s", r.Address, from) + } + return false, fmt.Errorf("%s answered %s when asked to mount it from %s", r.Address, begun.Status, from) +}