diff --git a/cmd/mesh-builder/holder.go b/cmd/mesh-builder/holder.go index bf127ccf..777ca6f5 100644 --- a/cmd/mesh-builder/holder.go +++ b/cmd/mesh-builder/holder.go @@ -41,6 +41,9 @@ type holder struct { say func(link.HolderState) error // remove runs what a kill needs outside the build: the containers left behind. remove builder.Runner + // scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where + // this machine publishes to no artifact store. + scratch builder.Scratch mu sync.Mutex paused bool @@ -131,6 +134,34 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) { } } +// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while +// after. Every holder sweeps the one store; what one let go of, another finds gone. +func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) { + sweep := func() { + if h.scratch == nil { + return + } + n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now()) + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err) + case n > 0: + fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n) + } + } + sweep() + tick := time.NewTicker(every) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + sweep() + } + } +} + // begin records the build this machine took, with the cancel that ends it. func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running { r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})} diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index e47b9ed7..4254462b 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -97,6 +97,9 @@ func run() error { if err := notReady(); err != nil { return err } + // **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds + // differently never takes a build another one made. + builder.Version = builder.SelfVersion(version) credential, err := brokerFrom() if err != nil { return err @@ -174,6 +177,9 @@ func run() error { fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry) publisher := builder.Registry{Address: registry, Run: builder.Command} + // Check builds are let go of after a week (novox/hq ADR 0350 §5). + h.scratch = publisher + go h.expireChecks(ctx, time.Hour, time.Now) return machine.Take(ctx, func(ctx context.Context, work link.Build) { answer(ctx, publisher, on, workspace, work, h) @@ -262,6 +268,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri if err == nil { say("packages", saysRegistry(npmrc)) } + registryHere, isRegistry := publisher.(builder.Registry) var built builder.Result if request.Check != nil { // **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built, @@ -271,12 +278,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri if r, ok := publisher.(builder.Registry); ok { registry = r.Address } + spec := checkSpecOf(request) + // What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the + // scratch namespace. **Never with the package registry's credential**: this is code nobody has approved + // yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages + // at build time has no reuse key anyway, so nothing is lost by refusing it here. + if isRegistry && len(spec.Builds) > 0 { + spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held, + Seats: request.Seats} + } var v builder.CheckVerdict - v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say) + v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say) if err == nil { result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report, Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo), - Head: v.Head, Trunk: v.Trunk, Beside: v.Beside} + Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built)} // What each of a group's heads was judged as (novox/hq ADR 0323 §6). for _, m := range v.Members { result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, @@ -289,9 +305,20 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri // after a clone that then fails at npm ci. // Every container it starts is labelled with its id, so a kill finds what outlived the // docker client (ADR 0219). - built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher, - request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, - forgeFrom(), say, request.Seats) + if request.Reuse != nil && isRegistry && !request.DryRun { + // **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2). + reuse := builder.Reuse{Scratch: registryHere} + for _, p := range request.Reuse.Pulls { + reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}) + } + built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse, + request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, + forgeFrom(), say, request.Seats) + } else { + built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher, + request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, + forgeFrom(), say, request.Seats) + } } // What it copied into the store, whatever became of the build (novox/hq ADR 0257). result.Mirrored = built.Mirrored @@ -327,6 +354,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri } result.Against = built.Against result.SourceFingerprint = built.Source + // What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4). + result.ReuseKey = built.ReuseKey + if built.Reused != nil { + f := built.Reused.From + result.Reused = built.Reused.Words() + result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo, + Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key} + } else if request.Reuse != nil { + result.BuiltWhy = built.Why + if result.BuiltWhy == "" { + result.BuiltWhy = "it has no artifact to build" + } + } result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches for _, r := range built.Read { result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref}) @@ -377,6 +417,10 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { for dir, b := range c.Beside { spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} } + for _, b := range c.Builds { + spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path, + Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip}) + } for _, m := range c.Members { spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository, Ref: m.Ref, Paths: m.Paths, Base: m.Base}) @@ -384,6 +428,19 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { return spec } +// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1). +func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt { + var out []link.CheckBuilt + for _, b := range built { + c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String()} + for _, m := range b.Made { + c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference}) + } + out = append(out, c) + } + return out +} + // layerOf is one layer of a check as the outcome carries it. func layerOf(l *builder.Layer) *link.CheckLayer { if l == nil { diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 23cdcfd3..d7eb9c2d 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -15,6 +15,7 @@ import ( "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -483,6 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa Seats: seatBases(ctx), DryRun: dryRun, } + request.Reuse = reuseAsked(ctx, dryRun) fmt.Printf("asked for %s", source) if source.Seat != "" { fmt.Printf(" (%s)", repository) @@ -669,6 +671,12 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, manifest.Module, short(result.Commit), err) } + // **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is + // registered only as the walk's copy in the module's own namespace. + if scratch := inScratch(result.Made); scratch != "" { + return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+ + "namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch) + } // **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard // twice, or replayed, would otherwise make the build a rollback put back what the module is again, // and the next push would send it. @@ -714,6 +722,16 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return manifest, kept, nil } +// inScratch is the first artifact a build made into the scratch namespace; empty when none is. +func inScratch(made []link.MadeArtifact) string { + for _, m := range made { + if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) { + return m.Reference + } + } + return "" +} + // errOffTheTrunk is a build of a commit off its repository's trunk, which is never published. var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " + "never published (ADR 0238) — merge it, and the merge builds it") diff --git a/cmd/mesh-controller/check_builds.go b/cmd/mesh-controller/check_builds.go new file mode 100644 index 00000000..7cf571de --- /dev/null +++ b/cmd/mesh-controller/check_builds.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + + "github.com/novox/mesh-controller/internal/builder" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact +// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the +// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which +// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store, +// as the controller's own collection does. The build seat's holders let go of anything a week old on their own, +// so a word lost here leaves a week's builds, never more. + +// checkBuildsIn is the scratch namespace as the controller reaches the artifact store. +func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, err + } + address, err := artifactStoreAddress(ctx, inv, shelf, "") + if err != nil { + return nil, err + } + if address == "" { + return nil, errors.New("the artifact store is not on the mesh's network") + } + return builder.Registry{Address: address}, nil +} + +// forgetChecks lets go of a pull request's check builds and answers what it did. +var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) { + scratch, err := checkBuildsIn(ctx, inv) + if err != nil { + return "", err + } + n, err := scratch.Forget(ctx, pull) + if err != nil { + return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err) + } + return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil +} + +// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge +// it answers: the walk took what it reuses. Said, never fatal. +func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) { + if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil { + return + } + for _, pull := range reuseAskOf(&p).Pulls { + said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number}) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err) + continue + } + fmt.Printf("%s: %s\n", p.ID, said) + } +} + +// forgetChecksCommand is `forget-checks `: the verb the delivery's owner asks when the forge +// says a pull request closed unmerged (novox/hq ADR 0350 §5). +func forgetChecksCommand(ctx context.Context, args []string) error { + if len(args) != 3 { + return errors.New("forget-checks ") + } + number, err := strconv.Atoi(args[2]) + if err != nil || number <= 0 { + return fmt.Errorf("%q is not a pull request's number", args[2]) + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number}) + if err != nil { + return err + } + fmt.Println(said) + return nil +} diff --git a/cmd/mesh-controller/check_builds_test.go b/cmd/mesh-controller/check_builds_test.go new file mode 100644 index 00000000..a4dda987 --- /dev/null +++ b/cmd/mesh-controller/check_builds_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/builder" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in. + +// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the +// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one +// that reads the repository through a context not at all. A change that moves no module builds nothing. +func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) { + entries := []inventory.Entry{ + {Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}}, + {Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}}, + {Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}}, + } + p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"} + scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}} + clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil } + + got := checkBuildsOf(p, scope, entries, clone) + want := []link.CheckBuild{ + {Module: "app", Path: "modules/app", Merged: true, Tier: 0}, + {Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"}, + {Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want) + } + if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil { + t.Fatalf("a change that moves no module asks for builds: %+v", got) + } +} + +// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry +// run never does, and neither does an ask that is not a walk's. +func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) { + p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ + {Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3}, + {Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}} + asked := reuseAskOf(p) + if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) { + t.Fatalf("the walk names %+v", asked.Pulls) + } + walk := withReuse(t.Context(), asked) + if reuseAsked(walk, false) != asked { + t.Fatal("a walk's ask does not carry its leave") + } + if reuseAsked(walk, true) != nil { + t.Fatal("a dry run carries a leave to take a check's build") + } + if reuseAsked(t.Context(), false) != nil { + t.Fatal("an ask that is not a walk's carries a leave") + } +} + +// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses. +func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) { + var forgot []builder.PullRef + was := forgetChecks + forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) { + forgot = append(forgot, pull) + return "", nil + } + t.Cleanup(func() { forgetChecks = was }) + p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ + {Repository: "novox/shop", Number: 7}}}} + + p.WalkSaid = inventory.WalkStarted + forgetChecksOfWalk(t.Context(), nil, p) + p.WalkSaid = "" + forgetChecksOfWalk(t.Context(), nil, p) + if len(forgot) != 0 { + t.Fatalf("a walk not ended let go of %v, which it may still take", forgot) + } + p.WalkSaid = inventory.WalkEnded + forgetChecksOfWalk(t.Context(), nil, p) + if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) { + t.Fatalf("the walk ended let go of %v", forgot) + } +} + +// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is. +func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) { + if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" { + t.Fatalf("a module's own build reads as a scratch build: %s", got) + } + scratch := "store.internal:5000/scratch/app/server@sha256:aa" + if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch { + t.Fatalf("a scratch build was not found: %q", got) + } + if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" { + t.Fatal("a scratch archive was not found") + } +} + +// The walk says per module what it did: "reused from …" or "built: " — and a build seat from before says +// neither, which is said. +func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) { + if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" { + t.Fatalf("said %q", got) + } + if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" { + t.Fatalf("said %q", got) + } + if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") { + t.Fatalf("said %q", got) + } +} + +// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the +// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered +// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and +// is named in testdata/store-tests. +func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + gateEvery = 0 + + digest := "sha256:" + strings.Repeat("c", 64) + reused := func(id, reference string) link.BuildResult { + manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{ + {"id": "svc", "type": "container", "name": "app", "image": reference}}}) + return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor", + Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"}, + Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}}, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}, + Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest, + ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9}, + Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}} + } + if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil || + !strings.Contains(err.Error(), "scratch") { + t.Fatalf("a build in the scratch namespace was registered: %v", err) + } + id := link.NewBuildID(time.Now().Add(time.Second)) + result := reused(id, "store.internal:5000/app/server@"+digest) + if _, _, err := takeIn(ctx, inv, result); err != nil { + t.Fatal(err) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil || current["app"].Commit != "c3" { + t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(shelf["app"].Resources) + if !strings.Contains(string(raw), "app/server@"+digest) { + t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw) + } + + // The walk asked it: the walk keeps what its build did, and its gate judges the registered build. + asked := time.Now().UTC() + p := g.plan(t) + p.Commit = "c3" + p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords()) + p = g.plan(t) + if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") { + t.Fatalf("the walk keeps %+v", m) + } + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p = g.plan(t) + gate := p.Modules["app"].Gate + if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed { + t.Fatalf("the gate judged %+v, not the reused build", gate) + } + if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" { + t.Fatalf("the first machine was sent %v, not the reused build", sent) + } +} diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 8a8265eb..b6243751 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -282,10 +282,59 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop Source: sourceOnSeat(source), Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New, - Manifests: scope.Manifests, Judge: scope.Judge}, + Manifests: scope.Manifests, Judge: scope.Judge, Builds: checkBuildsOf(p, scope, entries, clone)}, }, nil } +// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them +// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges +// into, from the merge the check makes; any other — a module standing on one — from its own repository at the +// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context, +// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that +// moves no module builds nothing. +func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry, + clone func(inventory.Source) (string, error)) []link.CheckBuild { + if len(scope.Reach.Plan.Tiers) == 0 { + return nil + } + m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL} + byName := map[string]inventory.Entry{} + for _, e := range entries { + byName[e.Manifest.Module] = e + } + skip := map[string]string{} + for _, e := range scope.Reach.Packaging { + skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " + + "cannot hand the merge" + } + for _, e := range scope.Reach.Deleted { + skip[e.Manifest.Module] = "the merge deletes it" + } + var out []link.CheckBuild + for tier, names := range scope.Reach.Plan.Tiers { + for _, name := range names { + e, known := byName[name] + b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]} + switch { + case !known: + b.Skip = "the catalogue does not hold it" + case b.Skip != "": + case sourceIs(e.Source, m): + b.Merged, b.Path = true, e.Source.Path + default: + url, err := clone(e.Source) + if err != nil { + b.Skip = "its repository cannot be cloned: " + err.Error() + break + } + b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref) + } + out = append(out, b) + } + } + return out +} + // besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under: // a core repository at the commit the mesh runs of the module built from it (`running`) — but the // catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge @@ -382,6 +431,7 @@ func checkedOf(result link.BuildResult) link.Checked { // What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the // repositories read beside it. c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took + c.Built = result.Check.Built if c.Gate == nil { // A build seat from before the layers: its verdict is the gate's. c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 7269d761..1e656622 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -104,6 +104,8 @@ func run() error { return replayCommand(ctx, args[1:]) case "kill": return killCommand(ctx, args[1:]) + case "forget-checks": + return forgetChecksCommand(ctx, args[1:]) case "pause", "resume": return pauseCommand(ctx, args[0], args[1:]) case "collection": @@ -429,8 +431,14 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { } fmt.Printf("%s: %s %s registered, built on %s from %s\n", result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit)) + // What the walk's build did, kept on the walk whose ask it answers (novox/hq ADR 0350 §4); a build seat from + // before it says nothing, and that is what is kept. + made := result.MadeWords() + if result.Reused != "" || result.BuiltWhy != "" { + fmt.Printf("%s: %s %s\n", result.ID, manifest.Module, made) + } saysWhenThePolicyActs(ctx, b.inv, manifest.Module) - planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID) + planBuiltSaying(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID, made) // A module registered may be one a machine is now behind: `status` is composed again. statusFrom.nudge() return nil diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index a50c7a1b..05bfc4c5 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -249,7 +249,13 @@ func serve(ctx context.Context) (err error) { // (novox/hq to-be 45 §9). checkEvents = bus // And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239). - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) } + inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { + sayPlanMoved(ctx, bus, open.inventory, p) + // A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5). + if p.WalkSaid == inventory.WalkEnded { + go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p) + } + } if err := server.Checks(following{open}); err != nil { return err } diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 452ae757..e2d30cdf 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -313,6 +313,59 @@ func init() { } } +// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2). +type reuseKey struct{} + +// withReuse is ctx carrying a walk's leave to take a check's build. +func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context { + return context.WithValue(ctx, reuseKey{}, r) +} + +// reuseIn is the leave ctx carries; nil for every ask but a walk's. +func reuseIn(ctx context.Context) *link.ReuseAsk { + r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk) + return r +} + +// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it +// takes nothing either (novox/hq ADR 0350 §2, §6). +func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk { + if dryRun { + return nil + } + return reuseIn(ctx) +} + +// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers. +func reuseAskOf(p *inventory.Plan) *link.ReuseAsk { + r := &link.ReuseAsk{} + if p.Delivery == nil { + return r + } + seen := map[link.PullRef]bool{} + for _, m := range p.Delivery.Merges { + pull, ok := pullOf(m) + if !ok { + continue + } + if !seen[pull] { + seen[pull] = true + r.Pulls = append(r.Pulls, pull) + } + } + return r +} + +// pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled +// (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none. +func pullOf(m inventory.PlanMerge) (link.PullRef, bool) { + parts := strings.Split(strings.Trim(strings.TrimSuffix(m.Repository, ".git"), "/"), "/") + if len(parts) < 2 || m.Number <= 0 || parts[len(parts)-2] == "" || parts[len(parts)-1] == "" { + return link.PullRef{}, false + } + return link.PullRef{Owner: parts[len(parts)-2], Repo: parts[len(parts)-1], Number: m.Number}, true +} + // askModule asks the build machine for one module of a plan and marks it asked, with the id it was // asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked. func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) { @@ -340,7 +393,9 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s if p.Delivery != nil && p.Delivery.Alone && carried != "" { ref = carried } - id, err := askABuild(ctx, source, e.Source.Path, ref) + // **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull + // requests it answers, whose checks name the part of a differing key. + id, err := askABuild(withReuse(ctx, reuseAskOf(p)), source, e.Source.Path, ref) if err != nil { state.State = "failed" state.Why = err.Error() @@ -359,7 +414,7 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s state.State = "asked" state.AskedAt = &now state.Build = id - state.Why, state.Commit, state.BuiltAt = "", "", nil + state.Why, state.Commit, state.BuiltAt, state.Made = "", "", nil, "" } // planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and @@ -371,6 +426,12 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s // would send machines, and the next tier, what the later merge replaced. asked is zero when the // build's request time is not known, and such an outcome is taken as before. func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) { + planBuiltSaying(ctx, open, module, commit, failed, asked, id, "") +} + +// planBuiltSaying is planBuilt keeping what the build did — "reused from " or "built: " (novox/hq +// ADR 0350 §4) — on the module of the walk it answers. +func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) { inv := open.inventory // One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather // than write over what the holder is about to save. Not taken, it is still in the build records, @@ -436,6 +497,7 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string, state.State = "built" state.BuiltAt = &now state.Commit = commit + state.Made = made } if err := inv.SavePlan(ctx, p); err != nil { fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index ff43179d..3cc62d1a 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -452,6 +452,11 @@ func (a *verbArguments) commandLine() ([]string, error) { case "queue": // As data, the table its output (novox/hq issue 455). return []string{"queue", "--json"}, nil + case "forget-checks": + if err := need("owner", "repo", "number"); err != nil { + return nil, err + } + return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil case "cancel", "kill": if err := need("id"); err != nil { return nil, err diff --git a/internal/artifacts/scratch.go b/internal/artifacts/scratch.go new file mode 100644 index 00000000..cb20b972 --- /dev/null +++ b/internal/artifacts/scratch.go @@ -0,0 +1,91 @@ +package artifacts + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" +) + +// Listing and letting go of tagged documents: the check builds the scratch namespace keeps (novox/hq ADR 0350 +// §5). A check build is found by its tag, and the build seat's holder lets each go when its pull request closes +// or it is seven days old, so the store keeps a week of check builds and no more. + +// Tags is every tag repository holds, in the store's order. None for a repository the store does not hold. +func (s Store) Tags(ctx context.Context, repository string) ([]string, error) { + if s.Address == "" { + return nil, fmt.Errorf("this mesh has no artifact store on its network to list %s in", repository) + } + var tags []string + next := "http://" + s.Address + "/v2/" + repository + "/tags/list?n=1000" + for next != "" { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, next, nil) + if err != nil { + return nil, err + } + response, err := s.client().Do(request) + if err != nil { + return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + response.Body.Close() + return tags, nil + default: + response.Body.Close() + return nil, fmt.Errorf("the artifact store answered %s for the tags of %s", response.Status, repository) + } + var page struct { + Tags []string `json:"tags"` + } + err = json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&page) + link := response.Header.Get("Link") + response.Body.Close() + if err != nil { + return nil, fmt.Errorf("the tags of %s could not be read: %w", repository, err) + } + tags = append(tags, page.Tags...) + next = nextPage(link, s.Address) + } + return tags, nil +} + +// nextPage is the registry's `Link: ; rel="next"`, made whole; empty when there is no next page. +func nextPage(link, address string) string { + if !strings.Contains(link, `rel="next"`) { + return "" + } + start, end := strings.Index(link, "<"), strings.Index(link, ">") + if start < 0 || end <= start+1 { + return "" + } + target := link[start+1 : end] + if strings.HasPrefix(target, "/") { + return "http://" + address + target + } + return target +} + +// DeleteTagged lets go of the document tag names in repository: its manifest, by digest, so the next +// collection takes its layer. Gone when the tag names nothing. +func (s Store) DeleteTagged(ctx context.Context, repository, tag string) error { + if s.Address == "" { + return fmt.Errorf("this mesh has no artifact store on its network to let go of %s:%s in", repository, tag) + } + digest, err := s.manifestDigest(ctx, repository, tag) + if err != nil { + return err + } + if digest == "" { + return Gone + } + err = s.remove(ctx, s.url(repository, "manifests", digest), repository+":"+tag) + if errors.Is(err, Gone) { + return Gone + } + return err +} diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 586b375f..0ff70c71 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -91,6 +91,16 @@ type Result struct { // (novox/hq issue 280). Source string + // ReuseKey is the build's reuse key (novox/hq ADR 0350 §1): its source fingerprint's parts with the builder's + // version, and ReuseParts those parts; empty, with NoKey saying why, where the source does not pin the build. + ReuseKey string + ReuseParts map[string]string + NoKey string + // Reused is the check build a walk took instead of building (ADR 0350 §2), and Why, when it built, why it did + // not take one (§3). Both empty for a build not asked to reuse. + Reused *Reused + Why string + // Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the // entries a changed file is tested against (SourceHolds). The module's own repository has an empty // Repository. Said only for a build of a commit on the trunk, and only for a repository whose every @@ -136,14 +146,41 @@ func Build(ctx context.Context, run Runner, publish Publisher, // one record that it is there. var mirrored []string result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log, - &mirrored, seats...) + &mirrored, nil, seats...) + result.Mirrored = mirrored + return result, err +} + +// Reuse is what a walk's build may take instead of building (novox/hq ADR 0350 §2): the scratch namespace's check +// builds, and the pull requests the walk answers, whose checks name a differing key. +type Reuse struct { + Scratch Scratch + Pulls []PullRef + // keyFirst is a check's build (§1): its key is taken before anything is built, and a module with none is not + // built at all — nothing could ever take it. + keyFirst bool +} + +// errNoKey is a check's build of a module whose source does not pin it: not built (ADR 0350 §1). +var errNoKey = errors.New("it has no reuse key") + +// BuildReusing is Build for the walk of a trunk commit (novox/hq ADR 0350 §2): before building, it takes the reuse +// key of the commit's tree, and where a check build holds the same key it copies that build into the module's +// namespace by digest and answers with it, building nothing. Otherwise it builds, and Result.Why says why (§3). +// Only a commit on its trunk takes a check build: anything else builds, as Build does. +func BuildReusing(ctx context.Context, run Runner, publish Publisher, reuse Reuse, + repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, + forge GitCredential, log Log, seats ...map[string]string) (Result, error) { + var mirrored []string + result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log, + &mirrored, &reuse, seats...) result.Mirrored = mirrored return result, err } func build(ctx context.Context, run Runner, publish Publisher, repository, path, ref, workspace string, held map[string]string, npmrc Npmrc, - forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) { + forge GitCredential, log Log, mirrored *[]string, reuse *Reuse, seats ...map[string]string) (Result, error) { // The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers // that hand none — tests of everything but contexts — read as they did. var seatBases map[string]string @@ -284,6 +321,9 @@ func build(ctx context.Context, run Runner, publish Publisher, var built []catalogue.Built // stoodOn is every base the build was handed, as resolved — the edges the catalogue derives. var stoodOn []string + // reused is the check build a walk took, and builtWhy why it built instead (novox/hq ADR 0350). + var reused *Reused + builtWhy := "" if manifest.Build != nil { // What this module said it stands on, answered with what this mesh actually holds. Done // before anything is built, so a missing base is refused in front of the person who can @@ -327,7 +367,35 @@ func build(ctx context.Context, run Runner, publish Publisher, // Ordered, so two builds of one commit do the same work in the same sequence and their // logs can be compared. sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) + // **A walk takes the check's build of the same key** (novox/hq ADR 0350 §2): only a commit on its trunk, and + // only when asked — a check, a dry run, `build --ref` and `rebuild` never are. Its key is taken by the same + // steps a build takes, stopped before anything is built; what it takes is copied by digest. + if reuse != nil && reuse.keyFirst { + _, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args, + held, npmrcPath, npmrc, seatBases, src, say) + if err == nil && key.Key == "" { + err = fmt.Errorf("%w: %s", errNoKey, key.Why) + } + if err != nil { + say("reuse", "not built for the merge: %v", err) + return Result{Commit: commit, NoKey: err.Error()}, err + } + } else if reuse != nil && reuse.Scratch != nil { + taken, keyed, why := takeACheckBuild(ctx, run, publish, *reuse, manifest, within, workspace, commit, + credentials, artifacts, args, held, npmrcPath, npmrc, seatBases, src, onTrunk, say) + if taken != nil { + built = taken.Made + src = keyed + reused = taken + } else { + builtWhy = why + say("reuse", "built: %s", why) + } + } for _, a := range artifacts { + if reused != nil { + break + } say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say) if err != nil { @@ -363,9 +431,102 @@ func build(ctx context.Context, run Runner, publish Publisher, say("source", "%d path(s) of %s", len(s.Paths), where) } } + // The reuse key (novox/hq ADR 0350 §1): what a check keeps its build under, and what a walk takes one by. + key := keyOf(src, manifest, Version) return Result{Manifest: resolved, Commit: commit, Built: built, Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint, - Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources}, nil + Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources, + ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil +} + +// takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by +// the steps a build takes, each artifact stopped before it is built; the build kept under it copied by digest into +// the module's namespace. Answers what it took and the source inputs its key was taken from, or why it builds. A +// copy that fails is said, and the walk builds (§3): it never fails the walk on its own. +func takeACheckBuild(ctx context.Context, run Runner, publish Publisher, reuse Reuse, manifest catalogue.Manifest, + within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string, + held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs, onTrunk bool, + say func(step, format string, args ...any)) (*Reused, *sourceInputs, string) { + if !onTrunk { + return nil, nil, "the commit is not on its trunk, and only a trunk commit's walk takes a check's build" + } + keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args, + held, npmrcPath, npmrc, seats, src, say) + if err != nil { + return nil, nil, err.Error() + } + if key.Key == "" { + return nil, nil, "it has no reuse key: " + key.Why + } + found, ok, err := reuse.Scratch.Find(ctx, manifest.Module, key.Key) + if err != nil { + return nil, nil, "the scratch namespace could not be read: " + err.Error() + } + if !ok { + return nil, nil, notKeptWhy(ctx, reuse, manifest.Module, key) + } + taken := &Reused{From: found} + for _, made := range found.Made { + copied, err := reuse.Scratch.Take(ctx, manifest.Module, made) + if err != nil { + why := fmt.Sprintf("copying %s's %s from %s failed (%v)", manifest.Module, made.Name, found.Said(), err) + say("reuse", "%s; built instead", why) + return nil, nil, why + } + taken.Made = append(taken.Made, copied) + } + // Every artifact the manifest declares, or none: a check build missing one is not this build. + if len(taken.Made) != len(artifacts) { + return nil, nil, fmt.Sprintf("%s holds %d artifact(s) of %s, which declares %d", found.Said(), len(taken.Made), + manifest.Module, len(artifacts)) + } + say("reuse", "%s", taken.Words()) + return taken, keyed, "" +} + +// keyFor takes a build's reuse key by the steps a build takes, each artifact stopped before anything is built or +// published (ADR 0350 §2): its own copy of the source inputs, so the build's are untouched. +func keyFor(ctx context.Context, run Runner, publish Publisher, manifest catalogue.Manifest, + within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string, + held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs, + say func(step, format string, args ...any)) (*sourceInputs, reuseKey, error) { + keyed := src.copy() + keyed.onlyKey = true + for _, a := range artifacts { + if _, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, + npmrcPath, npmrc, seats, keyed, say); err != nil { + return nil, reuseKey{}, fmt.Errorf("its reuse key could not be taken: %w", err) + } + } + keyed.onlyKey = false + return keyed, keyOf(keyed, manifest, Version), nil +} + +// notKeptWhy is why no check build has the key (§3): no check of the walk's pull requests built the module, the +// key differs — naming the part — or the build it made is gone. +func notKeptWhy(ctx context.Context, reuse Reuse, module string, key reuseKey) string { + if len(reuse.Pulls) == 0 { + return "no check built it with this key" + } + builds, err := reuse.Scratch.OfPulls(ctx, reuse.Pulls) + if err != nil { + return "no check build has this key, and which checks built it could not be read: " + err.Error() + } + for _, b := range builds { + if b.Module != module { + continue + } + if b.Key == key.Key { + return b.Said() + " built it with this key, and it was removed" + } + return "the key differs from " + b.Said() + "'s in " + strings.Join(differing(b.Parts, key.Parts), ", ") + } + var pulls []string + for _, p := range reuse.Pulls { + pulls = append(pulls, p.String()) + } + return "no check of " + strings.Join(pulls, ", ") + " built it (a module its check did not move, or a check " + + "that did not finish)" } // branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`. @@ -754,6 +915,10 @@ func one(ctx context.Context, run Runner, publish Publisher, switch a.Kind { case catalogue.ArtifactUpstream: + // Its key is the image it copies, read from the manifest (keyOf): nothing to take here. + if src.keyOnly() { + return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil + } // Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is // pinned by a digest this registry assigned rather than by a tag somebody else can move. // @@ -882,6 +1047,10 @@ func one(ctx context.Context, run Runner, publish Publisher, } else { src.ownWhole() } + // Taken for a reuse key (novox/hq ADR 0350 §2): everything its fingerprint reads is read; nothing is built. + if src.keyOnly() { + return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil + } invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...) if a.Target != "" { invocation = append(invocation, "--target", a.Target) @@ -949,6 +1118,9 @@ func one(ctx context.Context, run Runner, publish Publisher, src.notPinned(a.Name + " resolves packages of its own at build time") } } + if src.keyOnly() { + return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil + } // The module's own packages first, where the compiler and the bundler resolve them from // (dependencies.go); nothing at all for a module whose package.json names only the SDK. if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil { @@ -992,6 +1164,9 @@ func one(ctx context.Context, run Runner, publish Publisher, say("package", "building and publishing %s (%s)", a.Name, a.Language) src.ownWhole() src.notPinned(a.Name + " is a package, built from what the registry holds when it is built") + if src.keyOnly() { + return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil + } reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say) if err != nil { return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err) @@ -1000,6 +1175,12 @@ func one(ctx context.Context, run Runner, publish Publisher, return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil case catalogue.ArtifactArchive: + if src.keyOnly() { + if a.URL == "" { + src.ownHas(cleanEntry(a.From) + "/**") + } + return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil + } var body []byte var err error if a.URL != "" { diff --git a/internal/builder/check.go b/internal/builder/check.go index 0a08d40c..bf0ce5bc 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -98,6 +98,11 @@ type CheckSpec struct { // and composed with it by the gate as one future state. The repository's own check is not run for a // group: each member's pull request runs its own. Group []GroupHead + // Builds are the modules the delivery plan moves, in the walk's order, and BuildWith what they are built with + // (novox/hq ADR 0350 §1): built from the merge the check judged into the scratch namespace once it has judged, + // for the walk to take. None for a group's composed check, whose members' own checks build. + Builds []CheckBuildSpec + BuildWith *CheckBuilding } // GroupHead is one other head of a delivery group's composed check. @@ -155,6 +160,8 @@ type CheckVerdict struct { // Members are a delivery group's other heads as the composed check judged them, each with the trunk // commit it was merged into (ADR 0323 §6). Members []JudgedHead + // Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not. + Built []CheckBuilt } // Layer is one layer of a check, judged. @@ -572,9 +579,38 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began) say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary, strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second)) + + // **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it + // judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again. + if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 { + if why := skipsTheBuilds(v); why != "" { + say("check", "%s", why) + } else { + began := time.Now() + v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, tree, judged.Merge, judged, forge, + func(step, message string) { + if log != nil { + log(step, redact.redact(message)) + } + }, say) + say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built), + time.Since(began).Round(time.Second)) + } + } return named(v), nil } +// builtCount is how many of a check's builds for its merge were built. +func builtCount(built []CheckBuilt) int { + n := 0 + for _, b := range built { + if b.NotBuilt == "" { + n++ + } + } + return n +} + // ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the // layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller // answers as the context's error. diff --git a/internal/builder/check_builds.go b/internal/builder/check_builds.go new file mode 100644 index 00000000..6c9cd37d --- /dev/null +++ b/internal/builder/check_builds.go @@ -0,0 +1,151 @@ +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The builds a merge check makes for its merge (novox/hq ADR 0350 §1): every artifact of every module the delivery +// plan moves, built from the merged tree the check judged into the scratch namespace, kept under its reuse key, so +// the walk of the merge commit takes it instead of building again. Built after the check has judged, and only when +// it did not fail: a change that fails is not merged, and nothing it built would be taken. + +// CheckBuildSpec is one module a check builds for its merge, in the walk's order (Tier). +type CheckBuildSpec struct { + Module string + // Repository, Path and Ref are where it is built from; Merged says from the merge the check made instead — the + // module is built from the repository checked, at the branch merged into. + Repository string + Path string + Ref string + Merged bool + Tier int + // Skip says why it cannot be built as the walk would build it: not built, and nothing after it. + Skip string +} + +// CheckBuilding is what a check builds with: where it publishes and keeps its builds, and what a build is handed. +type CheckBuilding struct { + Publish Publisher + Scratch Scratch + Held map[string]string + Npmrc Npmrc + Seats map[string]string +} + +// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or +// why it was not built. +type CheckBuilt struct { + Module string + Key string + Made []catalogue.Built + // NotBuilt says why it was not: no key, a build that failed, or a module after one of those. + NotBuilt string + Took time.Duration +} + +// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds. +func skipsTheBuilds(v CheckVerdict) string { + for _, l := range []*Layer{v.Gate, v.Repo} { + if l != nil && (l.Verdict == "fail" || l.Verdict == "error") { + return "the check did not pass, so nothing is built for its merge" + } + } + return "" +} + +// buildForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace, and keeps what it +// built under its keys. A module standing after one not built is not built either: what it stands on in the walk +// is not what it would stand on here. tree is the check's clone and merge the merge commit it made there. +func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string, + judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt { + if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil { + return nil + } + ref := merge + if ref == "" { + ref = spec.Ref + } + held := map[string]string{} + for k, v := range with.Held { + held[k] = v + } + var out []CheckBuilt + var kept []CheckBuild + stopAt, stopWhy := -1, "" + for _, b := range spec.Builds { + began := time.Now() + if stopAt >= 0 && b.Tier > stopAt { + out = append(out, CheckBuilt{Module: b.Module, NotBuilt: stopWhy}) + continue + } + if b.Skip != "" { + say("build", "%s not built for the merge: %s", b.Module, b.Skip) + out = append(out, CheckBuilt{Module: b.Module, NotBuilt: b.Skip}) + if stopAt < 0 || b.Tier < stopAt { + stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge" + } + continue + } + repository, at := b.Repository, b.Ref + if b.Merged { + repository, at = tree, ref + } + say("build", "for the merge: %s (tier %d)", b.Module, b.Tier) + var mirrored []string + result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at, + filepath.Join(workspace, "for-the-merge"), held, with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true}, + with.Seats) + built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)} + switch { + case errors.Is(err, errNoKey): + built.NotBuilt = err.Error() + case err != nil: + built.NotBuilt = "its build failed: " + err.Error() + case result.ReuseKey == "": + built.NotBuilt = "it has no reuse key: " + result.NoKey + } + if built.NotBuilt != "" { + say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt) + out = append(out, built) + if stopAt < 0 || b.Tier < stopAt { + stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge" + } + continue + } + built.Made = result.Built + for _, m := range result.Built { + held[b.Module+"/"+m.Name] = m.Reference + } + say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]), built.Took.Round(time.Second)) + out = append(out, built) + if len(result.Built) > 0 { + kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts, + Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, + Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()}) + } + } + _ = os.RemoveAll(filepath.Join(workspace, "for-the-merge")) + if len(kept) > 0 { + if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID, + time.Now().UTC(), kept); err != nil { + // Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict. + say("build", "the builds for the merge could not be kept, and the walk will build: %v", err) + if err := with.Scratch.Discard(ctx, kept); err != nil { + say("build", "and what they made could not all be let go of: %v", err) + } + for i := range out { + if out[i].NotBuilt == "" { + out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err) + } + } + } + } + return out +} diff --git a/internal/builder/reuse.go b/internal/builder/reuse.go new file mode 100644 index 00000000..ad832e48 --- /dev/null +++ b/internal/builder/reuse.go @@ -0,0 +1,539 @@ +package builder + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/artifacts" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// One build per change (novox/hq ADR 0350). +// +// **The merge check builds what the merge would move** (§1): every artifact of every module the delivery plan +// moves, from the merged tree it checks, into the artifact store's scratch namespace — never a module's own — +// kept under a **reuse key**: the source fingerprint as the build takes it (the module's tree, its contexts' trees, +// its bases and toolchains by digest; issue 280) with the builder's own version and, for an upstream artifact, the +// image it copies. **The walk of a trunk commit takes that build** (§2): before building, it takes the reuse key +// of the trunk commit's tree the same way, and where a check build holds the same key it copies it by digest into +// the module's namespace and answers with that, building nothing. Otherwise it builds and says why (§3): no check +// built it, the key differs (naming the part), or the check build is gone; a copy that fails is said and built. +// +// **What the key names is what a fresh build could differ by.** The tree is every file the recipe and the +// compiler see; the contexts are the other repositories it reads, by tree; the bases and toolchains are what it +// stands on, by digest; the builder's version is how it is built. A build whose source does not pin it — one that +// resolves packages from the mesh's registry when it is built, or copies an upstream image named by tag — has no +// key, and is always built. + +// Version is this builder's own version, as the reuse key takes it: the digest of the running executable, so any +// change to how it builds is a change of key (§1). Set once by the build seat's holder (SelfVersion); empty — no +// key, every build its own — when it cannot be said. +var Version string + +// SelfVersion is the version the reuse key takes: the running executable's digest, or the version stamped at build +// when the executable cannot be read. Empty when neither says anything ("development" says nothing: every unstamped +// builder would read as one). +func SelfVersion(stamped string) string { + if exe, err := os.Executable(); err == nil { + if f, err := os.Open(exe); err == nil { + defer f.Close() + sum := sha256.New() + if _, err := io.Copy(sum, f); err == nil { + return "exe:sha256:" + hex.EncodeToString(sum.Sum(nil)) + } + } + } + if stamped == "" || stamped == "development" { + return "" + } + return stamped +} + +// ScratchPrefix is the namespace a check builds into (novox/hq ADR 0239 §9, ADR 0350 §1): `scratch//`. +// Registration refuses a reference into it; only the walk's copy, in the module's own namespace, is registered. +const ScratchPrefix = "scratch/" + +// checksRepository holds one document per check that built: which builds it made, of which pull request, when. +// What the holder lets go of by (§5). +const checksRepository = ScratchPrefix + "checks" + +// reuseTagPrefix names a check build's record under its module's scratch repository: `r1-`. +const reuseTagPrefix = "r1-" + +// CheckBuildsKept is how long a check build is kept in the scratch namespace (§5). +const CheckBuildsKept = 7 * 24 * time.Hour + +// The parts of a reuse key, as the walk names the one that differs (§3). +const ( + partModule = "the module" + partTree = "the build source" + partBases = "a base image" + partBuilder = "the builder's version" + partContext = "a context" + partTool = "a toolchain" + partUpstream = "an upstream image" +) + +// reuseKey is a build's reuse key and the parts it was made from; Key empty, with why, when it has none. +type reuseKey struct { + Key string + Parts map[string]string + Why string +} + +// keyOf is the reuse key of what src says the build was made from, built by version (ADR 0350 §1). +func keyOf(src *sourceInputs, manifest catalogue.Manifest, version string) reuseKey { + if src == nil || src.unpinned != "" || src.tree == "" { + why := "its source does not pin the build" + if src != nil && src.unpinned != "" { + why = src.unpinned + } else if src != nil && src.tree == "" { + why = "its tree could not be named" + } + return reuseKey{Why: why} + } + if version == "" { + return reuseKey{Why: "the builder's version cannot be said"} + } + parts := map[string]string{partModule: src.module, partTree: src.tree, partBuilder: version} + var bases []string + seen := map[string]bool{} + for _, b := range src.bases { + if d := referenceDigest(b); !seen[d] { + seen[d] = true + bases = append(bases, d) + } + } + sort.Strings(bases) + parts[partBases] = strings.Join(bases, " ") + for a, t := range src.contexts { + parts[partContext+" "+a] = t + } + for a, t := range src.toolchains { + parts[partTool+" "+a] = t + } + // An upstream artifact is copied, not built: what it copies is in module.json, and so in the tree — but a tag + // names whatever its registry says today, so only an image pinned by digest is the same image tomorrow. + if manifest.Build != nil { + for _, a := range manifest.Build.Artifacts { + if a.Kind != catalogue.ArtifactUpstream { + continue + } + if !strings.Contains(a.From, "@sha256:") { + return reuseKey{Why: fmt.Sprintf("%s copies %s, named by tag, which can move", a.Name, a.From)} + } + parts[partUpstream+" "+a.Name] = a.From + } + } + lines := make([]string, 0, len(parts)) + for k, v := range parts { + lines = append(lines, k+"="+v) + } + sort.Strings(lines) + sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) + return reuseKey{Key: "r1:" + hex.EncodeToString(sum[:]), Parts: parts} +} + +// tagOf is the tag a key's record is kept under. +func tagOf(key string) string { return reuseTagPrefix + strings.TrimPrefix(key, "r1:") } + +// differing names the parts of two keys that differ, in a fixed order: what the walk says when it builds (§3). +func differing(was, now map[string]string) []string { + names := map[string]bool{} + for k := range was { + names[k] = true + } + for k := range now { + names[k] = true + } + var out []string + for k := range names { + if was[k] != now[k] { + out = append(out, k) + } + } + sort.Strings(out) + return out +} + +// PullRef names a pull request. +type PullRef struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number"` +} + +func (p PullRef) String() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) } + +// slug is the pull request as a tag's first part: owner.repo.number, every other character a dash. +func (p PullRef) slug() string { + clean := func(s string) string { + return strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' { + return r + } + return '-' + }, s) + } + return clean(p.Owner) + "." + clean(p.Repo) + "." + strconv.Itoa(p.Number) +} + +// CheckBuild is one module a check built into the scratch namespace, kept under its reuse key (§1). +type CheckBuild struct { + Key string `json:"key"` + Module string `json:"module"` + Parts map[string]string `json:"parts"` + // Made are its artifacts, in the scratch namespace. + Made []catalogue.Built `json:"made"` + // Pull is the pull request checked, Head its head and Trunk the trunk commit it was merged into; Check the + // check's ask; At when it was built. + Pull PullRef `json:"pull"` + Head string `json:"head,omitempty"` + Trunk string `json:"trunk,omitempty"` + Check string `json:"check"` + At time.Time `json:"at"` +} + +// Said is the check build as the walk names it: "the check of owner/repo#n at head H (trunk T)". +func (c CheckBuild) Said() string { + s := "the check of " + c.Pull.String() + if c.Head != "" { + s += " at head " + twelve(c.Head) + } + if c.Trunk != "" { + s += " (merged into trunk " + twelve(c.Trunk) + ")" + } + return s +} + +// checkIndex is one check's document under checksRepository: every build it made. +type checkIndex struct { + Pull PullRef `json:"pull"` + Check string `json:"check"` + At time.Time `json:"at"` + Builds []CheckBuild `json:"builds"` +} + +// Scratch is where check builds are kept and taken from: the artifact store's scratch namespace (ADR 0350). The +// Registry is one; a test's is a map. +type Scratch interface { + // Keep records a check's builds: each under its module and key, and the check's own document. + Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error + // Find is the check build kept under a module's key; false when none is. + Find(ctx context.Context, module, key string) (CheckBuild, bool, error) + // OfPulls are the check builds kept for those pull requests, newest first: what the walk names a differing + // key from. + OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) + // Take copies one artifact of a check build by digest into the module's own namespace, and answers it there. + Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) + // Forget lets go of every check build of a pull request, and Expire of every check older than the bound; + // each answers how many checks it let go of. + Forget(ctx context.Context, pull PullRef) (int, error) + Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) + // Discard lets go of what builds a check could not keep made: nothing would ever find them. + Discard(ctx context.Context, builds []CheckBuild) error +} + +// scratchPublisher is a Publisher into the scratch namespace: what a check builds is never under a module's +// own name (ADR 0239 §9). A base the build stands on is copied where every build copies it. +type scratchPublisher struct{ to Publisher } + +func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository string) (string, error) { + return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository) +} + +func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) { + return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest) +} + +func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) { + m, can := s.to.(Mirrorer) + if !can { + return "", errors.New("the artifact store here cannot copy an upstream image") + } + return m.MirrorImage(ctx, from, ScratchPrefix+repository) +} + +func (s scratchPublisher) MirrorBase(ctx context.Context, from, repository, former string) (string, error) { + if m, can := s.to.(BaseMirrorer); can { + return m.MirrorBase(ctx, from, repository, former) + } + if m, can := s.to.(Mirrorer); can { + return m.MirrorImage(ctx, from, repository) + } + return "", errors.New("the artifact store here cannot copy a base") +} + +// The registry as the scratch namespace. + +func (r Registry) store() artifacts.Store { + return artifacts.Store{Address: r.Address, HTTP: r.client()} +} + +const recordMedia = "application/vnd.novox.mesh.check-build.v1+json" + +// Keep puts each build's record under its module's scratch repository, by key, and the check's document. +func (r Registry) Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error { + store := r.store() + for _, b := range builds { + body, err := json.Marshal(b) + if err != nil { + return err + } + if _, err := store.PutTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key), recordMedia, body); err != nil { + return fmt.Errorf("keeping %s's check build: %w", b.Module, err) + } + } + body, err := json.Marshal(checkIndex{Pull: pull, Check: check, At: at, Builds: builds}) + if err != nil { + return err + } + if _, err := store.PutTagged(ctx, checksRepository, pull.slug()+"."+check, recordMedia, body); err != nil { + return fmt.Errorf("keeping which builds the check of %s made: %w", pull, err) + } + return nil +} + +// Find reads the record kept under a module's key. +func (r Registry) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) { + body, _, err := r.store().GetTagged(ctx, ScratchPrefix+module, tagOf(key)) + if errors.Is(err, artifacts.ErrNoTag) { + return CheckBuild{}, false, nil + } + if err != nil { + return CheckBuild{}, false, err + } + var b CheckBuild + if err := json.Unmarshal(body, &b); err != nil { + return CheckBuild{}, false, fmt.Errorf("%s's check build under %s is not one: %w", module, tagOf(key), err) + } + if b.Key != key || b.Module != module { + return CheckBuild{}, false, nil + } + return b, true, nil +} + +// checks are the check documents kept, by tag; only those of the pull requests named when any are. +func (r Registry) checks(ctx context.Context, pulls []PullRef) (map[string]checkIndex, error) { + store := r.store() + tags, err := store.Tags(ctx, checksRepository) + if err != nil { + return nil, err + } + wanted := map[string]PullRef{} + for _, p := range pulls { + wanted[p.slug()+"."] = p + } + out := map[string]checkIndex{} + for _, tag := range tags { + if len(pulls) > 0 { + match := false + for prefix := range wanted { + match = match || strings.HasPrefix(tag, prefix) + } + if !match { + continue + } + } + body, _, err := store.GetTagged(ctx, checksRepository, tag) + if errors.Is(err, artifacts.ErrNoTag) { + continue + } + if err != nil { + return nil, err + } + var c checkIndex + if err := json.Unmarshal(body, &c); err != nil { + continue // not a document this wrote: left to nobody, said by nothing — it names no build + } + if len(pulls) > 0 { + if p := wanted[c.Pull.slug()+"."]; p != c.Pull { + continue // a slug two pull requests share: only the one named + } + } + out[tag] = c + } + return out, nil +} + +// OfPulls are the check builds of those pull requests, newest first. +func (r Registry) OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) { + if len(pulls) == 0 { + return nil, nil + } + checks, err := r.checks(ctx, pulls) + if err != nil { + return nil, err + } + var out []CheckBuild + for _, c := range checks { + out = append(out, c.Builds...) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].At.After(out[j].At) }) + return out, nil +} + +// Take copies one artifact of a check build into the module's namespace by digest: an image's manifests as they +// are, so the digest registered is the digest the check built; an archive's blob mounted across, and held. +func (r Registry) Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) { + repository := module + "/" + made.Name + from := ScratchPrefix + repository + if made.Digest != "" || strings.Contains(made.Reference, "/blobs/") { + digest := made.Digest + if digest == "" { + _, digest, _ = strings.Cut(made.Reference, "/blobs/") + } + where := upstream{base: "http://" + r.Address, repository: from, reference: digest} + if err := r.copyBlob(ctx, &source{client: r.client(), mountFrom: from}, where, digest, repository); err != nil { + return catalogue.Built{}, err + } + size, err := r.blobSize(ctx, repository, digest) + if err != nil { + return catalogue.Built{}, err + } + if err := r.hold(ctx, repository, digest, int(size)); err != nil { + return catalogue.Built{}, err + } + return catalogue.Built{Name: made.Name, Kind: made.Kind, Digest: digest, Launchers: made.Launchers, + Reference: "http://" + r.Address + "/v2/" + repository + "/blobs/" + digest}, nil + } + _, digest, pinned := strings.Cut(made.Reference, "@") + if !pinned || !strings.HasPrefix(digest, "sha256:") { + return catalogue.Built{}, fmt.Errorf("%s's check build %s is not pinned by digest", module, made.Reference) + } + reference, err := r.MirrorBase(ctx, r.Address+"/"+from+"@"+digest, repository, from) + if err != nil { + return catalogue.Built{}, err + } + if !strings.HasSuffix(reference, "@"+digest) { + return catalogue.Built{}, fmt.Errorf("%s's check build was copied as %s, not as %s", module, reference, digest) + } + return catalogue.Built{Name: made.Name, Kind: made.Kind, Reference: reference, Launchers: made.Launchers}, nil +} + +// blobSize is the size the store says a blob has. +func (r Registry) blobSize(ctx context.Context, repository, digest string) (int64, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+r.Address+"/v2/"+repository+"/blobs/"+digest, nil) + if err != nil { + return 0, err + } + response, err := r.client().Do(request) + if err != nil { + return 0, err + } + response.Body.Close() + if response.StatusCode != http.StatusOK || response.ContentLength < 0 { + return 0, fmt.Errorf("%s/blobs/%s: %s", repository, digest, response.Status) + } + return response.ContentLength, nil +} + +// Forget lets go of every check build of a pull request (§5): what each check built in the scratch namespace, the +// record under its key when it is still that check's, and the check's document. +func (r Registry) Forget(ctx context.Context, pull PullRef) (int, error) { + checks, err := r.checks(ctx, []PullRef{pull}) + if err != nil { + return 0, err + } + return r.letGo(ctx, checks) +} + +// Expire lets go of every check older than the bound (§5). +func (r Registry) Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) { + checks, err := r.checks(ctx, nil) + if err != nil { + return 0, err + } + old := map[string]checkIndex{} + for tag, c := range checks { + if now.Sub(c.At) > older { + old[tag] = c + } + } + return r.letGo(ctx, old) +} + +// Discard lets go of builds no check document names. +func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error { + _, err := r.letGo(ctx, map[string]checkIndex{"": {Builds: builds}}) + return err +} + +// letGo lets go of each check's builds, then its document; a check whose builds could not all be let go of keeps +// its document, so the next sweep asks again. +func (r Registry) letGo(ctx context.Context, checks map[string]checkIndex) (int, error) { + store := r.store() + gone := 0 + var failed []string + for tag, c := range checks { + ok := true + for _, b := range c.Builds { + for _, m := range b.Made { + reference := catalogue.Recorded(m.Reference) + if path, kept := catalogue.InArtifactStore(reference); !kept || !strings.HasPrefix(path, ScratchPrefix) { + continue // never anything outside the scratch namespace + } + if err := store.LetGo(ctx, reference); err != nil && !errors.Is(err, artifacts.Gone) { + ok = false + failed = append(failed, fmt.Sprintf("%s: %v", m.Reference, err)) + } + } + // The record under the key, only while it is still this check's: a later check of the same key + // replaced it, and that one is its own check's to let go of. + if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found && kept.Check == c.Check { + if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) { + ok = false + failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err)) + } + } + } + if !ok { + continue + } + if tag == "" { + gone++ + continue + } + if err := store.DeleteTagged(ctx, checksRepository, tag); err != nil && !errors.Is(err, artifacts.Gone) { + failed = append(failed, fmt.Sprintf("%s: %v", tag, err)) + continue + } + gone++ + } + if len(failed) > 0 { + sort.Strings(failed) + return gone, fmt.Errorf("not let go of: %s", strings.Join(failed, "; ")) + } + return gone, nil +} + +// Reused is what the walk took from a check instead of building (§2, §4). +type Reused struct { + From CheckBuild + // Made are the copies in the module's namespace: what is registered. + Made []catalogue.Built +} + +// Words is the walk's word for a module it took: "reused from ", with each digest (§4). +func (r Reused) Words() string { + var digests []string + for _, m := range r.Made { + d := m.Digest + if d == "" { + _, d, _ = strings.Cut(m.Reference, "@") + } + digests = append(digests, m.Name+" "+d) + } + return "reused from " + r.From.Said() + ": " + strings.Join(digests, ", ") +} diff --git a/internal/builder/reuse_test.go b/internal/builder/reuse_test.go new file mode 100644 index 00000000..a3e3c1f5 --- /dev/null +++ b/internal/builder/reuse_test.go @@ -0,0 +1,541 @@ +package builder + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// One build per change (novox/hq ADR 0350): the merge check builds what the merge would move into the scratch +// namespace under its reuse key, and the walk of the trunk commit takes that build by digest instead of building. + +// aScratch is the scratch namespace in memory: check builds by module and key, and what the walk took. +type aScratch struct { + builds map[string]CheckBuild + took []string + failTake bool +} + +func newScratch() *aScratch { return &aScratch{builds: map[string]CheckBuild{}} } + +func (s *aScratch) Keep(_ context.Context, _ PullRef, _ string, _ time.Time, builds []CheckBuild) error { + for _, b := range builds { + s.builds[b.Module+" "+b.Key] = b + } + return nil +} + +func (s *aScratch) Find(_ context.Context, module, key string) (CheckBuild, bool, error) { + b, ok := s.builds[module+" "+key] + return b, ok, nil +} + +func (s *aScratch) OfPulls(_ context.Context, pulls []PullRef) ([]CheckBuild, error) { + var out []CheckBuild + for _, b := range s.builds { + if slices.Contains(pulls, b.Pull) { + out = append(out, b) + } + } + return out, nil +} + +func (s *aScratch) Take(_ context.Context, module string, made catalogue.Built) (catalogue.Built, error) { + if s.failTake { + return catalogue.Built{}, errors.New("the store refused the mount") + } + s.took = append(s.took, made.Reference) + made.Reference = strings.Replace(made.Reference, ScratchPrefix, "", 1) + return made, nil +} + +func (s *aScratch) Forget(context.Context, PullRef) (int, error) { return 0, nil } +func (s *aScratch) Expire(context.Context, time.Duration, time.Time) (int, error) { return 0, nil } +func (s *aScratch) Discard(context.Context, []CheckBuild) error { return nil } + +const reusedModule = `{"module":"meshboard","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile"}, + {"name":"look","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"svc","type":"container","name":"meshboard","artifact":"server"}, + {"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}` + +// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk. +func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) { + t.Helper() + r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule, + "mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}} + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" { + r.ran = append(r.ran, name+" "+strings.Join(args, " ")) + return "origin/main\n", nil + } + return r.run(ctx, dir, name, args...) + } + return r, run, t.TempDir() +} + +func dockerBuilt(r *recorded) bool { + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") { + return true + } + } + return false +} + +var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7} + +// aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace. +func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { + t.Helper() + r, run, workspace := aModuleAt(t, tree) + spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, + Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", + Ref: "main"}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "", + CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil)) + if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { + t.Fatalf("the check did not build the module for its merge: %+v", built) + } + if !dockerBuilt(r) { + t.Fatal("the check built no image") + } + for repository := range r.images { + if !strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the check published %s outside the scratch namespace", repository) + } + } + for repository := range r.archives { + if !strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the check published %s outside the scratch namespace", repository) + } + } + return built[0] +} + +func withVersion(t *testing.T, v string) { + t.Helper() + was := Version + Version = v + t.Cleanup(func() { Version = was }) +} + +// A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest +// and registers that digest. +func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { + t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives) + } + if got.Reused == nil || got.Why != "" { + t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why) + } + if got.ReuseKey != checked.Key { + t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key) + } + if len(scratch.took) != 2 { + t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took) + } + for _, made := range checked.Made { + registered := strings.Replace(made.Reference, ScratchPrefix, "", 1) + found := false + for _, b := range got.Built { + found = found || (b.Reference == registered && b.Digest == made.Digest) + } + if !found { + t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference) + } + } + if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) { + t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image) + } + words := got.Reused.Words() + if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") || + !strings.Contains(words, "sha256:") { + t.Fatalf("the walk says %q", words) + } +} + +// A differing key builds, and says which part differs. +func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-two") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || got.Reused != nil { + t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused) + } + if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) { + t.Fatalf("the walk says %q, not which part of the key differs", got.Why) + } + + // And a builder of another version builds too, saying so. + withVersion(t, "builder-2") + r, run, workspace = aModuleAt(t, "tree-one") + got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) { + t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why) + } + + // No check of the walk's pull requests built it: said. + r, run, workspace = aModuleAt(t, "tree-one") + got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") { + t.Fatalf("the walk says %q", got.Why) + } +} + +// A copy that fails is said and falls back to a build; it never fails the walk. +func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + aCheckBuilt(t, scratch, "tree-one") + scratch.failTake = true + + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a failed copy failed the walk: %v", err) + } + if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 { + t.Fatal("a failed copy did not fall back to a build") + } + if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") { + t.Fatalf("the walk says %q", got.Why) + } + for repository := range r.images { + if strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the walk's own build went into the scratch namespace: %s", repository) + } + } +} + +// Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit +// off the trunk never takes one. +func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-one") + if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, + Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || len(scratch.took) > 0 { + t.Fatal("a build not asked to reuse took a check's build") + } + + r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it + got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch}, "https://forge.invalid/meshboard.git", + "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") { + t.Fatalf("a commit off the trunk took a check's build: %q", got.Why) + } +} + +// A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing. +func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) { + if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" { + t.Fatal("a check whose repository layer failed builds for its merge") + } + if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" { + t.Fatal("a check that passed with a warning builds nothing for its merge") + } + r, run, workspace := aModuleAt(t, "tree-one") + if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()}, + workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 { + t.Fatalf("a check moving no module built %v, ran %v", built, r.ran) + } +} + +// A module whose source does not pin its build has no key and is not built by the check; what stands after it +// is not built either. +func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) { + withVersion(t, "builder-1") + r, run, workspace := aModuleAt(t, "tree-one") + r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[ + {"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}` + spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{ + {Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0}, + {Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "", + CheckVerdict{}, GitCredential{}, nil, logging(nil)) + if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") || + !strings.Contains(built[1].NotBuilt, "stands after meshboard") { + t.Fatalf("built %+v", built) + } + if len(r.images) > 0 { + t.Fatalf("a module without a key was published: %v", r.images) + } +} + +// What the key names: every input a fresh build could differ by. +func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) { + src := newSourceInputs("m") + src.tree = "t1" + src.bases = []string{"store:5000/b/x@sha256:" + strings.Repeat("b", 64)} + src.contexts["server"] = "ctx1" + src.toolchains["tools"] = "go sha256:c 1234" + manifest := catalogue.Manifest{Module: "m"} + base := keyOf(src, manifest, "v1") + if base.Key == "" { + t.Fatalf("no key: %s", base.Why) + } + for name, change := range map[string]func(*sourceInputs) string{ + partTree: func(s *sourceInputs) string { s.tree = "t2"; return "v1" }, + partBases: func(s *sourceInputs) string { + s.bases = []string{"other@sha256:" + strings.Repeat("c", 64)} + return "v1" + }, + partContext: func(s *sourceInputs) string { s.contexts["server"] = "ctx2"; return "v1" }, + partTool: func(s *sourceInputs) string { s.toolchains["tools"] = "go sha256:d 1234"; return "v1" }, + partBuilder: func(*sourceInputs) string { return "v2" }, + } { + s := src.copy() + version := change(s) + if k := keyOf(s, manifest, version); k.Key == base.Key { + t.Errorf("a change of %s left the key the same", name) + } + } + // A base copied to another address is the same base: the key names it by digest. + moved := src.copy() + moved.bases = []string{"elsewhere:5000/scratch/b/x@sha256:" + strings.Repeat("b", 64)} + if keyOf(moved, manifest, "v1").Key != base.Key { + t.Error("the same base at another address changed the key") + } + if k := keyOf(src, manifest, ""); k.Key != "" { + t.Error("a builder whose version cannot be said keyed its build") + } + unpinned := src.copy() + unpinned.notPinned("it resolves packages from the mesh's registry at build time") + if k := keyOf(unpinned, manifest, "v1"); k.Key != "" || !strings.Contains(k.Why, "registry") { + t.Errorf("a build its source does not pin was keyed: %+v", k) + } +} + +// aTaggingRegistry is the in-memory registry with tags and their listing, as the store's scratch namespace needs. +type aTaggingRegistry struct { + *aRegistryOfRepositories + tags map[string]map[string]string // repository → tag → manifest digest +} + +func (m *aTaggingRegistry) serve() http.Handler { + under := m.handler() + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if repository, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list"); ok { + m.mu.Lock() + var tags []string + for tag := range m.tags[repository] { + tags = append(tags, tag) + } + m.mu.Unlock() + if tags == nil { + w.WriteHeader(http.StatusNotFound) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"name": repository, "tags": tags}) + return + } + repository, kind, rest := splitPath(r.URL.Path) + if kind == "blobs" && r.Method == http.MethodHead { + m.mu.Lock() + body, linked := m.blobs[rest], m.links[repository][rest] + m.mu.Unlock() + if !linked { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + return + } + if kind != "manifests" { + under.ServeHTTP(w, r) + return + } + if !strings.HasPrefix(rest, "sha256:") { + m.mu.Lock() + switch r.Method { + case http.MethodPut: + body, _ := readAll(r) + if m.tags[repository] == nil { + m.tags[repository] = map[string]string{} + } + m.tags[repository][rest] = digestOf(body) + m.manifests[repository+"@"+digestOf(body)] = body + m.mu.Unlock() + w.WriteHeader(http.StatusCreated) + return + default: + digest, ok := m.tags[repository][rest] + body, held := m.manifests[repository+"@"+digest] + m.mu.Unlock() + if !ok || !held { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Docker-Content-Digest", digest) + w.Header().Set("Content-Type", mediaTypeOf(body)) + if r.Method == http.MethodGet { + _, _ = w.Write(body) + return + } + w.WriteHeader(http.StatusOK) + return + } + } + if r.Method == http.MethodDelete { + m.mu.Lock() + for _, tags := range m.tags { + for tag, digest := range tags { + if digest == rest { + delete(tags, tag) + } + } + } + m.mu.Unlock() + } + under.ServeHTTP(w, r) + }) +} + +// A check's builds go after seven days, and when its pull request closes or is merged; the walk's copy in the +// module's own namespace stays. +func TestCheckBuildsExpireAfterSevenDaysAndGoWhenTheirPullRequestCloses(t *testing.T) { + m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}} + server := httptest.NewServer(m.serve()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + registry := Registry{Address: address} + ctx := t.Context() + now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) + + image := func(repository, body string) string { + digest := digestOf([]byte(body)) + m.manifests[repository+"@"+digest] = []byte(body) + return address + "/" + repository + "@" + digest + } + check := func(pull PullRef, id string, at time.Time, key string) string { + reference := image("scratch/meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"`+id+`"}`) + build := CheckBuild{Key: key, Module: "meshboard", Made: []catalogue.Built{{Name: "server", Kind: "image", + Reference: reference}}, Pull: pull, Check: id, At: at} + if err := registry.Keep(ctx, pull, id, at, []CheckBuild{build}); err != nil { + t.Fatal(err) + } + return reference + } + held := func(reference string) bool { + _, digest, _ := strings.Cut(reference, "@") + _, ok := m.manifests["scratch/meshboard/server@"+digest] + return ok + } + other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8} + old := check(aPull, "build-1", now.Add(-8*24*time.Hour), "r1:"+strings.Repeat("1", 64)) + fresh := check(aPull, "build-2", now.Add(-time.Hour), "r1:"+strings.Repeat("2", 64)) + others := check(other, "build-3", now.Add(-time.Hour), "r1:"+strings.Repeat("3", 64)) + walked := image("meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"walk"}`) + + if _, found, err := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("2", 64)); err != nil || !found { + t.Fatalf("a check build kept is not found by its key: %v %v", found, err) + } + n, err := registry.Expire(ctx, CheckBuildsKept, now) + if err != nil || n != 1 { + t.Fatalf("expired %d checks (%v), want the one eight days old", n, err) + } + if held(old) || !held(fresh) || !held(others) { + t.Fatalf("after the week's sweep: old %v, fresh %v, another's %v", held(old), held(fresh), held(others)) + } + if _, found, _ := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("1", 64)); found { + t.Fatal("the expired check build's record is still found") + } + + n, err = registry.Forget(ctx, aPull) + if err != nil || n != 1 { + t.Fatalf("forgot %d checks of %s (%v)", n, aPull, err) + } + if held(fresh) || !held(others) { + t.Fatalf("after %s closed: its build held %v, another pull request's %v", aPull, held(fresh), held(others)) + } + if _, ok := m.manifests["meshboard/server@"+strings.SplitN(walked, "@", 2)[1]]; !ok { + t.Fatal("letting go of check builds took the walk's copy in the module's namespace") + } + if left, _ := registry.OfPulls(ctx, []PullRef{aPull}); len(left) != 0 { + t.Fatalf("%s still has check builds: %+v", aPull, left) + } +} + +// The walk copies a check build by digest: the digest registered is the digest the check built, and the copy is +// in the module's own namespace. +func TestAChecksBuildIsCopiedByDigestIntoTheModulesNamespace(t *testing.T) { + m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}} + server := httptest.NewServer(m.serve()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + registry := Registry{Address: address} + + layer := []byte("layer") + m.blobs[digestOf(layer)] = layer + m.link("scratch/meshboard/server", digestOf(layer)) + manifest := []byte(`{"mediaType":"` + mediaManifestOCI + `","layers":[{"digest":"` + digestOf(layer) + `","size":5}]}`) + m.manifests["scratch/meshboard/server@"+digestOf(manifest)] = manifest + archive := []byte("archive") + m.blobs[digestOf(archive)] = archive + m.link("scratch/meshboard/look", digestOf(archive)) + + got, err := registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "server", Kind: "image", + Reference: address + "/scratch/meshboard/server@" + digestOf(manifest)}) + if err != nil { + t.Fatal(err) + } + if got.Reference != address+"/meshboard/server@"+digestOf(manifest) { + t.Fatalf("copied as %s", got.Reference) + } + if m.uploads != 0 || m.mounts != 1 { + t.Fatalf("the image's layer was moved (%d uploads, %d mounts), not mounted", m.uploads, m.mounts) + } + got, err = registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "look", Kind: "archive", + Digest: digestOf(archive), Reference: "http://" + address + "/v2/scratch/meshboard/look/blobs/" + digestOf(archive)}) + if err != nil { + t.Fatal(err) + } + if got.Digest != digestOf(archive) || got.Reference != "http://"+address+"/v2/meshboard/look/blobs/"+digestOf(archive) || + !m.links["meshboard/look"][digestOf(archive)] { + t.Fatalf("the archive was copied as %+v", got) + } +} diff --git a/internal/builder/source.go b/internal/builder/source.go index 7f14e9fe..7d9b9c0e 100644 --- a/internal/builder/source.go +++ b/internal/builder/source.go @@ -64,6 +64,43 @@ type sourceInputs struct { // contextsAtHead is false once a context was cloned at something other than its trunk's head: its // closure is not the one the next merge there meets, and the build says no build source. contextsAtHead bool + // onlyKey says the inputs are being taken for a reuse key (novox/hq ADR 0350 §2): each artifact goes as far + // as what its source fingerprint reads, and nothing is built or published. + onlyKey bool +} + +// keyOnly is whether these inputs are only being taken for a reuse key. +func (s *sourceInputs) keyOnly() bool { return s != nil && s.onlyKey } + +// copy is the inputs taken so far, apart: a walk's key is taken from them without touching the build's own. +func (s *sourceInputs) copy() *sourceInputs { + c := newSourceInputs(s.module) + c.tree, c.unpinned, c.prefix, c.ownIsAll, c.contextsAtHead = s.tree, s.unpinned, s.prefix, s.ownIsAll, s.contextsAtHead + c.bases = append([]string{}, s.bases...) + for k, v := range s.contexts { + c.contexts[k] = v + } + for k, v := range s.toolchains { + c.toolchains[k] = v + } + for k, v := range s.own { + c.own[k] = v + } + for k, v := range s.read { + if v == nil { + c.read[k] = nil + continue + } + set := map[string]bool{} + for e := range v { + set[e] = true + } + c.read[k] = set + } + for k, v := range s.readAs { + c.readAs[k] = v + } + return c } // contextOffHead says a context was not its trunk's head. diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 8b13beae..673f62f4 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -362,6 +362,12 @@ var ControllerVerbs = []Verb{ "register": "\"true\" to register what it builds", "older": "\"true\", with register: even though a newer build of the module is registered", }, []string{"id"}, "register", "older")}, + // novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged. + {Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " + + "the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " + + "when the walk answering its merge ends, and any check build goes after a week.", + Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository", + "number": "the pull request's number"}, []string{"owner", "repo", "number"})}, {Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " + "announces it failed, killed by hand — settled, never handed to another machine.", Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})}, diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index c17344a0..d3e4aa11 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -286,6 +286,9 @@ type PlanModule struct { FirstAt *time.Time `json:"first_at,omitempty"` Commit string `json:"commit,omitempty"` Why string `json:"why,omitempty"` + // Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from ", naming the pull + // request's head, the trunk commit and the digests, or "built: ". Said in `plan-moved` and kept here. + Made string `json:"made,omitempty"` // Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan // matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt // its module's name included. Empty in a plan from before it was kept, which is matched by diff --git a/internal/link/build.go b/internal/link/build.go index 8262b8b9..72640a2a 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -92,6 +92,24 @@ type BuildRequest struct { // facts snapshot, raises the throwaway stores the check needs, runs the repository's own // merge-check.sh and answers its verdict. Nothing is built, published or registered. Check *CheckRequest `json:"check,omitempty"` + // Reuse makes this ask a walk's build of a trunk commit (novox/hq ADR 0350 §2): the builder takes the reuse key + // of the commit's tree and, where a check build holds the same key, copies it by digest into the module's + // namespace instead of building. Pulls are the pull requests the walk answers, whose checks name the part of a + // differing key. Nil for every other ask — a check, a dry run, `build --ref`, `rebuild`, `replay` — which never + // takes a check's build. A builder from before it ignores it and builds. + Reuse *ReuseAsk `json:"reuse,omitempty"` +} + +// ReuseAsk is a walk's leave to take a check's build (novox/hq ADR 0350 §2). +type ReuseAsk struct { + Pulls []PullRef `json:"pulls,omitempty"` +} + +// PullRef names a pull request. +type PullRef struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number"` } // CheckRequest is what a merge check needs beyond the repository and its head. @@ -128,6 +146,34 @@ type CheckRequest struct { // repository's own check is not run for a group: each member's pull request runs its own. Group string `json:"group,omitempty"` Members []GroupMember `json:"members,omitempty"` + // Builds are the modules the delivery plan moves, in the walk's order (novox/hq ADR 0350 §1): the check builds + // each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to + // take. A build seat from before it builds nothing, and the walk builds as before. + Builds []CheckBuild `json:"builds,omitempty"` +} + +// CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from +// the repository checked, at the branch merged into — else from Repository at Ref, as the walk would. +type CheckBuild struct { + Module string `json:"module"` + Repository string `json:"repository,omitempty"` + Path string `json:"path,omitempty"` + Ref string `json:"ref,omitempty"` + Merged bool `json:"merged,omitempty"` + Tier int `json:"tier"` + // Skip says why the check cannot build it as the walk would — a module that reads the repository checked + // through a context, which a check cannot hand the merge — and nothing after it is built either. + Skip string `json:"skip,omitempty"` +} + +// CheckBuilt is one build a check made for its merge, as its verdict lists it (novox/hq ADR 0350 §1): the reuse key +// and the artifacts, in the scratch namespace — or why it was not built. +type CheckBuilt struct { + Module string `json:"module"` + Key string `json:"key,omitempty"` + Made []MadeArtifact `json:"made,omitempty"` + NotBuilt string `json:"not-built,omitempty"` + Took string `json:"took,omitempty"` } // GroupMember is one other head of a delivery group's composed check. @@ -175,6 +221,8 @@ type CheckOutcome struct { Trunk string `json:"trunk,omitempty"` Beside map[string]string `json:"beside,omitempty"` Members []CheckedMember `json:"members,omitempty"` + // Built are the builds it made for its merge (novox/hq ADR 0350 §1). + Built []CheckBuilt `json:"built,omitempty"` } // SourceOnSeat names a repository by the seat whose holder serves it and its path there. @@ -264,6 +312,14 @@ type BuildResult struct { // Failed is why, when it did. Failed string `json:"failed,omitempty"` + // ReuseKey is the build's reuse key (novox/hq ADR 0350 §1). Reused, on a walk's build that took a check's build + // instead of building, is "reused from " with its digests, and ReusedFrom which; BuiltWhy, on one + // that built, why it took none (§3, §4). Both empty from a builder that predates it. + ReuseKey string `json:"reuse-key,omitempty"` + Reused string `json:"reused,omitempty"` + ReusedFrom *ReusedFrom `json:"reused-from,omitempty"` + BuiltWhy string `json:"built-why,omitempty"` + // Source is the request's, echoed: the seat form of the repository, for whoever registers. Source *SourceOnSeat `json:"source,omitempty"` @@ -300,3 +356,25 @@ type MadeArtifact struct { Kind string `json:"kind"` Reference string `json:"reference"` } + +// ReusedFrom is the check build a walk's build took (novox/hq ADR 0350 §4): its pull request, the head and the trunk +// commit the check merged it into, the check's ask, and its key. +type ReusedFrom struct { + Pull PullRef `json:"pull"` + Head string `json:"head,omitempty"` + Trunk string `json:"trunk,omitempty"` + Check string `json:"check,omitempty"` + Key string `json:"key"` +} + +// MadeWords is what a walk says a module's build did (novox/hq ADR 0350 §4): "reused from ", or "built: +// ". A build seat from before it says neither, and that is said. +func (r BuildResult) MadeWords() string { + switch { + case r.Reused != "": + return r.Reused + case r.BuiltWhy != "": + return "built: " + r.BuiltWhy + } + return "built: the build seat did not say whether a check's build could be taken (it predates ADR 0350)" +} diff --git a/internal/link/events.go b/internal/link/events.go index f019b656..48b7ece6 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -290,6 +290,9 @@ type Checked struct { Trunk string `json:"trunk,omitempty"` Beside map[string]string `json:"beside,omitempty"` Took string `json:"took,omitempty"` + // Built are the builds the check made for its merge, each with its reuse key and digests, or why not + // (novox/hq ADR 0350 §1): what the walk of the merge takes instead of building. + Built []CheckBuilt `json:"built,omitempty"` } // CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq diff --git a/module.json b/module.json index 4ceb785d..1b84a8c4 100644 --- a/module.json +++ b/module.json @@ -76,6 +76,7 @@ "rebuild", "replay", "kill", + "forget-checks", "pause", "resume", "hand-act", diff --git a/testdata/store-tests b/testdata/store-tests index 0640e2b3..c1bb4df2 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -59,3 +59,4 @@ ./cmd/mesh-controller TestACrossingSendSupersedesASettingsGate ./cmd/mesh-controller TestASettingsWalkFailsOnARefusalThatCannotClear ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName +./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne