Build a change once: the check builds what its merge moves, the walk takes it by key

A merge built every moved module a second time although its check had judged
the same tree (issue 649). The check now builds into the scratch namespace
under a reuse key (source fingerprint plus the builder's own digest), and the
walk of the trunk commit copies a build of the same key by digest instead of
building, saying per module why when it cannot (novox/hq ADR 0350).

cmd/mesh-builder/holder.go

@@ -41,6 +41,9 @@ type holder struct {
 	say func(link.HolderState) error
 	// remove runs what a kill needs outside the build: the containers left behind.
 	remove builder.Runner
+	// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
+	// this machine publishes to no artifact store.
+	scratch builder.Scratch
 
 	mu      sync.Mutex
 	paused  bool
@@ -131,6 +134,34 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
 	}
 }
 
+// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
+// after. Every holder sweeps the one store; what one let go of, another finds gone.
+func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
+	sweep := func() {
+		if h.scratch == nil {
+			return
+		}
+		n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
+		switch {
+		case err != nil:
+			fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
+		case n > 0:
+			fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
+		}
+	}
+	sweep()
+	tick := time.NewTicker(every)
+	defer tick.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+			sweep()
+		}
+	}
+}
+
 // begin records the build this machine took, with the cancel that ends it.
 func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
 	r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}

cmd/mesh-builder/main.go

@@ -97,6 +97,9 @@ func run() error {
 	if err := notReady(); err != nil {
 		return err
 	}
+	// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
+	// differently never takes a build another one made.
+	builder.Version = builder.SelfVersion(version)
 	credential, err := brokerFrom()
 	if err != nil {
 		return err
@@ -174,6 +177,9 @@ func run() error {
 
 	fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
 	publisher := builder.Registry{Address: registry, Run: builder.Command}
+	// Check builds are let go of after a week (novox/hq ADR 0350 §5).
+	h.scratch = publisher
+	go h.expireChecks(ctx, time.Hour, time.Now)
 
 	return machine.Take(ctx, func(ctx context.Context, work link.Build) {
 		answer(ctx, publisher, on, workspace, work, h)
@@ -262,6 +268,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 	if err == nil {
 		say("packages", saysRegistry(npmrc))
 	}
+	registryHere, isRegistry := publisher.(builder.Registry)
 	var built builder.Result
 	if request.Check != nil {
 		// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
@@ -271,12 +278,21 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		if r, ok := publisher.(builder.Registry); ok {
 			registry = r.Address
 		}
+		spec := checkSpecOf(request)
+		// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
+		// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
+		// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
+		// at build time has no reuse key anyway, so nothing is lost by refusing it here.
+		if isRegistry && len(spec.Builds) > 0 {
+			spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
+				Seats: request.Seats}
+		}
 		var v builder.CheckVerdict
-		v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
+		v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
 		if err == nil {
 			result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
 				Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
-				Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
+				Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built)}
 			// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
 			for _, m := range v.Members {
 				result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
@@ -289,9 +305,20 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		// after a clone that then fails at npm ci.
 		// Every container it starts is labelled with its id, so a kill finds what outlived the
 		// docker client (ADR 0219).
-		built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
-			request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
-			forgeFrom(), say, request.Seats)
+		if request.Reuse != nil && isRegistry && !request.DryRun {
+			// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
+			reuse := builder.Reuse{Scratch: registryHere}
+			for _, p := range request.Reuse.Pulls {
+				reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
+			}
+			built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
+				request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
+				forgeFrom(), say, request.Seats)
+		} else {
+			built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
+				request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
+				forgeFrom(), say, request.Seats)
+		}
 	}
 	// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
 	result.Mirrored = built.Mirrored
@@ -327,6 +354,19 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 			}
 			result.Against = built.Against
 			result.SourceFingerprint = built.Source
+			// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
+			result.ReuseKey = built.ReuseKey
+			if built.Reused != nil {
+				f := built.Reused.From
+				result.Reused = built.Reused.Words()
+				result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
+					Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
+			} else if request.Reuse != nil {
+				result.BuiltWhy = built.Why
+				if result.BuiltWhy == "" {
+					result.BuiltWhy = "it has no artifact to build"
+				}
+			}
 			result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
 			for _, r := range built.Read {
 				result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
@@ -377,6 +417,10 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
 	for dir, b := range c.Beside {
 		spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
 	}
+	for _, b := range c.Builds {
+		spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
+			Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
+	}
 	for _, m := range c.Members {
 		spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
 			Ref: m.Ref, Paths: m.Paths, Base: m.Base})
@@ -384,6 +428,19 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
 	return spec
 }
 
+// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
+func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
+	var out []link.CheckBuilt
+	for _, b := range built {
+		c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String()}
+		for _, m := range b.Made {
+			c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
+		}
+		out = append(out, c)
+	}
+	return out
+}
+
 // layerOf is one layer of a check as the outcome carries it.
 func layerOf(l *builder.Layer) *link.CheckLayer {
 	if l == nil {