Merge pull request 'A module moves only when its program changes (hq issue 721)' (#336) from fix/a-module-moves-only-when-its-program-changes into main

cmd/mesh-builder/main.go

@@ -329,7 +329,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		// at build time has no reuse key anyway, so nothing is lost by refusing it here.
 		if isRegistry && len(spec.Builds) > 0 {
 			spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
-				Seats: request.Seats, Named: namedOf(request.Check.Named)}
+				Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)}
 		}
 		var v builder.CheckVerdict
 		v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
@@ -525,7 +525,7 @@ func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
 	var out []link.CheckBuilt
 	for _, b := range built {
 		c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(),
-			Kept: b.Kept}
+			Kept: b.Kept, Same: b.Same, SameAs: b.SameAs}
 		for _, m := range b.Made {
 			c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
 		}

cmd/mesh-controller/batches.go

@@ -1000,10 +1000,15 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
 	plan.Delivery.Merges = named
 	plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone
 	plan.Delivery.Withheld = withheld
+	// **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as
+	// the build the mesh holds.
+	for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) {
+		unchanged[n] = why
+	}
 	plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules)
 	// **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on.
 	plan.Times = walkTimesAtCut(*batch, plan, entries, now)
-	if len(moved) == 0 {
+	if len(moved) == 0 || len(plan.Modules) == 0 {
 		plan.State = inventory.PlanDone
 		plan.Tiers = [][]string{}
 		plan.Note = "moved nothing the mesh holds: nothing to walk"

cmd/mesh-controller/build.go

@@ -899,6 +899,34 @@ type answers struct {
 //
 // Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
 // base is recorded by digest and path, and a build machine needs something it can pull.
+// runningBy is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module, as a
+// builder reaches the artifact store (novox/hq issue 721): what a check compares its builds with. Empty when it cannot
+// be read, said, and then nothing is said unchanged.
+func runningBy(ctx context.Context) map[string]string {
+	open, err := openStores(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
+		// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
+		return nil
+	}
+	defer open.Close()
+	running, err := open.inventory.RegisteredHeld(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err)
+		// empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move
+		return nil
+	}
+	address, err := whereABuilderReachesTheStore(ctx, open.inventory)
+	if err != nil || address == "" {
+		return running
+	}
+	routed := make(map[string]string, len(running))
+	for key, reference := range running {
+		routed[key] = catalogue.Rerouted(reference, address)
+	}
+	return routed
+}
+
 func heldBy(ctx context.Context) map[string]string {
 	open, err := openStores(ctx)
 	if err != nil {

cmd/mesh-controller/checks.go

@@ -269,7 +269,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
 		Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
 			Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
 			Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
-			Release: release},
+			Release: release, Running: runningBy(ctx)},
 	}, nil
 }
 

cmd/mesh-controller/same_program.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **A module moves only when its program changes** (novox/hq issue 721). A merge check builds every module the
+// delivery plan moves and compares each with the build the mesh holds of it; a walk is cut without a module whose
+// check found its whole image the same as that build, while the mesh still holds it, and says so among the modules
+// it left unchanged. A merge of one sentence in a package a program imports but never reaches moves no build of it.
+
+// sameAsRunning are the modules of a walk its checks found the same as the build the mesh holds, each with what its
+// check said. **Only when every pull request the walk answers was checked at the head it merged, and every one of
+// those checks built the module and found it the same as the build the mesh still holds**: a merge the walk answers
+// without a pull request, a check of another head, a check that built it differently, or a held build that moved
+// since, leaves the module in the walk.
+func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[string]string) map[string]string {
+	if plan.Delivery == nil || len(plan.Delivery.Merges) == 0 {
+		return nil
+	}
+	for _, m := range plan.Delivery.Merges {
+		if _, ok := pullOf(m); !ok {
+			return nil
+		}
+	}
+	pulls := reuseAskOf(plan).Pulls
+	heads := mergedHeadsOf(plan)
+	out := map[string]string{}
+	for name := range plan.Modules {
+		said := ""
+		for _, pull := range pulls {
+			b, ok := checkedBuildOf(builds, pull, heads[pull], name)
+			if !ok || !stillHeld(b.Built, name, held) {
+				said = ""
+				break
+			}
+			if said == "" {
+				said = b.Built.Same
+			}
+		}
+		if said != "" {
+			out[name] = said
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
+// checkedBuildOf is the newest build a check of the pull request at head made of the module; false when none did.
+func checkedBuildOf(builds []link.CheckedBuild, pull link.PullRef, head, module string) (link.CheckedBuild, bool) {
+	if head == "" {
+		return link.CheckedBuild{}, false
+	}
+	for _, b := range builds {
+		if b.Pull == pull && b.Built.Module == module && link.SameHead(b.Head, head) {
+			return b, true
+		}
+	}
+	return link.CheckedBuild{}, false
+}
+
+// stillHeld is whether a check found every artifact of the module the same as a build the mesh still holds.
+func stillHeld(b link.CheckBuilt, module string, held map[string]string) bool {
+	if b.Same == "" || b.NotBuilt != "" || len(b.Made) == 0 || len(b.SameAs) != len(b.Made) {
+		return false
+	}
+	for _, m := range b.Made {
+		was := pinnedDigest(b.SameAs[m.Name])
+		if was == "" || was != pinnedDigest(held[module+"/"+m.Name]) {
+			return false
+		}
+	}
+	return true
+}
+
+// pinnedDigest is a reference's pinned digest, empty when it is not pinned.
+func pinnedDigest(ref string) string {
+	if i := strings.LastIndex(ref, "@"); i >= 0 {
+		return ref[i+1:]
+	}
+	return ""
+}
+
+// leaveSameAsRunning takes from a walk being cut every module its checks found the same as the build the mesh holds
+// (sameAsRunning), and every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read leaves the walk as it was**, said.
+func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string,
+	edges []inventory.Edge) map[string]string {
+	if plan.Delivery == nil || len(plan.Modules) == 0 {
+		return nil
+	}
+	pulls := reuseAskOf(plan).Pulls
+	if len(pulls) == 0 {
+		return nil
+	}
+	builds, err := checkedBuildsOf(ctx, pulls)
+	if err != nil {
+		fmt.Printf("  the checks' builds could not be read, so every module moves: %v\n", err)
+		// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
+		return nil
+	}
+	held, err := inv.RegisteredHeld(ctx)
+	if err != nil {
+		fmt.Printf("  what the mesh holds could not be read, so every module moves: %v\n", err)
+		// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
+		return nil
+	}
+	same := withDependentsOf(sameAsRunning(plan, builds, held), plan, moved, edges)
+	if len(same) == 0 {
+		return nil
+	}
+	names := make([]string, 0, len(same))
+	for n := range same {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	withhold(plan, names, nil)
+	for _, n := range names {
+		fmt.Printf("  %s left out: %s\n", n, same[n])
+	}
+	return same
+}
+
+// withDependentsOf adds to the modules found the same every module of the walk that is in it only as a dependent of
+// one of them (novox/hq ADR 0267 rule 3): **a dependent is left out when its base comes out the same**, since the
+// base it stands on stays the build the mesh holds. A module a merge moves itself, or a dependent of a base that
+// still moves, stays. moved are the modules the walk's merges move; edges the graph the walk was planned along.
+func withDependentsOf(same map[string]string, plan *inventory.Plan, moved []string, edges []inventory.Edge) map[string]string {
+	if len(same) == 0 {
+		return same
+	}
+	var still []string
+	for _, n := range moved {
+		if same[n] == "" {
+			still = append(still, n)
+		}
+	}
+	keep := map[string]bool{}
+	for _, n := range reachableFrom(still, edges) {
+		keep[n] = true
+	}
+	// Until nothing more is found: a dependent of a dependent left out is left out too.
+	for grew := true; grew; {
+		grew = false
+		for n := range plan.Modules {
+			if keep[n] || same[n] != "" {
+				continue
+			}
+			var bases []string
+			for _, e := range edges {
+				ordering := e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf ||
+					e.Kind == inventory.EdgePackages || e.Kind == edgeGroupOrder
+				if !ordering && e.From == n && same[e.To] != "" && !slices.Contains(bases, e.To) {
+					bases = append(bases, e.To)
+				}
+			}
+			sort.Strings(bases)
+			if len(bases) == 0 {
+				// In the walk for no base found the same: whatever brought it in is not said here, so it moves.
+				continue
+			}
+			same[n] = "unchanged: it is in the walk only for " + strings.Join(bases, ", ") +
+				", which came out the same as the build the mesh holds (ADR 0267 rule 3)"
+			grew = true
+		}
+	}
+	return same
+}

cmd/mesh-controller/same_program_test.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// TestAWalkLeavesOutAModuleWhoseProgramDidNotChange is novox/hq issue 721: mesh-controller #335 changed one sentence
+// that mesh-builder never reaches; its check found build-agent the same as the build the mesh holds, so the walk moves
+// mesh-controller alone and says build-agent is unchanged.
+func TestAWalkLeavesOutAModuleWhoseProgramDidNotChange(t *testing.T) {
+	pull := link.PullRef{Owner: "novox", Repo: "mesh-controller", Number: 335}
+	head := "34050dc6aaaabbbbccccdddd"
+	aPlan := func() *inventory.Plan {
+		return &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}},
+			Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}},
+			Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-controller",
+				Commit: "c2ec0a60", Number: 335, Head: head}}}}
+	}
+	held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa",
+		"mesh-controller/controller": "store/mesh-controller@sha256:cccc"}
+	builds := []link.CheckedBuild{
+		{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "build-agent", Key: "k1",
+			Made:   []link.MadeArtifact{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}},
+			Same:   "unchanged: its program is the same as running build agent aaaa",
+			SameAs: map[string]string{"agent": "10.0.0.1:5000/build-agent@sha256:aaaa"}}},
+		{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "mesh-controller", Key: "k2",
+			Made: []link.MadeArtifact{{Name: "controller", Kind: "image", Reference: "scratch/mesh-controller@sha256:dddd"}}}},
+	}
+
+	p := aPlan()
+	same := sameAsRunning(p, builds, held)
+	if len(same) != 1 || same["build-agent"] == "" {
+		t.Fatalf("the walk leaves out %v, want build-agent alone", same)
+	}
+	withhold(p, []string{"build-agent"}, nil)
+	if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" {
+		t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers)
+	}
+
+	// **The held build moved since the check**: what it was compared with no longer runs.
+	moved := map[string]string{"build-agent/agent": "store/build-agent@sha256:eeee"}
+	if same := sameAsRunning(aPlan(), builds, moved); len(same) != 0 {
+		t.Fatalf("a module compared with a build the mesh no longer holds is left out: %v", same)
+	}
+	// **A check of another head** says nothing of this merge.
+	other := aPlan()
+	other.Delivery.Merges[0].Head = "ffffffffffffffff"
+	if same := sameAsRunning(other, builds, held); len(same) != 0 {
+		t.Fatalf("a check of another head left out %v", same)
+	}
+	// **Another pull request the walk answers** that did not find it the same leaves it in.
+	two := aPlan()
+	two.Delivery.Merges = append(two.Delivery.Merges, inventory.PlanMerge{Repository: "novox/mesh-catalog",
+		Commit: "1234", Number: 9, Head: "abcdefabcdefabcdef"})
+	if same := sameAsRunning(two, builds, held); len(same) != 0 {
+		t.Fatalf("a module another merge moves is left out: %v", same)
+	}
+	// **A merge with no pull request** leaves the walk as it is.
+	direct := aPlan()
+	direct.Delivery.Merges[0].Number = 0
+	if same := sameAsRunning(direct, builds, held); len(same) != 0 {
+		t.Fatalf("a walk answering a merge without a pull request left out %v", same)
+	}
+}
+
+// TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk
+// for its base; when the base comes out the same as the build the mesh holds, the dependent stands on that build
+// still, and leaves the walk with it — and so does a dependent of that dependent. One a merge moves itself stays.
+func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) {
+	edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeDeclared},
+		{From: "app2", To: "app", Kind: inventory.EdgeDeclared}, {From: "own", To: "base", Kind: inventory.EdgeDeclared}}
+	plan := &inventory.Plan{Modules: map[string]*inventory.PlanModule{"base": {}, "app": {}, "app2": {}, "own": {}}}
+	same := withDependentsOf(map[string]string{"base": "unchanged: its program is the same"}, plan,
+		[]string{"base", "own"}, edges)
+	if !strings.Contains(same["app"], "only for base") || !strings.Contains(same["app2"], "only for app") {
+		t.Fatalf("the dependents of a base that came out the same are not left out: %v", same)
+	}
+	if _, left := same["own"]; left {
+		t.Fatalf("a module its merge moves itself is left out for its base: %v", same)
+	}
+	// A dependent of a base that still moves stays.
+	edges = append(edges, inventory.Edge{From: "app", To: "own", Kind: inventory.EdgeDeclared})
+	same = withDependentsOf(map[string]string{"base": "unchanged"}, plan, []string{"base", "own"}, edges)
+	if _, left := same["app"]; left {
+		t.Fatalf("a dependent of a base that still moves is left out: %v", same)
+	}
+}
+
+// Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one
+// module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the
+// module unchanged and why — the walk mesh-delivery reads as delivered, never stranded.
+func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) {
+	open := windowed(t)
+	endsItsLine(t)
+	asksRecorded(t)
+	ctx := t.Context()
+	inv := open.inventory
+	if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main",
+		Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour),
+		Made:    []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}},
+		Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil {
+		t.Fatal(err)
+	}
+	// What runs: the module registered with that build's image (RegisteredHeld reads the module table).
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
+		{"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}},
+		inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7}
+	was := checkedBuildsOf
+	checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
+		return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k",
+			Made:   []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}},
+			Same:   "unchanged: its program is the same as running build one aaaa",
+			SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil
+	}
+	t.Cleanup(func() { checkedBuildsOf = was })
+	code := repoMerge("one", "one1aaaaaaaa", t0)
+	code.Number, code.HeadSHA = 7, "headaaaaaaaa1111"
+	hear(t, open, code, t0.Add(time.Second))
+	cutAt(t, open, t0.Add(2*time.Minute))
+	ws, _ := walks(t, open)
+	if len(ws) != 1 {
+		t.Fatalf("walks: %+v", ws)
+	}
+	w := ws[0]
+	if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 {
+		t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules)
+	}
+	if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") {
+		t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery)
+	}
+}

internal/builder/check_builds.go

@@ -36,6 +36,10 @@ type CheckBuilding struct {
 	Publish Publisher
 	Scratch Scratch
 	Held    map[string]string
+	// Running is each artifact of the build the mesh has registered for its module, keyed "<module>/<artifact>"
+	// (novox/hq issue 721): what a build for the merge is compared with — never Held's newest build, which may be one
+	// its gate failed and the mesh put back.
+	Running map[string]string
 	Npmrc   Npmrc
 	Seats   map[string]string
 	// Named are the builds the build seat's verdicts of earlier checks of this pull request at this head name
@@ -57,6 +61,11 @@ type CheckBuilt struct {
 	// Kept is the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
 	// the verdict names that build, by key and digest, as that check's. Empty when this check built it.
 	Kept string
+	// Same says every artifact came out the same as the build the mesh holds of the module (novox/hq issue 721), and
+	// SameAs names, per artifact, the held reference it was compared with: the walk leaves the module where it is
+	// while the mesh still holds that build. Empty when it differs, or could not be compared.
+	Same   string
+	SameAs map[string]string
 }
 
 // skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
@@ -171,6 +180,18 @@ func makeForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with Check
 				case result.ReuseKey == "":
 					built.NotBuilt = "it has no reuse key: " + result.NoKey
 				}
+				if built.NotBuilt == "" {
+					// **Compared with what runs** (novox/hq issue 721): a failed comparison says nothing, and the
+					// module moves as before.
+					same, against, err := sameAsHeld(ctx, run, dir, b.Module, result.Built, with.Running)
+					switch {
+					case err != nil:
+						say("build", "%s could not be compared with the build the mesh holds, so it moves: %v", b.Module, err)
+					case same != "":
+						built.Same, built.SameAs = same, against
+						say("build", "%s %s", b.Module, same)
+					}
+				}
 				out[i], results[i] = built, result
 			}(i, b, repository, at, filepath.Join(place, name))
 		}

internal/builder/same_program.go

@@ -0,0 +1,180 @@
+package builder
+
+import (
+	"archive/tar"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// A module moves only when its program changes (novox/hq issue 721). A merge check builds every module the delivery
+// plan moves; a module whose every artifact comes out the same as the build the mesh holds of it — the same files,
+// the same modes and owners, the same entrypoint and environment — is said unchanged, and the walk leaves it where it
+// is. **The whole image is compared, never its main binary alone**: a module whose image differs in any file moves.
+// What is ignored is only what a rebuild cannot help changing: each file's time, the image's labels and its creation
+// time. **A comparison that cannot be made says nothing**, and the module is built and moved as before.
+
+// sameAsHeld says whether every artifact made is the same as the build the mesh holds of the module: the sentence
+// that says so and, per artifact, the held reference it was compared with. Empty when any artifact differs, is not
+// an image, has no held build, or could not be compared.
+func sameAsHeld(ctx context.Context, run Runner, dir, module string, made []catalogue.Built,
+	held map[string]string) (string, map[string]string, error) {
+	if len(made) == 0 {
+		return "", nil, nil
+	}
+	against := map[string]string{}
+	for _, m := range made {
+		if m.Kind != "image" || m.Reference == "" {
+			return "", nil, nil
+		}
+		was := held[module+"/"+m.Name]
+		if was == "" {
+			return "", nil, nil
+		}
+		if digestPart(was) != "" && digestPart(was) == digestPart(m.Reference) {
+			against[m.Name] = was
+			continue
+		}
+		now, err := imageContent(ctx, run, dir, m.Reference)
+		if err != nil {
+			return "", nil, fmt.Errorf("%s's new image: %w", m.Name, err)
+		}
+		before, err := imageContent(ctx, run, dir, was)
+		if err != nil {
+			return "", nil, fmt.Errorf("%s's held image: %w", m.Name, err)
+		}
+		if now != before {
+			return "", nil, nil
+		}
+		against[m.Name] = was
+	}
+	var names []string
+	for name, ref := range against {
+		names = append(names, name+" "+shortRef(ref))
+	}
+	sort.Strings(names)
+	return "unchanged: its program is the same as running build " + strings.Join(names, ", "), against, nil
+}
+
+// digestPart is a reference's "sha256:…", empty when it is not pinned.
+func digestPart(ref string) string {
+	if i := strings.LastIndex(ref, "@"); i >= 0 {
+		return ref[i+1:]
+	}
+	return ""
+}
+
+// shortRef names a reference by its digest's first twelve characters when it has one.
+func shortRef(ref string) string {
+	if d := strings.TrimPrefix(digestPart(ref), "sha256:"); d != "" {
+		return twelve(d)
+	}
+	return ref
+}
+
+// imageContent is an image's content as one digest: its platform, its configuration without labels, and every file
+// of its filesystem by path, type, mode, owner, link, extended attributes and bytes — never by time.
+func imageContent(ctx context.Context, run Runner, dir, ref string) (string, error) {
+	if _, err := run(ctx, dir, "docker", "pull", "--quiet", ref); err != nil {
+		return "", fmt.Errorf("pulling %s: %w", ref, err)
+	}
+	config, err := run(ctx, dir, "docker", "image", "inspect", "--format",
+		"{{.Os}}/{{.Architecture}}/{{.Variant}} {{json .Config}}", ref)
+	if err != nil {
+		return "", fmt.Errorf("reading %s's configuration: %w", ref, err)
+	}
+	said, err := configContent(config)
+	if err != nil {
+		return "", fmt.Errorf("reading %s's configuration: %w", ref, err)
+	}
+	id, err := run(ctx, dir, "docker", "create", "--entrypoint", "/", ref)
+	if err != nil {
+		return "", fmt.Errorf("opening %s's filesystem: %w", ref, err)
+	}
+	id = strings.TrimSpace(id)
+	if id == "" {
+		return "", errors.New("the container engine named no container for " + ref)
+	}
+	defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rm", "--force", id) }()
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return "", err
+	}
+	file := filepath.Join(dir, "content-"+twelve(id)+".tar")
+	defer os.Remove(file)
+	if _, err := run(ctx, dir, "docker", "export", "--output", file, id); err != nil {
+		return "", fmt.Errorf("reading %s's filesystem: %w", ref, err)
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	files, err := treeContent(f)
+	if err != nil {
+		return "", fmt.Errorf("reading %s's filesystem: %w", ref, err)
+	}
+	sum := sha256.Sum256([]byte(said + "\n" + files))
+	return "sha256:" + hex.EncodeToString(sum[:]), nil
+}
+
+// configContent is an image's platform and configuration as compared: Labels, the parent's id and the hostname the
+// builder ran under are left out — the first holds the commit, the others no part of what runs.
+func configContent(inspected string) (string, error) {
+	platform, raw, ok := strings.Cut(strings.TrimSpace(inspected), " ")
+	if !ok {
+		return "", fmt.Errorf("unexpected inspection %q", inspected)
+	}
+	var config map[string]json.RawMessage
+	if err := json.Unmarshal([]byte(raw), &config); err != nil {
+		return "", err
+	}
+	for _, k := range []string{"Labels", "Image", "Hostname", "Domainname"} {
+		delete(config, k)
+	}
+	out, err := json.Marshal(config) // map keys are sorted
+	if err != nil {
+		return "", err
+	}
+	return platform + " " + string(out), nil
+}
+
+// treeContent is a filesystem tarball's entries, sorted by path, each with everything but its times.
+func treeContent(r io.Reader) (string, error) {
+	tr := tar.NewReader(r)
+	var lines []string
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+		sum := sha256.New()
+		if _, err := io.Copy(sum, tr); err != nil {
+			return "", err
+		}
+		var xattrs []string
+		for k, v := range h.PAXRecords {
+			if strings.HasPrefix(k, "SCHILY.xattr.") {
+				xattrs = append(xattrs, k+"="+hex.EncodeToString([]byte(v)))
+			}
+		}
+		sort.Strings(xattrs)
+		lines = append(lines, fmt.Sprintf("%q %c %o %d:%d %q %d,%d %d %x %s",
+			strings.TrimPrefix(h.Name, "./"), h.Typeflag, h.Mode, h.Uid, h.Gid, h.Linkname, h.Devmajor, h.Devminor,
+			h.Size, sum.Sum(nil), strings.Join(xattrs, ";")))
+	}
+	sort.Strings(lines)
+	return strings.Join(lines, "\n"), nil
+}

internal/builder/same_program_test.go

@@ -0,0 +1,126 @@
+package builder
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// aFilesystem is a filesystem tarball of files, each written at the time given.
+func aFilesystem(t *testing.T, at time.Time, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	for _, name := range []string{"usr/local/bin/mesh-builder", "etc/apk/world"} {
+		body, ok := files[name]
+		if !ok {
+			continue
+		}
+		if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), ModTime: at,
+			Typeflag: tar.TypeReg}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := tw.Write([]byte(body)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return buf.Bytes()
+}
+
+// anEngine answers the container engine's commands for images by reference: each image's filesystem and labels.
+func anEngine(images map[string][]byte, labels map[string]string, failing string) Runner {
+	created := map[string]string{}
+	return func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		joined := strings.Join(args, " ")
+		ref := args[len(args)-1]
+		if failing != "" && strings.Contains(joined, failing) {
+			return "", fmt.Errorf("the engine refused %s", joined)
+		}
+		switch {
+		case args[0] == "pull", args[0] == "rm":
+			return "", nil
+		case args[0] == "image":
+			return fmt.Sprintf(`linux/amd64/ {"Entrypoint":["/usr/local/bin/mesh-builder"],"Labels":{"commit":%q}}`,
+				labels[ref]), nil
+		case args[0] == "create":
+			id := fmt.Sprintf("container%04d", len(created))
+			created[id] = ref
+			return id + "\n", nil
+		case args[0] == "export":
+			return "", os.WriteFile(args[2], images[created[ref]], 0o600)
+		}
+		return "", fmt.Errorf("unexpected %s %s", name, joined)
+	}
+}
+
+// TestAModuleWhoseProgramDidNotChangeIsSaidUnchanged is novox/hq issue 721: mesh-controller #335 changed one sentence
+// in a package mesh-builder imports and never reaches, and build-agent moved. Built again, its image differs only in
+// the files' times and its commit label, and the check says it is the same as the build the mesh holds.
+func TestAModuleWhoseProgramDidNotChangeIsSaidUnchanged(t *testing.T) {
+	held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa"}
+	made := []catalogue.Built{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}}
+	files := map[string]string{"usr/local/bin/mesh-builder": "the same program", "etc/apk/world": "docker-cli\ngit\n"}
+	images := map[string][]byte{
+		"store/build-agent@sha256:aaaa":   aFilesystem(t, time.Unix(1000, 0), files),
+		"scratch/build-agent@sha256:bbbb": aFilesystem(t, time.Unix(2000, 0), files),
+	}
+	labels := map[string]string{"store/build-agent@sha256:aaaa": "a5500f73", "scratch/build-agent@sha256:bbbb": "c2ec0a60"}
+	same, against, err := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent",
+		made, held)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(same, "unchanged: its program is the same as running build agent") ||
+		against["agent"] != held["build-agent/agent"] {
+		t.Fatalf("an image the same but for its times and labels is not said unchanged: %q %v", same, against)
+	}
+
+	// **Any file differing moves it**, not only its program.
+	changed := map[string]string{"usr/local/bin/mesh-builder": "the same program", "etc/apk/world": "docker-cli\ngit\ncurl\n"}
+	images["scratch/build-agent@sha256:bbbb"] = aFilesystem(t, time.Unix(2000, 0), changed)
+	if same, _, err := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent", made,
+		held); err != nil || same != "" {
+		t.Fatalf("an image with another file is said unchanged: %q %v", same, err)
+	}
+
+	// **A comparison that fails says nothing**, and the module moves.
+	images["scratch/build-agent@sha256:bbbb"] = aFilesystem(t, time.Unix(2000, 0), files)
+	if same, _, err := sameAsHeld(context.Background(), anEngine(images, labels, "export"), t.TempDir(), "build-agent",
+		made, held); err == nil || same != "" {
+		t.Fatalf("a comparison that failed said %q, %v", same, err)
+	}
+
+	// No held build, or an artifact that is not an image: nothing compared.
+	if same, _, _ := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent", made,
+		nil); same != "" {
+		t.Fatalf("a module the mesh holds no build of is said unchanged: %q", same)
+	}
+	archive := []catalogue.Built{{Name: "agent", Kind: "archive", Reference: "scratch/build-agent@sha256:bbbb"}}
+	if same, _, _ := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent",
+		archive, held); same != "" {
+		t.Fatalf("an archive is said unchanged without being compared: %q", same)
+	}
+}
+
+// TestAnImageWithAnotherEntrypointIsNotTheSame: the configuration counts, but for its labels.
+func TestAnImageWithAnotherEntrypointIsNotTheSame(t *testing.T) {
+	a, err := configContent(`linux/amd64/ {"Entrypoint":["/a"],"Labels":{"commit":"1"}}`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := configContent(`linux/amd64/ {"Labels":{"commit":"2"},"Entrypoint":["/a"]}`)
+	c, _ := configContent(`linux/amd64/ {"Entrypoint":["/b"],"Labels":{"commit":"1"}}`)
+	if a != b || a == c {
+		t.Fatalf("labels count, or the entrypoint does not: %q %q %q", a, b, c)
+	}
+}

internal/inventory/builds.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/jackc/pgx/v5"
@@ -301,6 +302,51 @@ func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
 	return held, nil
 }
 
+// RegisteredHeld is Held narrowed to the build each module has registered (novox/hq issue 721): an artifact is kept
+// only when its reference's digest is in the manifest the module table holds for it, which is what its machines are
+// sent. After a gate failed and the mesh put the earlier build back (RestoreModule), the newest successful build is
+// not what runs, and its artifacts are left out here, so nothing is compared with them.
+func (i *Inventory) RegisteredHeld(ctx context.Context) (map[string]string, error) {
+	held, err := i.Held(ctx)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := i.store.Pool().Query(ctx, `select name, coalesce(manifest::text, '') from module`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	registered := map[string]string{}
+	for rows.Next() {
+		var name, manifest string
+		if err := rows.Scan(&name, &manifest); err != nil {
+			return nil, err
+		}
+		registered[name] = manifest
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return registeredOnly(held, registered), nil
+}
+
+// registeredOnly keeps of held each "<module>/<artifact>" whose reference's digest the module's registered manifest
+// names.
+func registeredOnly(held, registered map[string]string) map[string]string {
+	out := map[string]string{}
+	for key, ref := range held {
+		module, _, ok := strings.Cut(key, "/")
+		at := strings.LastIndex(ref, "@")
+		if !ok || at < 0 || len(ref) <= at+1 {
+			continue
+		}
+		if strings.Contains(registered[module], ref[at+1:]) {
+			out[key] = ref
+		}
+	}
+	return out
+}
+
 // BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
 // edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
 // module standing on nothing looks like: an edge the mesh has not derived is not an edge.

internal/inventory/registered_held_test.go

@@ -0,0 +1,72 @@
+package inventory
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack is novox/hq issue 721's review: the newest build of a module
+// failed its gate and the mesh put the build before it back. Held names the newest; what runs, and what a merge's build
+// is compared with, is the build put back — so a merge whose bytes equal the failed build is never said unchanged.
+func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) {
+	inv := fresh(t)
+	ctx := t.Context()
+	manifestOf := func(image string) catalogue.Manifest {
+		return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
+			{"id": "app", "type": "container", "name": "one", "image": image}}}
+	}
+	record := func(id, image string, asked time.Time) Build {
+		t.Helper()
+		raw, _ := json.Marshal(manifestOf(image))
+		b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw,
+			Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit,
+			Asked: asked}); err != nil {
+			t.Fatal(err)
+		}
+		return b
+	}
+	x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111"
+	y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222"
+	before := record("x", x, time.Now().Add(-2*time.Hour))
+	failed := record("y", y, time.Now().Add(-time.Minute))
+	if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit,
+		Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RestoreModule(ctx, before); err != nil {
+		t.Fatal(err)
+	}
+	held, err := inv.Held(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	running, err := inv.RegisteredHeld(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if running["one/one"] == y {
+		t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"])
+	}
+	if got := running["one/one"]; got != "" && got != x {
+		t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x)
+	}
+}
+
+// TestOnlyArtifactsTheRegisteredManifestNamesAreRunning: registeredOnly keeps a held artifact only when its digest is
+// in its module's registered manifest.
+func TestOnlyArtifactsTheRegisteredManifestNamesAreRunning(t *testing.T) {
+	held := map[string]string{"one/app": "store/one@sha256:aaaa", "two/app": "store/two@sha256:bbbb", "three/app": "nodigest"}
+	registered := map[string]string{"one": `{"resources":[{"image":"store/one@sha256:aaaa"}]}`,
+		"two": `{"resources":[{"image":"store/two@sha256:cccc"}]}`, "three": "nodigest"}
+	got := registeredOnly(held, registered)
+	if len(got) != 1 || got["one/app"] == "" {
+		t.Fatalf("running reads %v, want one/app alone", got)
+	}
+}

internal/link/build.go

@@ -182,6 +182,11 @@ type CheckRequest struct {
 	// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
 	// take. A build seat from before it builds nothing, and the walk builds as before.
 	Builds []CheckBuild `json:"builds,omitempty"`
+	// Running is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module — the
+	// build its machines are sent (novox/hq issue 721): what a build for the merge is compared with, never the newest
+	// build, which may be one its gate failed and the mesh put back. A module absent is compared with nothing, and
+	// moves. A controller from before it sends none, and nothing is said unchanged.
+	Running map[string]string `json:"running,omitempty"`
 	// Named are the builds the seat's verdicts of earlier checks of this pull request at this head name, within the
 	// week check builds are kept (novox/hq issue 703): asked again because only the trunk moved, the check keeps a
 	// build kept under a module's unchanged key, instead of building it again, only when one of these names its key
@@ -238,6 +243,11 @@ type CheckBuilt struct {
 	// Kept names the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
 	// a walk takes it as that check's build. Empty when this check built it.
 	Kept string `json:"kept,omitempty"`
+	// Same says every artifact came out the same as the build the mesh holds of the module, and SameAs the held
+	// reference each was compared with (novox/hq issue 721): a walk cut while the mesh still holds those builds leaves
+	// the module out. Empty when it differs, or could not be compared.
+	Same   string            `json:"same,omitempty"`
+	SameAs map[string]string `json:"same-as,omitempty"`
 }
 
 // GroupMember is one other head of a delivery group's composed check.

testdata/store-tests

@@ -45,6 +45,7 @@
 ./cmd/mesh-controller TestReplay348AFaultFromBeforeTheSendFailsNoGate
 ./cmd/mesh-controller TestReplay239AModuleNameIsNotTakenOverByAnotherRepository
 ./cmd/mesh-controller TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns
+./cmd/mesh-controller TestAWalkWhoseEveryBuildCameOutTheSameIsDone
 ./cmd/mesh-controller TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles
 ./internal/inventory TestACredentialIssuedOutsideIsNeverMade
 ./internal/inventory TestOneAcceptForAProviderIsSealedToEveryConsumer
@@ -72,3 +73,4 @@
 ./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport
 ./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition
 ./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn
+./internal/inventory TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack