Merge pull request 'The builder takes build-agent's newest trunk build as its builder input (issue 1727)' (#16) from fix/1727-builder-input into main

modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"sync/atomic"
 	"syscall"
 
 	"github.com/nats-io/nats.go"
@@ -260,12 +261,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		return err
 	}
 	known := index.NewBuilds()
+	// A build of the module that builds the builder moves the builder's Builder input and so every fingerprint of the new
+	// form, with no merge (issue 1727): the new-form fingerprints are computed again when one is heard, once the indexer exists.
+	var started atomic.Pointer[index.Indexer]
 	each := func(e jetstream.KeyValueEntry) {
 		if opOf(e) {
 			known.Put(e.Key(), e.Value())
 		} else {
 			known.Delete(e.Key())
 		}
+		if ix := started.Load(); ix != nil && strings.HasPrefix(e.Key(), index.BuilderBuiltBy+".build.") {
+			if err := ix.RecomputeForBuild(ctx); err != nil {
+				logger.Printf("computing the new-form fingerprints again after %s: %v", e.Key(), err)
+			}
+		}
 	}
 	w, err := settle(ctx, builds, each)
 	if err != nil {
@@ -300,6 +309,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	if err := tools.start(); err != nil {
 		return err
 	}
+	// Only now, with the toolchains read, are the builds heard to compute from (a build before it would write every Go
+	// module as having no toolchain reported, and back); the builds heard until now are computed once.
+	started.Store(ix)
+	if err := ix.RecomputeForBuild(ctx); err != nil {
+		logger.Printf("computing the new-form fingerprints at start: %v", err)
+	}
 	// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
 	// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
 	// with the build module, as with the gate's source.

modules/mesh-delivery-indexer/internal/index/builds.go

@@ -138,3 +138,18 @@ func (b *Builds) Modules(repository string) []Known {
 	sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
 	return out
 }
+
+// TrunkOf is the newest successful trunk build record of a module, in whatever repository it was built from; nil when
+// there is none (issue 1727).
+func (b *Builds) TrunkOf(module string) *delivery.BuildRecord {
+	var best *delivery.BuildRecord
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	for _, r := range b.records {
+		if r.Module == module && r.IsTrunkBuild() && (best == nil || newer(r, *best)) {
+			c := r
+			best = &c
+		}
+	}
+	return best
+}

modules/mesh-delivery-indexer/internal/index/toolchains.go

@@ -22,8 +22,10 @@ const (
 	// BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is
 	// its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it.
 	BuilderModule = "mesh-delivery-builder"
-	// BuilderBuiltBy is the module that builds BuilderModule until release R4g: its fingerprint of today's form is the
-	// builder's own Builder input, so a change of build-agent moves the builder's value (ADR 0453 clause 5).
+	// BuilderBuiltBy is the module that builds BuilderModule until release R4g: the source fingerprint (src1) the controller
+	// recorded on its newest successful trunk build is the builder's own Builder input, so a change of build-agent moves the
+	// builder's value (ADR 0453 clause 5). Its fingerprint of today's form is no input: build-agent has none before release
+	// R4 (ADR 0437 clause 5), so taking it left every module without a new-form fingerprint (issue 1727).
 	BuilderBuiltBy = "build-agent"
 )
 
@@ -235,11 +237,11 @@ func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]deliver
 		}
 		var noBuilder string
 		if module == BuilderModule {
-			// Built by build-agent until release R4g: its value of today's form.
-			if a := held[BuilderBuiltBy]; a.FP != "" {
-				kept.Builder = a.FP
+			// Built by build-agent until release R4g: the src1 of its newest successful trunk build record (issue 1727).
+			if src, none := ix.builderInput(); src != "" {
+				kept.Builder = src
 			} else {
-				noBuilder = fmt.Sprintf("%s, which builds it, has no fingerprint (%s)", BuilderBuiltBy, a.None)
+				noBuilder = none
 			}
 		} else if r := of(BuilderModule, chain); r.fp != "" {
 			kept.Builder = r.fp
@@ -383,3 +385,30 @@ func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.Toolc
 	}
 	return out
 }
+
+// builderInput is the Builder input of BuilderModule while BuilderBuiltBy builds it: the source fingerprint the controller
+// recorded on that module's newest successful trunk build (issue 1727). With no such build, or one that recorded no
+// fingerprint, there is none, and the reason says so.
+func (ix *Indexer) builderInput() (src, none string) {
+	if ix.Builds != nil {
+		if r := ix.Builds.TrunkOf(BuilderBuiltBy); r != nil && r.SourceFingerprint != "" {
+			return r.SourceFingerprint, ""
+		}
+	}
+	return "", fmt.Sprintf("%s, which builds it, has no successful trunk build that recorded a source fingerprint", BuilderBuiltBy)
+}
+
+// RecomputeForBuild is Recompute for a build of BuilderBuiltBy heard on the builds bucket, or the one at start (issue 1727).
+// While the indexer has read no toolchain at all (it has just started and the toolchains buckets are not settled), every
+// Go module would be written as having no toolchain reported and then back: it computes nothing then, and the settling of
+// the toolchains computes them.
+func (ix *Indexer) RecomputeForBuild(ctx context.Context) error {
+	ix.mu.Lock()
+	none := len(ix.toolchains) == 0
+	ix.mu.Unlock()
+	if none {
+		ix.logf("the builds changed before any toolchain was read: the new-form fingerprints are computed when they are")
+		return nil
+	}
+	return ix.Recompute(ctx)
+}

modules/mesh-delivery-indexer/internal/index/toolchains_test.go

@@ -36,6 +36,25 @@ func standingOn(t *testing.T, raw, base string) string {
 
 const buildAgentRepo = "novox/mesh-controller"
 
+// build-agent's trunk builds as `mesh-controller.builds` showed them on 2026-10-21 (ids, times and source fingerprints
+// as listed; the repository is the clone URL the record names, the ref "main"; the inputs are the shape of aInputs, which
+// the indexer does not read for this): the older one at 13:03 and the newer at 15:03 (issue 1727).
+const (
+	agentSrcX     = "src1:80e5fb129a98754a85fa3a43b1dded12dddedd25823d12dbe8783287faa0c59c"
+	agentSrcY     = "src1:44ed217d74a7c7e38400bb0bf32358ed96502cb385d24f1435fe971ad44c7b88"
+	agentCloneURL = "http://novox.internal:20000/novox/mesh-controller.git"
+)
+
+func agentBuild(id, src, at string) [2]any {
+	commit := "cada7e3c"
+	if id == "build-1792587792086548572" {
+		commit = "f10fbda1"
+	}
+	raw := `{"id":"` + id + `","repository":"` + agentCloneURL + `","ref":"main","module":"build-agent","commit":"` + commit + `",` +
+		`"path":"modules/build-agent","at":"` + at + `","source-fingerprint":"` + src + `","source-inputs":` + aInputs + `}`
+	return [2]any{delivery.BuildKey("build-agent", id), []byte(raw)}
+}
+
 type mod struct {
 	name, manifest string
 }
@@ -48,10 +67,7 @@ func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) {
 	if err := ix.Apply(context.Background(), f); err != nil {
 		t.Fatal(err)
 	}
-	// build-agent's value of today's form, from a merge of the controller (modules.build-agent holds its fingerprint).
-	if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-1"})); err != nil {
-		t.Fatal(err)
-	}
+	// build-agent holds no fingerprint of today's form, as live (no merge of the controller is applied; issue 1727).
 	return ix, store
 }
 
@@ -67,7 +83,7 @@ func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, del
 		trees[dir], manifests[dir] = "tree-"+m.name, m.manifest
 		locks[dir] = map[string]string{"go.sum": avahiGoSum, "package-lock.json": strings.Repeat("c", 63) + string(rune('a'+i))}
 	}
-	recs = append(recs, rec(t, "build-agent-1", "build-agent", buildAgentRepo, "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", aInputs))
+	recs = append(recs, agentBuild("build-1792580585452696147", agentSrcX, "2026-10-21T13:03:00Z"))
 	ix, store, _ := manifestsIndexer(t, recs...)
 	return ix, store, withManifests(mergedMsg(catalogueRepo, trees), manifests, locks)
 }
@@ -144,14 +160,10 @@ func fp2(t *testing.T, in delivery.SourceInputs2) string {
 	return fp
 }
 
-func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) { // 1444/AC7, AC8
+func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) { // 1444/AC7, AC8 as 1727 corrects it
 	ix, mods := newFormIndexer(t, builderModules(t)...)
 	reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
-	agent, _ := moduleOf(t, mods, "build-agent")
-	if agent.FP == "" {
-		t.Fatalf("build-agent has no fingerprint of today's form: %+v", agent)
-	}
-	builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agent.FP))
+	builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agentSrcX))
 	toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", builderFP))
 	want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", builderFP, toolsFP))}
 	for name, fp := range want {
@@ -162,15 +174,14 @@ func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) {
 	if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
 		t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2)
 	}
-	// 1444/AC8: build-agent changes at a merge of the controller; the builder's value moves with it, and so do the modules built by it.
-	if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-2"})); err != nil {
+	// 1727: a new trunk build of build-agent records another source fingerprint; with no merge, the builder's value moves
+	// with it, and so do the modules built by it.
+	a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z")
+	ix.Builds.Put(a[0].(string), a[1].([]byte))
+	if err := ix.Recompute(context.Background()); err != nil {
 		t.Fatal(err)
 	}
-	moved, _ := moduleOf(t, mods, "build-agent")
-	if moved.FP == agent.FP {
-		t.Fatal("build-agent's fingerprint did not change")
-	}
-	newBuilder := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", moved.FP))
+	newBuilder := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agentSrcY))
 	if m, _ := moduleOf(t, mods, "mesh-delivery-builder"); m.FP2 != newBuilder || newBuilder == builderFP {
 		t.Errorf("builder: fp2 = %q, want %s (was %s)", m.FP2, newBuilder, builderFP)
 	}
@@ -179,6 +190,46 @@ func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) {
 	}
 }
 
+func TestABuildHeardBeforeAnyToolchainIsReadDropsNoFingerprint(t *testing.T) { // 1727 (review: a restart flapped every FP2)
+	ix, mods := newFormIndexer(t, builderModules(t)...)
+	reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
+	before, _ := moduleOf(t, mods, "mesh-tools")
+	if before.FP2 == "" {
+		t.Fatalf("mesh-tools has no fp2 to begin with: %+v", before)
+	}
+	// a restarted indexer has read no toolchains bucket yet
+	ix.mu.Lock()
+	ix.toolchains = nil
+	ix.mu.Unlock()
+	if err := ix.RecomputeForBuild(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != before.FP2 || m.None2 != "" {
+		t.Errorf("mesh-tools: fp2 = %q none2 = %q, want fp2 %q kept", m.FP2, m.None2, before.FP2)
+	}
+}
+
+func TestTheBuilderHasNoFingerprintWithoutATrunkBuildOfBuildAgentAndSaysSo(t *testing.T) { // 1727
+	ix, mods := newFormIndexer(t, builderModules(t)...)
+	reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
+	ix.Builds.Delete(delivery.BuildKey("build-agent", "build-1792580585452696147"))
+	// a failed build and a build of a branch are no trunk build
+	failed, fv := record(t, "b-failed", "build-agent", agentCloneURL, "modules/build-agent", "main", "failed", "2026-10-21T15:03:00Z", aInputs)
+	branch, bv := record(t, "b-branch", "build-agent", agentCloneURL, "modules/build-agent", "fix/x", "", "2026-10-21T15:04:00Z", aInputs)
+	ix.Builds.Put(branch, bv)
+	ix.Builds.Put(failed, fv)
+	if err := ix.Recompute(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	m, _ := moduleOf(t, mods, "mesh-delivery-builder")
+	if m.FP2 != "" || !strings.Contains(m.None2, "build-agent") || !strings.Contains(m.None2, "no successful trunk build") {
+		t.Errorf("builder: fp2 = %q none2 = %q, want none2 naming build-agent's missing trunk build", m.FP2, m.None2)
+	}
+	if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != "" || !strings.Contains(m.None2, "builder") {
+		t.Errorf("mesh-tools: fp2 = %q none2 = %q, want none2 naming the builder", m.FP2, m.None2)
+	}
+}
+
 func TestAToolchainEntryChangedToTheSameValuesWritesNothing(t *testing.T) { // 1444/AC9
 	ix, mods := newFormIndexer(t, builderModules(t)...)
 	reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")