Compute every module's fingerprint with no builder input and read no build-agent record for it, so a change of the builder rebuilds nothing (issue 1887 I1, ADR 0530 §1)

This commit is contained in:
2026-10-23 16:14:19 +02:00
parent 69577f739d
commit a4629bc649
8 changed files with 52 additions and 131 deletions
@@ -185,7 +185,7 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b
// TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged
// message on FORGE makes modules.<m> within 5 s, the fingerprint of the merge's tree and the recorded inputs.
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851)
// The bucket holds only build-agent's record, the one the indexer reads until R4g (the builder's own input).
// The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1).
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`
if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil {
@@ -276,8 +276,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4
// TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state
// toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying
// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; the bucket made after the indexer started, a put of the key n1.go and the build module's event toolchain.reported (the build
// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, none2 "builder's
// fingerprint not yet known".
// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, its fingerprint:
// no builder input is waited for (ADR 0530 §1; issue 1887 I1).
func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
@@ -342,8 +342,8 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
if _, err := js.Publish(ctx, "mesh.stream.mesh-build-go.event.toolchain.reported", []byte(`{"node":"n1","language":"go"}`)); err != nil {
t.Fatal(err)
}
if got := none2("builder's fingerprint not yet known"); got != "builder's fingerprint not yet known" {
t.Errorf("none2 = %q within 5 s of the toolchain key, want the builder's fingerprint not yet known", got)
if got := none2(""); got != "" {
t.Errorf("none2 = %q within 5 s of the toolchain key, want a fingerprint and no reason", got)
}
}
@@ -18,7 +18,6 @@ import (
"os"
"os/signal"
"strings"
"sync/atomic"
"syscall"
"github.com/nats-io/nats.go"
@@ -262,20 +261,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
known := index.NewBuilds()
// A build of the module that builds the builder moves the builder's Builder input and so every fingerprint of the new
// form, with no merge (issue 1727): the new-form fingerprints are computed again when one is heard, once the indexer exists.
var started atomic.Pointer[index.Indexer]
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
known.Put(e.Key(), e.Value())
} else {
known.Delete(e.Key())
}
if ix := started.Load(); ix != nil && strings.HasPrefix(e.Key(), index.BuilderBuiltBy+".build.") {
if err := ix.RecomputeForBuild(ctx); err != nil {
logger.Printf("computing the new-form fingerprints again after %s: %v", e.Key(), err)
}
}
}
w, err := settle(ctx, builds, each)
if err != nil {
@@ -327,10 +320,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := previews.ensure(); err != nil {
return err
}
// Only now, with the toolchains read, are the builds heard to compute from (a build before it would write every Go
// module as having no toolchain reported, and back); the builds heard until now are computed once.
started.Store(ix)
if err := ix.RecomputeForBuild(ctx); err != nil {
// Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as
// having no toolchain reported, and back).
if err := ix.RecomputeAtStart(ctx); err != nil {
logger.Printf("computing the new-form fingerprints at start: %v", err)
}
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
@@ -16,8 +16,9 @@ import (
const (
// None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge).
None2NoManifest = "the message carries no manifest"
// None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing.
None2Builder = "builder's fingerprint not yet known"
// None2Pending is for a module with no artifact of a toolchain whose fingerprint the recompute of its chain gives (its
// bases' fingerprints; no builder input since ADR 0530 §1).
None2Pending = "the fingerprint is computed with its chain"
// None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none
// either, NoneContext).
None2Context = "context not pinned"
@@ -146,7 +147,7 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, b
case len(langs) > 0:
out.None2 = delivery.ToolchainNoReport
default:
out.None2 = None2Builder
out.None2 = None2Pending
}
return out
}
@@ -154,7 +154,10 @@ func TestAManifestTheParserRefusesLeavesNoInputsAndSaysWhy(t *testing.T) { // 14
func TestAContextIsNotPinned(t *testing.T) { // 1450 (the rule 1444 keeps: context not pinned, as today's form)
// build-agent's manifest from mesh-controller at f2d7bc94 has an image artifact built from a context in another repository.
in := inputsOf("build-agent", fixture(t, "controller-f2d7bc94-build-agent"), "t", nil, noBases)
// Its claim of the retired seat node-build-agent is read as a claim of a build seat: the parser refuses a retired seat
// since R4g (mesh-sdk A9a), and the claim is not what this test is about.
raw := strings.ReplaceAll(fixture(t, "controller-f2d7bc94-build-agent"), `"node-build-agent"`, `"node-build-go"`)
in := inputsOf("build-agent", raw, "t", nil, noBases)
if in.Kept == nil || in.None2 != None2Context {
t.Errorf("kept %+v none2 %q, want none2 %q", in.Kept, in.None2, None2Context)
}
@@ -31,14 +31,16 @@ func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t
}
}
func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2
// 1887/I1, ADR 0530 §1: a module with no artifact of a toolchain waits for no builder input: its fingerprint is written with
// the merge, its inputs naming no builder.
func TestAModuleWithNoToolchainArtifactGetsItsFingerprintAtOnce(t *testing.T) { // 1444/AC2
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil {
t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok)
if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.FP2 == "" || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" {
t.Errorf("module = %+v (%v), want its fingerprint with no builder input", m, ok)
}
}
@@ -234,7 +234,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
out.None2 = in.None2
return out
}
if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Builder {
if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Pending {
out.None2 = in.None2 // a context, or a base with no fingerprint
return out
}
@@ -248,22 +248,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
}
toolchains[artifact] = t.tc
}
inputs := *in.Kept
if m == BuilderModule {
src, none := ix.builderInput()
if src == "" {
out.None2 = none
return out
}
inputs.Builder = src
} else {
fp, none := ix.baseFP(ctx)(BuilderModule)
if fp == "" {
out.None2 = strings.TrimSuffix(None2Builder+": "+none, ": ")
return out
}
inputs.Builder = fp
}
inputs := *in.Kept // no builder input (ADR 0530 §1)
fp, none := delivery.SourceFingerprint2(inputs.With(m, toolchains))
if fp == "" {
out.None2 = none
@@ -14,20 +14,12 @@ import (
// The new-form fingerprint of a module is written, and written again with no merge, when what it is of changes
// (novox/hq ADR 0465 clause 2; issue 1444): a module's tree and lock hashes come with a merge and are kept in
// modules.<m>.Inputs2; its builder's, its bases' and its toolchain's come from here. The indexer holds no graph, so each
// recompute reads the manifest of every module from manifests.<digest> and takes the modules in the order their bases
// and the builder come first.
const (
// BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is
// its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it.
BuilderModule = "mesh-delivery-builder"
// BuilderBuiltBy is the module that builds BuilderModule until release R4g: the source fingerprint (src1) the controller
// recorded on its newest successful trunk build is the builder's own Builder input, so a change of build-agent moves the
// builder's value (ADR 0453 clause 5). Its fingerprint of today's form is no input: build-agent has none before release
// R4 (ADR 0437 clause 5), so taking it left every module without a new-form fingerprint (issue 1727).
BuilderBuiltBy = "build-agent"
)
// modules.<m>.Inputs2; its bases' and its toolchain's come from here. The indexer holds no graph, so each recompute reads
// the manifest of every module from manifests.<digest> and takes the modules in the order their bases come first.
//
// **No module's fingerprint takes the builder as an input** (novox/hq ADR 0530 §1, issue 1887 I1): the builder orders and
// records builds and makes no artifact's bytes (ADR 0498 §1), so its Builder input is empty for every module, the builder's
// own included; a change of how an artifact is built is a change of its build module's recipe (ADR 0530 §2).
// ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the
// indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on.
@@ -246,22 +238,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
}
toolchains[artifact] = t.tc
}
var noBuilder string
if module == BuilderModule {
// Built by build-agent until release R4g: the src1 of its newest successful trunk build record (issue 1727).
if src, none := ix.builderInput(); src != "" {
kept.Builder = src
} else {
noBuilder = none
}
} else if r := of(BuilderModule, chain); r.fp != "" {
kept.Builder = r.fp
} else {
noBuilder = None2Builder
if _, has := held[BuilderModule]; has {
noBuilder += ": " + r.none
}
}
var r result
switch {
case k.contexts:
@@ -270,8 +246,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
r.none = strings.Join(missing, "; ")
case noToolchain != "":
r.none = noToolchain
case noBuilder != "":
r.none = noBuilder
default:
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
if r.fp != "" {
@@ -289,7 +263,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
if _, merged := overlay[key]; merged {
final := overlay[key]
final.FP2, final.None2, final.Inputs2, final.Toolchains = r.fp, r.none, k.cur.Inputs2, r.tcs
// What the bases and the builder gave is kept as it was while the value it gave does not change, so a merge
// What the bases gave is kept as it was while the value it gave does not change, so a merge
// that moves nothing for the module writes nothing (as Recompute does).
if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
if old, err := delivery.DecodeModule(value); err == nil && old.Inputs2 != nil && old.FP2 == r.fp && old.None2 == r.none {
@@ -397,28 +371,15 @@ func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.Toolc
return out
}
// builderInput is the Builder input of BuilderModule while BuilderBuiltBy builds it: the source fingerprint the controller
// recorded on that module's newest successful trunk build (issue 1727). With no such build, or one that recorded no
// fingerprint, there is none, and the reason says so.
func (ix *Indexer) builderInput() (src, none string) {
if ix.Builds != nil {
if r := ix.Builds.TrunkOf(BuilderBuiltBy); r != nil && r.SourceFingerprint != "" {
return r.SourceFingerprint, ""
}
}
return "", fmt.Sprintf("%s, which builds it, has no successful trunk build that recorded a source fingerprint", BuilderBuiltBy)
}
// RecomputeForBuild is Recompute for a build of BuilderBuiltBy heard on the builds bucket, or the one at start (issue 1727).
// While the indexer has read no toolchain at all (it has just started and the toolchains buckets are not settled), every
// RecomputeAtStart is Recompute once the indexer has started. While the indexer has read no toolchain at all (it has just started and the toolchains buckets are not settled), every
// Go module would be written as having no toolchain reported and then back: it computes nothing then, and the settling of
// the toolchains computes them.
func (ix *Indexer) RecomputeForBuild(ctx context.Context) error {
func (ix *Indexer) RecomputeAtStart(ctx context.Context) error {
ix.mu.Lock()
none := len(ix.toolchains) == 0
ix.mu.Unlock()
if none {
ix.logf("the builds changed before any toolchain was read: the new-form fingerprints are computed when they are")
ix.logf("no toolchain is read yet: the new-form fingerprints are computed when one is")
return nil
}
return ix.Recompute(ctx)
@@ -117,8 +117,8 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
m, _ := moduleOf(t, mods, "avahi")
if m.None2 != None2Builder || m.FP2 != "" {
t.Errorf("none2 = %q fp2 = %q, want none2 %q", m.None2, m.FP2, None2Builder)
if m.None2 != "" || m.FP2 == "" {
t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.FP2)
}
}
@@ -160,33 +160,31 @@ func fp2(t *testing.T, in delivery.SourceInputs2) string {
return fp
}
func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) { // 1444/AC7, AC8 as 1727 corrects it
// 1887/I1, ADR 0530 §1: bases are computed before the modules taking them, and no module's fingerprint takes the builder,
// the builder's own included: a trunk build of build-agent (or of the builder) moves nothing.
func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // 1444/AC7, AC8
ix, mods := newFormIndexer(t, builderModules(t)...)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agentSrcX))
toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", builderFP))
want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", builderFP, toolsFP))}
builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", ""))
toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", ""))
want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", "", toolsFP))}
for name, fp := range want {
if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" {
t.Errorf("%s: fp2 = %q none2 = %q, want %s", name, m.FP2, m.None2, fp)
if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" {
t.Errorf("%s: fp2 = %q none2 = %q inputs %+v, want %s with no builder input", name, m.FP2, m.None2, m.Inputs2, fp)
}
}
if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2)
}
// 1727: a new trunk build of build-agent records another source fingerprint; with no merge, the builder's value moves
// with it, and so do the modules built by it.
a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z")
ix.Builds.Put(a[0].(string), a[1].([]byte))
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
newBuilder := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agentSrcY))
if m, _ := moduleOf(t, mods, "mesh-delivery-builder"); m.FP2 != newBuilder || newBuilder == builderFP {
t.Errorf("builder: fp2 = %q, want %s (was %s)", m.FP2, newBuilder, builderFP)
}
if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != fp2(t, goInputs(t, ix, mods, "mesh-tools", newBuilder)) {
t.Errorf("mesh-tools did not follow the builder: %+v", m)
for name, fp := range want {
if m, _ := moduleOf(t, mods, name); m.FP2 != fp {
t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.FP2, fp)
}
}
}
@@ -201,7 +199,7 @@ func TestABuildHeardBeforeAnyToolchainIsReadDropsNoFingerprint(t *testing.T) { /
ix.mu.Lock()
ix.toolchains = nil
ix.mu.Unlock()
if err := ix.RecomputeForBuild(context.Background()); err != nil {
if err := ix.RecomputeAtStart(context.Background()); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != before.FP2 || m.None2 != "" {
@@ -209,27 +207,6 @@ func TestABuildHeardBeforeAnyToolchainIsReadDropsNoFingerprint(t *testing.T) { /
}
}
func TestTheBuilderHasNoFingerprintWithoutATrunkBuildOfBuildAgentAndSaysSo(t *testing.T) { // 1727
ix, mods := newFormIndexer(t, builderModules(t)...)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
ix.Builds.Delete(delivery.BuildKey("build-agent", "build-1792580585452696147"))
// a failed build and a build of a branch are no trunk build
failed, fv := record(t, "b-failed", "build-agent", agentCloneURL, "modules/build-agent", "main", "failed", "2026-10-21T15:03:00Z", aInputs)
branch, bv := record(t, "b-branch", "build-agent", agentCloneURL, "modules/build-agent", "fix/x", "", "2026-10-21T15:04:00Z", aInputs)
ix.Builds.Put(branch, bv)
ix.Builds.Put(failed, fv)
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "mesh-delivery-builder")
if m.FP2 != "" || !strings.Contains(m.None2, "build-agent") || !strings.Contains(m.None2, "no successful trunk build") {
t.Errorf("builder: fp2 = %q none2 = %q, want none2 naming build-agent's missing trunk build", m.FP2, m.None2)
}
if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != "" || !strings.Contains(m.None2, "builder") {
t.Errorf("mesh-tools: fp2 = %q none2 = %q, want none2 naming the builder", m.FP2, m.None2)
}
}
func TestAToolchainEntryChangedToTheSameValuesWritesNothing(t *testing.T) { // 1444/AC9
ix, mods := newFormIndexer(t, builderModules(t)...)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
@@ -261,8 +238,8 @@ func TestAMergeWritesEachModuleOnceWithItsFinalValueAndAReplayWritesNothing(t *t
t.Fatal(err)
}
m, _ := moduleOf(t, mods, "avahi")
if mods.writes != 1 || m.None2 != None2Builder {
t.Errorf("writes = %d none2 = %q after one merge, want one write with %q", mods.writes, m.None2, None2Builder)
if mods.writes != 1 || m.FP2 == "" {
t.Errorf("writes = %d fp2 = %q after one merge, want one write with its fingerprint", mods.writes, m.FP2)
}
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)