Recompute the new-form fingerprints when a build module reports its toolchain (issue 1444), part 1 of 2

This commit is contained in:
2026-10-20 20:47:35 +00:00
parent 340bf64388
commit 74fef7dd5a
4 changed files with 510 additions and 4 deletions
@@ -48,8 +48,10 @@ type Indexer struct {
Builds *Builds
Logf func(format string, args ...any)
mu sync.Mutex // guards source, which the source watch and the forge messages both reach
source sourceNow
mu sync.Mutex // guards source and toolchains, which the watches and the forge messages all reach
source sourceNow
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
recompute sync.Mutex // one Recompute at a time
}
func (ix *Indexer) logf(format string, args ...any) {
@@ -50,7 +50,8 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) {
// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already
// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it
// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449,
// 1450); this release writes no fingerprint of the new form for any module.
// 1450); the fingerprint itself is computed by Recompute (toolchains.go) once the merge is written, and again when a build
// module reports its toolchain.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
digests, err := ix.keepManifests(ctx, f)
if err != nil {
@@ -89,7 +90,8 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
return fmt.Errorf("modules.%s: %w", key, err)
}
}
return nil
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
return ix.Recompute(ctx)
}
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
@@ -0,0 +1,306 @@
package index
import (
"context"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The new-form fingerprint of a module is written, and written again with no merge, when what it is of changes
// (novox/hq ADR 0465 clause 2; issue 1444): a module's tree and lock hashes come with a merge and are kept in
// modules.<m>.Inputs2; its builder's, its bases' and its toolchain's come from here. The indexer holds no graph, so each
// recompute reads the manifest of every module from manifests.<digest> and takes the modules in the order their bases
// and the builder come first.
const (
// BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is
// its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it.
BuilderModule = "mesh-delivery-builder"
// BuilderBuiltBy is the module that builds BuilderModule until release R4g: its fingerprint of today's form is the
// builder's own Builder input, so a change of build-agent moves the builder's value (ADR 0453 clause 5).
BuilderBuiltBy = "build-agent"
)
// ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the
// indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on.
var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "mesh-build-node"}
// PutToolchain takes one change of a build module's toolchains bucket in, without writing any module: a put of a node's
// toolchain of a language (key <node>.<language>, delivery.ToolchainKey), or its delete. It answers whether what the
// language's toolchain is of (version and recipe) changed. A key, or a value, that is not a toolchain of the module's
// language is said in the log and left out.
func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool) bool {
language := ""
for l, m := range ToolchainModules {
if m == module {
language = l
}
}
if language == "" {
ix.logf("toolchains of %q are not read: it is no build module the indexer knows", module)
return false
}
node, keyLanguage, err := delivery.ParseToolchainKey(key)
if err != nil || keyLanguage != language {
ix.logf("%s_toolchains.%s is not a %s toolchain's key, left out: %v", module, key, language, err)
return false
}
ix.mu.Lock()
defer ix.mu.Unlock()
if deleted {
if _, had := ix.toolchains[language][node]; !had {
return false
}
delete(ix.toolchains[language], node)
return true
}
tc, err := delivery.DecodeToolchain(value)
if err != nil || tc.Language != language {
ix.logf("%s_toolchains.%s is not a %s toolchain, left out: %v", module, key, language, err)
return false
}
if ix.toolchains == nil {
ix.toolchains = map[string]map[string]delivery.Toolchain{}
}
if ix.toolchains[language] == nil {
ix.toolchains[language] = map[string]delivery.Toolchain{}
}
cur, had := ix.toolchains[language][node]
ix.toolchains[language][node] = tc
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
}
// ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain
// changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing.
func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error {
if !ix.PutToolchain(module, key, value, deleted) {
return nil
}
return ix.Recompute(ctx)
}
// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
func (ix *Indexer) agreed() map[string]toolchainOrNone {
ix.mu.Lock()
defer ix.mu.Unlock()
out := map[string]toolchainOrNone{}
for language := range ToolchainModules {
var entries []delivery.NodeToolchain
for _, node := range slices.Sorted(maps.Keys(ix.toolchains[language])) {
entries = append(entries, delivery.NodeToolchain{Node: node, Toolchain: ix.toolchains[language][node]})
}
tc, none := delivery.AgreedToolchain(entries)
out[language] = toolchainOrNone{tc, none}
}
return out
}
type toolchainOrNone struct {
tc delivery.Toolchain2
none string
}
// known is a module the indexer holds the inputs of: its record as read, the manifest's shape and the key it is written at.
type known struct {
key string
cur delivery.Module
languages map[string]string
bases []string
contexts bool
}
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
// the inputs kept at its merge (Inputs2), its manifest (manifests.<digest>), the fingerprints of its bases and of the
// builder, and the toolchains the build modules report. A module is computed after its bases and the builder, whatever
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
// no kept inputs (a message that carried no manifest) is left as it is.
func (ix *Indexer) Recompute(ctx context.Context) error {
if ix.Modules == nil || ix.Manifests == nil {
return nil
}
ix.recompute.Lock()
defer ix.recompute.Unlock()
keys, err := ix.Modules.Keys(ctx)
if err != nil {
return fmt.Errorf("listing the modules: %w", err)
}
slices.Sort(keys)
held := map[string]delivery.Module{}
all := map[string]*known{}
for _, key := range keys {
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
if !found {
continue
}
cur, err := delivery.DecodeModule(value)
if err != nil {
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
continue
}
held[key] = cur
if cur.Manifest == "" || cur.Inputs2 == nil {
continue
}
m, err := ix.manifestOf(ctx, cur.Manifest)
if err != nil {
ix.logf("modules.%s: its manifest %s cannot be read, left: %v", key, cur.Manifest, err)
continue
}
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
}
tools := ix.agreed()
type result struct{ fp, none string }
done := map[string]result{}
visiting := map[string]bool{}
var of func(module string, chain []string) result
of = func(module string, chain []string) result {
if r, ok := done[module]; ok {
return r
}
k, ours := all[module]
if !ours { // a module the indexer holds no inputs of: its record, as written
cur, has := held[module]
switch {
case !has:
return result{"", "the indexer holds no record of it"}
case cur.FP2 != "":
return result{cur.FP2, ""}
case cur.None2 != "":
return result{"", cur.None2}
}
return result{"", "its record carries no fingerprint of the new form"}
}
if visiting[module] {
return result{"", "a cycle of bases: " + strings.Join(append(chain, module), " -> ")}
}
visiting[module] = true
defer delete(visiting, module)
chain = append(chain, module)
kept := *k.cur.Inputs2
kept.Bases, kept.Builder = nil, ""
var missing []string
for _, b := range k.bases {
if r := of(b, chain); r.fp != "" {
kept.Bases = append(kept.Bases, r.fp)
} else {
missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", b, r.none))
}
}
slices.Sort(kept.Bases)
slices.Sort(missing)
toolchains := map[string]delivery.Toolchain2{}
var noToolchain string
for _, artifact := range slices.Sorted(maps.Keys(k.languages)) {
t := tools[k.languages[artifact]]
if t.none != "" {
if noToolchain == "" {
noToolchain = t.none
}
continue
}
toolchains[artifact] = t.tc
}
var noBuilder string
if module == BuilderModule {
// Built by build-agent until release R4g: its value of today's form.
if a := held[BuilderBuiltBy]; a.FP != "" {
kept.Builder = a.FP
} else {
noBuilder = fmt.Sprintf("%s, which builds it, has no fingerprint (%s)", BuilderBuiltBy, a.None)
}
} else if r := of(BuilderModule, chain); r.fp != "" {
kept.Builder = r.fp
} else {
noBuilder = None2Builder
if _, has := held[BuilderModule]; has {
noBuilder += ": " + r.none
}
}
var r result
switch {
case k.contexts:
r.none = None2Context
case len(missing) > 0:
r.none = strings.Join(missing, "; ")
case noToolchain != "":
r.none = noToolchain
case noBuilder != "":
r.none = noBuilder
default:
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
}
done[module] = r
k.cur.Inputs2 = &kept
return r
}
for _, key := range slices.Sorted(maps.Keys(all)) {
k := all[key]
r := of(key, nil)
if k.cur.FP2 == r.fp && k.cur.None2 == r.none {
continue
}
digest, inputs := k.cur.Manifest, k.cur.Inputs2
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
cur, err := delivery.DecodeModule(value)
if err != nil || cur.Manifest != digest || cur.Inputs2 == nil {
return nil, false, nil // a merge changed it meanwhile; its own recompute follows
}
if cur.FP2 == r.fp && cur.None2 == r.none {
return nil, false, nil
}
cur.FP2, cur.None2 = r.fp, r.none
cur.Inputs2 = inputs
data, err := delivery.EncodeModule(cur)
return data, false, err
})
if err != nil {
return fmt.Errorf("modules.%s: %w", key, err)
}
}
return nil
}
// manifestOf is the parsed manifest manifests.<digest> holds.
func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Manifest, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return catalogue.Manifest{}, err
}
value, _, found, err := ix.Manifests.Get(ctx, key)
if err != nil {
return catalogue.Manifest{}, err
}
if !found {
return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key)
}
kept, err := delivery.DecodeManifest(value)
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.ParseManifest([]byte(kept.JSON))
}
// hasContext is whether an artifact of the manifest reads another repository's tree.
func hasContext(m catalogue.Manifest) bool {
if m.Build == nil {
return false
}
for _, a := range m.Build.Artifacts {
if a.Context != nil {
return true
}
}
return false
}
@@ -0,0 +1,196 @@
package index
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The fixtures here are mesh-catalog a67f0196's manifests (testdata, captured from that commit) with the module renamed (and,
// where a test needs one, a build.on entry added as the controller's own test has it, standing_on_test.go:22): the manifest
// parser, not a hand-written manifest, reads them.
func renamed(t *testing.T, from, to string) string {
t.Helper()
raw := catalogueManifest(t, from)
out := strings.Replace(raw, `"module": "`+from+`"`, `"module": "`+to+`"`, 1)
if out == raw {
t.Fatalf("%s's manifest has no module line", from)
}
return out
}
func standingOn(t *testing.T, raw, base string) string {
t.Helper()
out := strings.Replace(raw, `"build": {`, `"build": {
"on": [{"arg": "RUNTIME_BASE", "module": "`+base+`", "artifact": "runtime"}],`, 1)
if out == raw {
t.Fatal("the manifest has no build")
}
return out
}
const buildAgentRepo = "novox/mesh-controller"
type mod struct {
name, manifest string
}
// merged applies one merge of the catalogue carrying these modules (every one a Go bundle's go.sum, a TypeScript bundle's
// package-lock.json) and returns the indexer.
func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) {
t.Helper()
var recs [][2]any
trees, manifests := map[string]string{}, map[string]string{}
locks := map[string]map[string]string{}
for i, m := range mods {
dir := "modules/" + m.name
recs = append(recs, rec(t, "build-"+m.name, m.name, catalogueRepo, dir, "main", "", "2026-10-19T09:00:00Z", aInputs))
trees[dir], manifests[dir] = "tree-"+m.name, m.manifest
locks[dir] = map[string]string{"go.sum": avahiGoSum, "package-lock.json": strings.Repeat("c", 63) + string(rune('a'+i))}
}
recs = append(recs, rec(t, "build-agent-1", "build-agent", buildAgentRepo, "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", aInputs))
ix, store, _ := manifestsIndexer(t, recs...)
f := withManifests(mergedMsg(catalogueRepo, trees), manifests, locks)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
// build-agent's value of today's form, from a merge of the controller (modules.build-agent holds its fingerprint).
if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-1"})); err != nil {
t.Fatal(err)
}
return ix, store
}
// toolchainValue is a toolchains value as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain).
func toolchainValue(t *testing.T, language, version, recipe string) []byte {
t.Helper()
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now()})
if err != nil {
t.Fatal(err)
}
return b
}
func reportToolchain(t *testing.T, ix *Indexer, module, node, language, version string) {
t.Helper()
key, err := delivery.ToolchainKey(node, language)
if err != nil {
t.Fatal(err)
}
if err := ix.ApplyToolchain(context.Background(), module, key, toolchainValue(t, language, version, "1"), false); err != nil {
t.Fatal(err)
}
}
func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 1444/AC5
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
if m, _ := moduleOf(t, mods, "avahi"); m.None2 != delivery.ToolchainNoReport {
t.Fatalf("before: none2 = %q", m.None2)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
m, _ := moduleOf(t, mods, "avahi")
if m.None2 != None2Builder || m.FP2 != "" {
t.Errorf("none2 = %q fp2 = %q, want none2 %q", m.None2, m.FP2, None2Builder)
}
}
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
m, _ := moduleOf(t, mods, "avahi")
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
}
}
// builderModules are the builder, a base, a module on the base (named so that it is stored before its base) and a module
// whose base is a TypeScript module with no toolchain reported.
func builderModules(t *testing.T) []mod {
return []mod{
{"mesh-delivery-builder", renamed(t, "avahi", "mesh-delivery-builder")},
{"mesh-tools", renamed(t, "avahi", "mesh-tools")},
{"avahi", standingOn(t, catalogueManifest(t, "avahi"), "mesh-tools")},
{"other", standingOn(t, renamed(t, "avahi", "other"), "ts-base")},
{"ts-base", renamed(t, "audit-logger", "ts-base")},
}
}
func goInputs(t *testing.T, ix *Indexer, mods *memStore, module, builder string, bases ...string) delivery.SourceInputs2 {
t.Helper()
return delivery.SourceInputs2{Module: module, Tree: "tree-" + module, Builder: builder, Bases: bases,
Locks: map[string]string{"tools": avahiGoSum},
Toolchains: map[string]delivery.Toolchain2{"tools": {Language: "go", Version: "go1.27.1", Recipe: "1"}}}
}
func fp2(t *testing.T, in delivery.SourceInputs2) string {
t.Helper()
fp, none := delivery.SourceFingerprint2(in)
if none != "" {
t.Fatal(none)
}
return fp
}
func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) { // 1444/AC7, AC8
ix, mods := newFormIndexer(t, builderModules(t)...)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
agent, _ := moduleOf(t, mods, "build-agent")
if agent.FP == "" {
t.Fatalf("build-agent has no fingerprint of today's form: %+v", agent)
}
builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agent.FP))
toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", builderFP))
want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", builderFP, toolsFP))}
for name, fp := range want {
if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" {
t.Errorf("%s: fp2 = %q none2 = %q, want %s", name, m.FP2, m.None2, fp)
}
}
if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2)
}
// 1444/AC8: build-agent changes at a merge of the controller; the builder's value moves with it, and so do the modules built by it.
if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-2"})); err != nil {
t.Fatal(err)
}
moved, _ := moduleOf(t, mods, "build-agent")
if moved.FP == agent.FP {
t.Fatal("build-agent's fingerprint did not change")
}
newBuilder := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", moved.FP))
if m, _ := moduleOf(t, mods, "mesh-delivery-builder"); m.FP2 != newBuilder || newBuilder == builderFP {
t.Errorf("builder: fp2 = %q, want %s (was %s)", m.FP2, newBuilder, builderFP)
}
if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != fp2(t, goInputs(t, ix, mods, "mesh-tools", newBuilder)) {
t.Errorf("mesh-tools did not follow the builder: %+v", m)
}
}
func TestAToolchainEntryChangedToTheSameValuesWritesNothing(t *testing.T) { // 1444/AC9
ix, mods := newFormIndexer(t, builderModules(t)...)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
before := mods.writes
time.Sleep(2 * time.Millisecond)
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") // the same version and recipe, a later time
if mods.writes != before {
t.Errorf("modules written %d times after the same values, want none", mods.writes-before)
}
}
func TestModulesThatStandOnEachOtherAreNamedAsACycle(t *testing.T) { // 1444 (the indexer has no graph: a cycle is a reason)
ix, mods := newFormIndexer(t,
mod{"aa", standingOn(t, renamed(t, "avahi", "aa"), "bb")},
mod{"bb", standingOn(t, renamed(t, "avahi", "bb"), "aa")})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
for _, name := range []string{"aa", "bb"} {
if m, _ := moduleOf(t, mods, name); m.FP2 != "" || !strings.Contains(m.None2, "cycle") {
t.Errorf("%s: fp2 = %q none2 = %q, want a reason naming the cycle", name, m.FP2, m.None2)
}
}
}