diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 747700c..80ab566 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -48,8 +48,10 @@ type Indexer struct { Builds *Builds Logf func(format string, args ...any) - mu sync.Mutex // guards source, which the source watch and the forge messages both reach - source sourceNow + mu sync.Mutex // guards source and toolchains, which the watches and the forge messages all reach + source sourceNow + toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go) + recompute sync.Mutex // one Recompute at a time } func (ix *Indexer) logf(format string, args ...any) { diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 753df87..1538192 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -50,7 +50,8 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) { // (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already // holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it // also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449, -// 1450); this release writes no fingerprint of the new form for any module. +// 1450); the fingerprint itself is computed by Recompute (toolchains.go) once the merge is written, and again when a build +// module reports its toolchain. func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { digests, err := ix.keepManifests(ctx, f) if err != nil { @@ -89,7 +90,8 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error { return fmt.Errorf("modules.%s: %w", key, err) } } - return nil + // What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order. + return ix.Recompute(ctx) } // baseFP answers the new-form fingerprint of a base module as modules. holds it, or why it has none. diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go new file mode 100644 index 0000000..fac1331 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -0,0 +1,306 @@ +package index + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The new-form fingerprint of a module is written, and written again with no merge, when what it is of changes +// (novox/hq ADR 0465 clause 2; issue 1444): a module's tree and lock hashes come with a merge and are kept in +// modules..Inputs2; its builder's, its bases' and its toolchain's come from here. The indexer holds no graph, so each +// recompute reads the manifest of every module from manifests. and takes the modules in the order their bases +// and the builder come first. + +const ( + // BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is + // its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it. + BuilderModule = "mesh-delivery-builder" + // BuilderBuiltBy is the module that builds BuilderModule until release R4g: its fingerprint of today's form is the + // builder's own Builder input, so a change of build-agent moves the builder's value (ADR 0453 clause 5). + BuilderBuiltBy = "build-agent" +) + +// ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the +// indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on. +var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "mesh-build-node"} + +// PutToolchain takes one change of a build module's toolchains bucket in, without writing any module: a put of a node's +// toolchain of a language (key ., delivery.ToolchainKey), or its delete. It answers whether what the +// language's toolchain is of (version and recipe) changed. A key, or a value, that is not a toolchain of the module's +// language is said in the log and left out. +func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool) bool { + language := "" + for l, m := range ToolchainModules { + if m == module { + language = l + } + } + if language == "" { + ix.logf("toolchains of %q are not read: it is no build module the indexer knows", module) + return false + } + node, keyLanguage, err := delivery.ParseToolchainKey(key) + if err != nil || keyLanguage != language { + ix.logf("%s_toolchains.%s is not a %s toolchain's key, left out: %v", module, key, language, err) + return false + } + ix.mu.Lock() + defer ix.mu.Unlock() + if deleted { + if _, had := ix.toolchains[language][node]; !had { + return false + } + delete(ix.toolchains[language], node) + return true + } + tc, err := delivery.DecodeToolchain(value) + if err != nil || tc.Language != language { + ix.logf("%s_toolchains.%s is not a %s toolchain, left out: %v", module, key, language, err) + return false + } + if ix.toolchains == nil { + ix.toolchains = map[string]map[string]delivery.Toolchain{} + } + if ix.toolchains[language] == nil { + ix.toolchains[language] = map[string]delivery.Toolchain{} + } + cur, had := ix.toolchains[language][node] + ix.toolchains[language][node] = tc + return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe +} + +// ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain +// changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing. +func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error { + if !ix.PutToolchain(module, key, value, deleted) { + return nil + } + return ix.Recompute(ctx) +} + +// agreed is the toolchain of each language every reporting node agrees on, or why there is none. +func (ix *Indexer) agreed() map[string]toolchainOrNone { + ix.mu.Lock() + defer ix.mu.Unlock() + out := map[string]toolchainOrNone{} + for language := range ToolchainModules { + var entries []delivery.NodeToolchain + for _, node := range slices.Sorted(maps.Keys(ix.toolchains[language])) { + entries = append(entries, delivery.NodeToolchain{Node: node, Toolchain: ix.toolchains[language][node]}) + } + tc, none := delivery.AgreedToolchain(entries) + out[language] = toolchainOrNone{tc, none} + } + return out +} + +type toolchainOrNone struct { + tc delivery.Toolchain2 + none string +} + +// known is a module the indexer holds the inputs of: its record as read, the manifest's shape and the key it is written at. +type known struct { + key string + cur delivery.Module + languages map[string]string + bases []string + contexts bool +} + +// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from +// the inputs kept at its merge (Inputs2), its manifest (manifests.), the fingerprints of its bases and of the +// builder, and the toolchains the build modules report. A module is computed after its bases and the builder, whatever +// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written +// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with +// no kept inputs (a message that carried no manifest) is left as it is. +func (ix *Indexer) Recompute(ctx context.Context) error { + if ix.Modules == nil || ix.Manifests == nil { + return nil + } + ix.recompute.Lock() + defer ix.recompute.Unlock() + keys, err := ix.Modules.Keys(ctx) + if err != nil { + return fmt.Errorf("listing the modules: %w", err) + } + slices.Sort(keys) + held := map[string]delivery.Module{} + all := map[string]*known{} + for _, key := range keys { + value, _, found, err := ix.Modules.Get(ctx, key) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) + } + if !found { + continue + } + cur, err := delivery.DecodeModule(value) + if err != nil { + ix.logf("modules.%s holds what is not a module, left: %v", key, err) + continue + } + held[key] = cur + if cur.Manifest == "" || cur.Inputs2 == nil { + continue + } + m, err := ix.manifestOf(ctx, cur.Manifest) + if err != nil { + ix.logf("modules.%s: its manifest %s cannot be read, left: %v", key, cur.Manifest, err) + continue + } + all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)} + } + tools := ix.agreed() + type result struct{ fp, none string } + done := map[string]result{} + visiting := map[string]bool{} + var of func(module string, chain []string) result + of = func(module string, chain []string) result { + if r, ok := done[module]; ok { + return r + } + k, ours := all[module] + if !ours { // a module the indexer holds no inputs of: its record, as written + cur, has := held[module] + switch { + case !has: + return result{"", "the indexer holds no record of it"} + case cur.FP2 != "": + return result{cur.FP2, ""} + case cur.None2 != "": + return result{"", cur.None2} + } + return result{"", "its record carries no fingerprint of the new form"} + } + if visiting[module] { + return result{"", "a cycle of bases: " + strings.Join(append(chain, module), " -> ")} + } + visiting[module] = true + defer delete(visiting, module) + chain = append(chain, module) + kept := *k.cur.Inputs2 + kept.Bases, kept.Builder = nil, "" + var missing []string + for _, b := range k.bases { + if r := of(b, chain); r.fp != "" { + kept.Bases = append(kept.Bases, r.fp) + } else { + missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", b, r.none)) + } + } + slices.Sort(kept.Bases) + slices.Sort(missing) + toolchains := map[string]delivery.Toolchain2{} + var noToolchain string + for _, artifact := range slices.Sorted(maps.Keys(k.languages)) { + t := tools[k.languages[artifact]] + if t.none != "" { + if noToolchain == "" { + noToolchain = t.none + } + continue + } + toolchains[artifact] = t.tc + } + var noBuilder string + if module == BuilderModule { + // Built by build-agent until release R4g: its value of today's form. + if a := held[BuilderBuiltBy]; a.FP != "" { + kept.Builder = a.FP + } else { + noBuilder = fmt.Sprintf("%s, which builds it, has no fingerprint (%s)", BuilderBuiltBy, a.None) + } + } else if r := of(BuilderModule, chain); r.fp != "" { + kept.Builder = r.fp + } else { + noBuilder = None2Builder + if _, has := held[BuilderModule]; has { + noBuilder += ": " + r.none + } + } + var r result + switch { + case k.contexts: + r.none = None2Context + case len(missing) > 0: + r.none = strings.Join(missing, "; ") + case noToolchain != "": + r.none = noToolchain + case noBuilder != "": + r.none = noBuilder + default: + r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains)) + } + done[module] = r + k.cur.Inputs2 = &kept + return r + } + for _, key := range slices.Sorted(maps.Keys(all)) { + k := all[key] + r := of(key, nil) + if k.cur.FP2 == r.fp && k.cur.None2 == r.none { + continue + } + digest, inputs := k.cur.Manifest, k.cur.Inputs2 + err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { + if !found { + return nil, false, nil + } + cur, err := delivery.DecodeModule(value) + if err != nil || cur.Manifest != digest || cur.Inputs2 == nil { + return nil, false, nil // a merge changed it meanwhile; its own recompute follows + } + if cur.FP2 == r.fp && cur.None2 == r.none { + return nil, false, nil + } + cur.FP2, cur.None2 = r.fp, r.none + cur.Inputs2 = inputs + data, err := delivery.EncodeModule(cur) + return data, false, err + }) + if err != nil { + return fmt.Errorf("modules.%s: %w", key, err) + } + } + return nil +} + +// manifestOf is the parsed manifest manifests. holds. +func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Manifest, error) { + key, err := delivery.ManifestsKey(digest) + if err != nil { + return catalogue.Manifest{}, err + } + value, _, found, err := ix.Manifests.Get(ctx, key) + if err != nil { + return catalogue.Manifest{}, err + } + if !found { + return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key) + } + kept, err := delivery.DecodeManifest(value) + if err != nil { + return catalogue.Manifest{}, err + } + return catalogue.ParseManifest([]byte(kept.JSON)) +} + +// hasContext is whether an artifact of the manifest reads another repository's tree. +func hasContext(m catalogue.Manifest) bool { + if m.Build == nil { + return false + } + for _, a := range m.Build.Artifacts { + if a.Context != nil { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go new file mode 100644 index 0000000..32de130 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -0,0 +1,196 @@ +package index + +import ( + "context" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The fixtures here are mesh-catalog a67f0196's manifests (testdata, captured from that commit) with the module renamed (and, +// where a test needs one, a build.on entry added as the controller's own test has it, standing_on_test.go:22): the manifest +// parser, not a hand-written manifest, reads them. + +func renamed(t *testing.T, from, to string) string { + t.Helper() + raw := catalogueManifest(t, from) + out := strings.Replace(raw, `"module": "`+from+`"`, `"module": "`+to+`"`, 1) + if out == raw { + t.Fatalf("%s's manifest has no module line", from) + } + return out +} + +func standingOn(t *testing.T, raw, base string) string { + t.Helper() + out := strings.Replace(raw, `"build": {`, `"build": { + "on": [{"arg": "RUNTIME_BASE", "module": "`+base+`", "artifact": "runtime"}],`, 1) + if out == raw { + t.Fatal("the manifest has no build") + } + return out +} + +const buildAgentRepo = "novox/mesh-controller" + +type mod struct { + name, manifest string +} + +// merged applies one merge of the catalogue carrying these modules (every one a Go bundle's go.sum, a TypeScript bundle's +// package-lock.json) and returns the indexer. +func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) { + t.Helper() + var recs [][2]any + trees, manifests := map[string]string{}, map[string]string{} + locks := map[string]map[string]string{} + for i, m := range mods { + dir := "modules/" + m.name + recs = append(recs, rec(t, "build-"+m.name, m.name, catalogueRepo, dir, "main", "", "2026-10-19T09:00:00Z", aInputs)) + trees[dir], manifests[dir] = "tree-"+m.name, m.manifest + locks[dir] = map[string]string{"go.sum": avahiGoSum, "package-lock.json": strings.Repeat("c", 63) + string(rune('a'+i))} + } + recs = append(recs, rec(t, "build-agent-1", "build-agent", buildAgentRepo, "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", aInputs)) + ix, store, _ := manifestsIndexer(t, recs...) + f := withManifests(mergedMsg(catalogueRepo, trees), manifests, locks) + if err := ix.Apply(context.Background(), f); err != nil { + t.Fatal(err) + } + // build-agent's value of today's form, from a merge of the controller (modules.build-agent holds its fingerprint). + if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-1"})); err != nil { + t.Fatal(err) + } + return ix, store +} + +// toolchainValue is a toolchains value as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain). +func toolchainValue(t *testing.T, language, version, recipe string) []byte { + t.Helper() + b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now()}) + if err != nil { + t.Fatal(err) + } + return b +} + +func reportToolchain(t *testing.T, ix *Indexer, module, node, language, version string) { + t.Helper() + key, err := delivery.ToolchainKey(node, language) + if err != nil { + t.Fatal(err) + } + if err := ix.ApplyToolchain(context.Background(), module, key, toolchainValue(t, language, version, "1"), false); err != nil { + t.Fatal(err) + } +} + +func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 1444/AC5 + ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) + if m, _ := moduleOf(t, mods, "avahi"); m.None2 != delivery.ToolchainNoReport { + t.Fatalf("before: none2 = %q", m.None2) + } + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1") + m, _ := moduleOf(t, mods, "avahi") + if m.None2 != None2Builder || m.FP2 != "" { + t.Errorf("none2 = %q fp2 = %q, want none2 %q", m.None2, m.FP2, None2Builder) + } +} + +func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6 + ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0") + m, _ := moduleOf(t, mods, "avahi") + if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") { + t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2) + } +} + +// builderModules are the builder, a base, a module on the base (named so that it is stored before its base) and a module +// whose base is a TypeScript module with no toolchain reported. +func builderModules(t *testing.T) []mod { + return []mod{ + {"mesh-delivery-builder", renamed(t, "avahi", "mesh-delivery-builder")}, + {"mesh-tools", renamed(t, "avahi", "mesh-tools")}, + {"avahi", standingOn(t, catalogueManifest(t, "avahi"), "mesh-tools")}, + {"other", standingOn(t, renamed(t, "avahi", "other"), "ts-base")}, + {"ts-base", renamed(t, "audit-logger", "ts-base")}, + } +} + +func goInputs(t *testing.T, ix *Indexer, mods *memStore, module, builder string, bases ...string) delivery.SourceInputs2 { + t.Helper() + return delivery.SourceInputs2{Module: module, Tree: "tree-" + module, Builder: builder, Bases: bases, + Locks: map[string]string{"tools": avahiGoSum}, + Toolchains: map[string]delivery.Toolchain2{"tools": {Language: "go", Version: "go1.27.1", Recipe: "1"}}} +} + +func fp2(t *testing.T, in delivery.SourceInputs2) string { + t.Helper() + fp, none := delivery.SourceFingerprint2(in) + if none != "" { + t.Fatal(none) + } + return fp +} + +func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) { // 1444/AC7, AC8 + ix, mods := newFormIndexer(t, builderModules(t)...) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + agent, _ := moduleOf(t, mods, "build-agent") + if agent.FP == "" { + t.Fatalf("build-agent has no fingerprint of today's form: %+v", agent) + } + builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agent.FP)) + toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", builderFP)) + want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", builderFP, toolsFP))} + for name, fp := range want { + if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" { + t.Errorf("%s: fp2 = %q none2 = %q, want %s", name, m.FP2, m.None2, fp) + } + } + if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") { + t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2) + } + // 1444/AC8: build-agent changes at a merge of the controller; the builder's value moves with it, and so do the modules built by it. + if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-2"})); err != nil { + t.Fatal(err) + } + moved, _ := moduleOf(t, mods, "build-agent") + if moved.FP == agent.FP { + t.Fatal("build-agent's fingerprint did not change") + } + newBuilder := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", moved.FP)) + if m, _ := moduleOf(t, mods, "mesh-delivery-builder"); m.FP2 != newBuilder || newBuilder == builderFP { + t.Errorf("builder: fp2 = %q, want %s (was %s)", m.FP2, newBuilder, builderFP) + } + if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != fp2(t, goInputs(t, ix, mods, "mesh-tools", newBuilder)) { + t.Errorf("mesh-tools did not follow the builder: %+v", m) + } +} + +func TestAToolchainEntryChangedToTheSameValuesWritesNothing(t *testing.T) { // 1444/AC9 + ix, mods := newFormIndexer(t, builderModules(t)...) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + before := mods.writes + time.Sleep(2 * time.Millisecond) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") // the same version and recipe, a later time + if mods.writes != before { + t.Errorf("modules written %d times after the same values, want none", mods.writes-before) + } +} + +func TestModulesThatStandOnEachOtherAreNamedAsACycle(t *testing.T) { // 1444 (the indexer has no graph: a cycle is a reason) + ix, mods := newFormIndexer(t, + mod{"aa", standingOn(t, renamed(t, "avahi", "aa"), "bb")}, + mod{"bb", standingOn(t, renamed(t, "avahi", "bb"), "aa")}) + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + for _, name := range []string{"aa", "bb"} { + if m, _ := moduleOf(t, mods, name); m.FP2 != "" || !strings.Contains(m.None2, "cycle") { + t.Errorf("%s: fp2 = %q none2 = %q, want a reason naming the cycle", name, m.FP2, m.None2) + } + } +}