From f70faed5b67be3a83ad6adbe7e6090ecea9b3add Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 16:27:57 +0100 Subject: [PATCH] =?UTF-8?q?The=20indexer=20keeps=20the=20module=20graph=20?= =?UTF-8?q?as=20its=20state=20graph:=20each=20module's=20build=20and=20pro?= =?UTF-8?q?vision=20edges=20written=20at=20every=20recompute=20when=20they?= =?UTF-8?q?=20change,=20at=20most=202000=20modules,=20a=20module's=20key?= =?UTF-8?q?=20deleted=20once=20the=20modules=20state=20no=20longer=20holds?= =?UTF-8?q?=20it=20(ADR=200603=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cmd/mesh-delivery-indexer/main.go | 4 +- .../internal/index/candidates.go | 2 + .../internal/index/graph.go | 131 ++++++++++++++++++ .../internal/index/manifests_test.go | 2 +- .../internal/index/toolchains.go | 10 +- .../internal/index/toolchains_test.go | 54 ++++++++ .../mesh-delivery-indexer/manifest_test.go | 3 +- modules/mesh-delivery-indexer/module.json | 3 +- 8 files changed, 203 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/graph.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index bbb31c68..eb111122 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -299,12 +299,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then // the judge refuses no use of an old section. oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf) + // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. + graph := process.AwaitBucket(ctx, js, conn, delivery.GraphBucket, "no module graph is written", logger.Printf) ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node)} + OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}} // The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each // new one indexes the module it names when that module is registered and the bucket holds none (registered.go). var started atomic.Bool diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 2e207599..e0cba4fe 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -67,6 +67,8 @@ type Indexer struct { // Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none, // and every fingerprint takes the version line. Packages Store + // Graph is the module graph's state (delivery.GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph. + Graph Store // Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each // with its git object id (gitea.gitea_commit_trees with files): the listing a package's digest is computed from. // Nil publishes no package. diff --git a/modules/mesh-delivery-indexer/internal/index/graph.go b/modules/mesh-delivery-indexer/internal/index/graph.go new file mode 100644 index 00000000..efcfad44 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/graph.go @@ -0,0 +1,131 @@ +package index + +import ( + "context" + "fmt" + "maps" + "slices" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The module graph (novox/hq ADR 0603 §2): at every recompute the indexer writes, in its state graph +// (delivery.GraphBucket), one key per module it holds the inputs of, its build and provision edges as that recompute +// computed them, and deletes the key of a module the modules state no longer holds. A key is written only when its value +// changes, so its history of 5 is five changes of the module's edges. The delivery view, the checker and the seat +// mesh-catalog's verbs read it; nothing stores an order (ADR 0603 §1). + +// GraphMost is the most modules the graph holds (ADR 0603 §2): past it, the modules after the first GraphMost in name +// order are not written, and it is said in the log. The cap is the indexer's code, not a manifest field, until a judge +// that knows such a field runs (the lead's decision of L3). +const GraphMost = 2000 + +// graphEdges is what a recompute knows of one module for its graph node. +type graphEdges struct { + module string + cur delivery.Module + manifest catalogue.Manifest + bases []string + fp, none string + tcs map[string]delivery.ToolchainUsed + pkgs delivery.PackageVersions +} + +// graphNode is the graph node of e: its edges, the awaited provisions judged against providers (catalogue.Awaited, as +// the assigner judges them). +func graphNode(e graphEdges, providers func(string) []catalogue.Manifest) delivery.GraphNode { + if e.fp == "" && e.none == "" { + e.none = "its fingerprint was not computed" + } + n := delivery.GraphNode{FP: e.fp, None: e.none, Repository: e.cur.Repository, Path: e.cur.Path, Bases: e.bases, + Packages: slices.Sorted(maps.Keys(e.pkgs))} + tcs := e.tcs + if len(tcs) == 0 { + tcs = e.cur.Toolchains // no fingerprint now: the toolchain it was last computed with still builds it + } + for language, t := range tcs { + builder, known := ToolchainModules[language] + if !known { + continue + } + if n.Toolchains == nil { + n.Toolchains = map[string]delivery.GraphToolchain{} + } + n.Toolchains[language] = delivery.GraphToolchain{Module: builder, Version: t.Version, Recipe: t.Recipe} + } + for _, o := range e.manifest.Provides { + n.Provides = append(n.Provides, o.Name) + } + for _, r := range e.manifest.Requires { + if !e.manifest.OptionalRequires[r] { + n.Requires = append(n.Requires, r) + } + } + n.Awaits = catalogue.Awaited(e.manifest, providers) + return n +} + +// writeGraph writes the graph node of every module of edges whose value changed, at most GraphMost of them, then +// deletes each graph key whose module the modules state no longer holds. Nil Graph writes nothing. +func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges) error { + if ix.Graph == nil { + return nil + } + names := slices.Sorted(maps.Keys(edges)) + if len(names) > GraphMost { + ix.logf("the module graph holds at most %d modules: %d are known, so %s and the %d after it are not written", + GraphMost, len(names), names[GraphMost], len(names)-GraphMost-1) + names = names[:GraphMost] + } + providers := func(provision string) []catalogue.Manifest { + var out []catalogue.Manifest + for _, name := range slices.Sorted(maps.Keys(edges)) { + if slices.Contains(edges[name].manifest.Offers(), provision) { + out = append(out, edges[name].manifest) + } + } + return out + } + for _, name := range names { + key, err := delivery.GraphKey(name) + if err != nil { + ix.logf("graph.%s is not written: %v", name, err) + continue + } + next := graphNode(edges[name], providers) + err = write(ctx, ix.Graph, key, func(value []byte, found bool) ([]byte, bool, error) { + if found { + if cur, err := delivery.DecodeGraphNode(value); err == nil && delivery.SameGraphNode(cur, next) { + return nil, false, nil + } + } + data, err := delivery.EncodeGraphNode(next) + if err != nil { + return nil, false, fmt.Errorf("graph.%s: %w", key, err) + } + return data, false, nil + }) + if err != nil { + ix.logf("graph.%s is not written: %v", key, err) + } + } + held, err := ix.Modules.Keys(ctx) + if err != nil { + return fmt.Errorf("listing the modules for the graph: %w", err) + } + keys, err := ix.Graph.Keys(ctx) + if err != nil { + return fmt.Errorf("listing the graph: %w", err) + } + for _, key := range keys { + if slices.Contains(held, key) { + continue + } + if err := write(ctx, ix.Graph, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { + return fmt.Errorf("graph.%s: %w", key, err) + } + ix.logf("graph.%s deleted: the modules state holds no module %s", key, key) + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 064dc488..835092ca 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -33,7 +33,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt t.Helper() ix, mods := modulesIndexer(t, recs...) manifests := newMem() - ix.Manifests = manifests + ix.Manifests, ix.Graph = manifests, newMem() return ix, mods, manifests } diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 269f8fd1..78d0dc83 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -111,6 +111,7 @@ type toolchainOrNone struct { type known struct { key string cur delivery.Module + manifest catalogue.Manifest languages map[string]string bases []string contexts bool @@ -196,7 +197,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli cur, stale[key] = again, true held[key] = cur } - all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before} + all[key] = &known{key: key, cur: cur, manifest: m, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before} } tools := ix.agreed() published := ix.publishedAll(ctx) @@ -278,9 +279,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli return r } finals := map[string]delivery.Module{} + edges := map[string]graphEdges{} // the module graph's nodes of this recompute (graph.go) for _, key := range slices.Sorted(maps.Keys(all)) { k := all[key] r := of(key, nil) + edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, bases: k.bases, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs} if _, merged := overlay[key]; merged { final := overlay[key] final.FP2, final.None2, final.Inputs2, final.Toolchains, final.Packages = r.fp, r.none, k.cur.Inputs2, r.tcs, r.pkgs @@ -332,7 +335,10 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli w.wrote(key) } } - return ix.writeOverlay(ctx, overlay, finals, w) + if err := ix.writeOverlay(ctx, overlay, finals, w); err != nil { + return err + } + return ix.writeGraph(ctx, edges) } // writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index 5ee8da5b..b9bd4efc 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -60,6 +60,20 @@ type mod struct { // merged applies one merge of the catalogue carrying these modules (every one a Go bundle's go.sum, a TypeScript bundle's // package-lock.json) and returns the indexer. +// graphOf is the graph node the indexer holds of module. +func graphOf(t *testing.T, ix *Indexer, module string) delivery.GraphNode { + t.Helper() + v, ok := ix.Graph.(*memStore).vals[module] + if !ok { + t.Fatalf("graph.%s is not held", module) + } + g, err := delivery.DecodeGraphNode(v) + if err != nil { + t.Fatal(err) + } + return g +} + func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) { t.Helper() ix, store, f := newFormIndexerUnmerged(t, mods...) @@ -122,9 +136,24 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14 if m, _ := moduleOf(t, mods, "avahi"); m.Src != 7 || m.FPSrc != 7 { t.Fatalf("before: src %d fp-src %d, want 7 and 7", m.Src, m.FPSrc) } + if g := graphOf(t, ix, "avahi"); g.None != delivery.ToolchainNoReport || g.Toolchains != nil { + t.Errorf("before: graph.avahi = %+v, want no toolchain edge and the reason", g) + } reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1") m, _ := moduleOf(t, mods, "avahi") + // L3 (ADR 0603 §2, §3): the agreement moving rewrites the module's toolchain edge, and again when it moves on. + if g := graphOf(t, ix, "avahi"); g.FP != m.FP2 || g.Toolchains["go"].Version != "go1.27.1" || g.Toolchains["go"].Module != "mesh-build-go" { + t.Errorf("after: graph.avahi = %+v, want the toolchain edge of go1.27.1", g) + } + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.28.0") + reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.28.0") + if g := graphOf(t, ix, "avahi"); g.Toolchains["go"].Version != "go1.28.0" { + t.Errorf("moved on: graph.avahi = %+v, want the toolchain edge of go1.28.0", g) + } + reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") + reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1") + m, _ = moduleOf(t, mods, "avahi") if m.None2 != "" || m.FP2 == "" { t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.FP2) } @@ -202,6 +231,31 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") { t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2) } + // L3 (ADR 0603 §2): the module graph holds each module's edges as this recompute computed them. + goTool := map[string]delivery.GraphToolchain{"go": {Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}} + if g := graphOf(t, ix, "avahi"); g.FP != want["avahi"] || !reflect.DeepEqual(g.Bases, []string{"mesh-tools"}) || + !reflect.DeepEqual(g.Toolchains, goTool) || g.Repository != catalogueRepo || g.Path != "modules/avahi" { + t.Errorf("graph.avahi = %+v", g) + } + if g := graphOf(t, ix, "other"); g.FP != "" || !strings.Contains(g.None, "ts-base") || !reflect.DeepEqual(g.Bases, []string{"ts-base"}) { + t.Errorf("graph.other = %+v, want its base and its reason", g) + } + graph := ix.Graph.(*memStore) + written := graph.writes + if err := ix.Recompute(context.Background()); err != nil || graph.writes != written { + t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err) + } + // A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute. + delete(mods.vals, "other") + if err := ix.Recompute(context.Background()); err != nil { + t.Fatal(err) + } + if _, held := graph.vals["other"]; held { + t.Error("graph.other is still held after its module went") + } + if _, held := graph.vals["avahi"]; !held { + t.Error("graph.avahi went with another module") + } a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z") _ = ix.Builds.Put(a[1].([]byte)) if err := ix.Recompute(context.Background()); err != nil { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 0c6c61f5..c77c7b80 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -90,7 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, + {Name: delivery.GraphState, History: delivery.GraphHistory}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 45067500..11fac677 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -25,7 +25,8 @@ { "name": "takeins", "history": 1 }, { "name": "verdicts", "history": 5 }, { "name": "oldsections", "history": 1 }, - { "name": "packages", "history": 5 } + { "name": "packages", "history": 5 }, + { "name": "graph", "history": 5 } ], "emits": [ "manifest-refused",