The indexer keeps the module graph as its state graph: each module's build and provision edges written at every recompute when they change, at most 2000 modules, a module's key deleted once the modules state no longer holds it (ADR 0603 §2)

This commit is contained in:
2026-11-03 16:45:06 +01:00
parent 5ee3b4c15f
commit f70faed5b6
8 changed files with 203 additions and 6 deletions
@@ -299,12 +299,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
// the judge refuses no use of an old section.
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
graph := process.AwaitBucket(ctx, js, conn, delivery.GraphBucket, "no module graph is written", logger.Printf)
ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node)}
OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}}
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
var started atomic.Bool
@@ -67,6 +67,8 @@ type Indexer struct {
// Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line.
Packages Store
// Graph is the module graph's state (delivery.GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph.
Graph Store
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
// with its git object id (gitea.gitea_commit_trees with files): the listing a package's digest is computed from.
// Nil publishes no package.
@@ -0,0 +1,131 @@
package index
import (
"context"
"fmt"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The module graph (novox/hq ADR 0603 §2): at every recompute the indexer writes, in its state graph
// (delivery.GraphBucket), one key per module it holds the inputs of, its build and provision edges as that recompute
// computed them, and deletes the key of a module the modules state no longer holds. A key is written only when its value
// changes, so its history of 5 is five changes of the module's edges. The delivery view, the checker and the seat
// mesh-catalog's verbs read it; nothing stores an order (ADR 0603 §1).
// GraphMost is the most modules the graph holds (ADR 0603 §2): past it, the modules after the first GraphMost in name
// order are not written, and it is said in the log. The cap is the indexer's code, not a manifest field, until a judge
// that knows such a field runs (the lead's decision of L3).
const GraphMost = 2000
// graphEdges is what a recompute knows of one module for its graph node.
type graphEdges struct {
module string
cur delivery.Module
manifest catalogue.Manifest
bases []string
fp, none string
tcs map[string]delivery.ToolchainUsed
pkgs delivery.PackageVersions
}
// graphNode is the graph node of e: its edges, the awaited provisions judged against providers (catalogue.Awaited, as
// the assigner judges them).
func graphNode(e graphEdges, providers func(string) []catalogue.Manifest) delivery.GraphNode {
if e.fp == "" && e.none == "" {
e.none = "its fingerprint was not computed"
}
n := delivery.GraphNode{FP: e.fp, None: e.none, Repository: e.cur.Repository, Path: e.cur.Path, Bases: e.bases,
Packages: slices.Sorted(maps.Keys(e.pkgs))}
tcs := e.tcs
if len(tcs) == 0 {
tcs = e.cur.Toolchains // no fingerprint now: the toolchain it was last computed with still builds it
}
for language, t := range tcs {
builder, known := ToolchainModules[language]
if !known {
continue
}
if n.Toolchains == nil {
n.Toolchains = map[string]delivery.GraphToolchain{}
}
n.Toolchains[language] = delivery.GraphToolchain{Module: builder, Version: t.Version, Recipe: t.Recipe}
}
for _, o := range e.manifest.Provides {
n.Provides = append(n.Provides, o.Name)
}
for _, r := range e.manifest.Requires {
if !e.manifest.OptionalRequires[r] {
n.Requires = append(n.Requires, r)
}
}
n.Awaits = catalogue.Awaited(e.manifest, providers)
return n
}
// writeGraph writes the graph node of every module of edges whose value changed, at most GraphMost of them, then
// deletes each graph key whose module the modules state no longer holds. Nil Graph writes nothing.
func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges) error {
if ix.Graph == nil {
return nil
}
names := slices.Sorted(maps.Keys(edges))
if len(names) > GraphMost {
ix.logf("the module graph holds at most %d modules: %d are known, so %s and the %d after it are not written",
GraphMost, len(names), names[GraphMost], len(names)-GraphMost-1)
names = names[:GraphMost]
}
providers := func(provision string) []catalogue.Manifest {
var out []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(edges)) {
if slices.Contains(edges[name].manifest.Offers(), provision) {
out = append(out, edges[name].manifest)
}
}
return out
}
for _, name := range names {
key, err := delivery.GraphKey(name)
if err != nil {
ix.logf("graph.%s is not written: %v", name, err)
continue
}
next := graphNode(edges[name], providers)
err = write(ctx, ix.Graph, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := delivery.DecodeGraphNode(value); err == nil && delivery.SameGraphNode(cur, next) {
return nil, false, nil
}
}
data, err := delivery.EncodeGraphNode(next)
if err != nil {
return nil, false, fmt.Errorf("graph.%s: %w", key, err)
}
return data, false, nil
})
if err != nil {
ix.logf("graph.%s is not written: %v", key, err)
}
}
held, err := ix.Modules.Keys(ctx)
if err != nil {
return fmt.Errorf("listing the modules for the graph: %w", err)
}
keys, err := ix.Graph.Keys(ctx)
if err != nil {
return fmt.Errorf("listing the graph: %w", err)
}
for _, key := range keys {
if slices.Contains(held, key) {
continue
}
if err := write(ctx, ix.Graph, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
return fmt.Errorf("graph.%s: %w", key, err)
}
ix.logf("graph.%s deleted: the modules state holds no module %s", key, key)
}
return nil
}
@@ -33,7 +33,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt
t.Helper()
ix, mods := modulesIndexer(t, recs...)
manifests := newMem()
ix.Manifests = manifests
ix.Manifests, ix.Graph = manifests, newMem()
return ix, mods, manifests
}
@@ -111,6 +111,7 @@ type toolchainOrNone struct {
type known struct {
key string
cur delivery.Module
manifest catalogue.Manifest
languages map[string]string
bases []string
contexts bool
@@ -196,7 +197,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
cur, stale[key] = again, true
held[key] = cur
}
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
all[key] = &known{key: key, cur: cur, manifest: m, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
}
tools := ix.agreed()
published := ix.publishedAll(ctx)
@@ -278,9 +279,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
return r
}
finals := map[string]delivery.Module{}
edges := map[string]graphEdges{} // the module graph's nodes of this recompute (graph.go)
for _, key := range slices.Sorted(maps.Keys(all)) {
k := all[key]
r := of(key, nil)
edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, bases: k.bases, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs}
if _, merged := overlay[key]; merged {
final := overlay[key]
final.FP2, final.None2, final.Inputs2, final.Toolchains, final.Packages = r.fp, r.none, k.cur.Inputs2, r.tcs, r.pkgs
@@ -332,7 +335,10 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
w.wrote(key)
}
}
return ix.writeOverlay(ctx, overlay, finals, w)
if err := ix.writeOverlay(ctx, overlay, finals, w); err != nil {
return err
}
return ix.writeGraph(ctx, edges)
}
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
@@ -60,6 +60,20 @@ type mod struct {
// merged applies one merge of the catalogue carrying these modules (every one a Go bundle's go.sum, a TypeScript bundle's
// package-lock.json) and returns the indexer.
// graphOf is the graph node the indexer holds of module.
func graphOf(t *testing.T, ix *Indexer, module string) delivery.GraphNode {
t.Helper()
v, ok := ix.Graph.(*memStore).vals[module]
if !ok {
t.Fatalf("graph.%s is not held", module)
}
g, err := delivery.DecodeGraphNode(v)
if err != nil {
t.Fatal(err)
}
return g
}
func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) {
t.Helper()
ix, store, f := newFormIndexerUnmerged(t, mods...)
@@ -122,9 +136,24 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
if m, _ := moduleOf(t, mods, "avahi"); m.Src != 7 || m.FPSrc != 7 {
t.Fatalf("before: src %d fp-src %d, want 7 and 7", m.Src, m.FPSrc)
}
if g := graphOf(t, ix, "avahi"); g.None != delivery.ToolchainNoReport || g.Toolchains != nil {
t.Errorf("before: graph.avahi = %+v, want no toolchain edge and the reason", g)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
m, _ := moduleOf(t, mods, "avahi")
// L3 (ADR 0603 §2, §3): the agreement moving rewrites the module's toolchain edge, and again when it moves on.
if g := graphOf(t, ix, "avahi"); g.FP != m.FP2 || g.Toolchains["go"].Version != "go1.27.1" || g.Toolchains["go"].Module != "mesh-build-go" {
t.Errorf("after: graph.avahi = %+v, want the toolchain edge of go1.27.1", g)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.28.0")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.28.0")
if g := graphOf(t, ix, "avahi"); g.Toolchains["go"].Version != "go1.28.0" {
t.Errorf("moved on: graph.avahi = %+v, want the toolchain edge of go1.28.0", g)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
m, _ = moduleOf(t, mods, "avahi")
if m.None2 != "" || m.FP2 == "" {
t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.FP2)
}
@@ -202,6 +231,31 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2)
}
// L3 (ADR 0603 §2): the module graph holds each module's edges as this recompute computed them.
goTool := map[string]delivery.GraphToolchain{"go": {Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}}
if g := graphOf(t, ix, "avahi"); g.FP != want["avahi"] || !reflect.DeepEqual(g.Bases, []string{"mesh-tools"}) ||
!reflect.DeepEqual(g.Toolchains, goTool) || g.Repository != catalogueRepo || g.Path != "modules/avahi" {
t.Errorf("graph.avahi = %+v", g)
}
if g := graphOf(t, ix, "other"); g.FP != "" || !strings.Contains(g.None, "ts-base") || !reflect.DeepEqual(g.Bases, []string{"ts-base"}) {
t.Errorf("graph.other = %+v, want its base and its reason", g)
}
graph := ix.Graph.(*memStore)
written := graph.writes
if err := ix.Recompute(context.Background()); err != nil || graph.writes != written {
t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err)
}
// A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute.
delete(mods.vals, "other")
if err := ix.Recompute(context.Background()); err != nil {
t.Fatal(err)
}
if _, held := graph.vals["other"]; held {
t.Error("graph.other is still held after its module went")
}
if _, held := graph.vals["avahi"]; !held {
t.Error("graph.avahi went with another module")
}
a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z")
_ = ix.Builds.Put(a[1].([]byte))
if err := ix.Recompute(context.Background()); err != nil {
@@ -90,7 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits)
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory},
{Name: delivery.GraphState, History: delivery.GraphHistory}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
+2 -1
View File
@@ -25,7 +25,8 @@
{ "name": "takeins", "history": 1 },
{ "name": "verdicts", "history": 5 },
{ "name": "oldsections", "history": 1 },
{ "name": "packages", "history": 5 }
{ "name": "packages", "history": 5 },
{ "name": "graph", "history": 5 }
],
"emits": [
"manifest-refused",