The indexer keeps the module graph as its state graph: each module's build and provision edges written at every recompute when they change, at most 2000 modules, a module's key deleted once the modules state no longer holds it (ADR 0603 §2)
This commit is contained in:
@@ -299,12 +299,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
|
||||
// the judge refuses no use of an old section.
|
||||
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
|
||||
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
|
||||
graph := process.AwaitBucket(ctx, js, conn, delivery.GraphBucket, "no module graph is written", logger.Printf)
|
||||
ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
|
||||
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
|
||||
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node)}
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}}
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
var started atomic.Bool
|
||||
|
||||
@@ -67,6 +67,8 @@ type Indexer struct {
|
||||
// Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
|
||||
// and every fingerprint takes the version line.
|
||||
Packages Store
|
||||
// Graph is the module graph's state (delivery.GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph.
|
||||
Graph Store
|
||||
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
|
||||
// with its git object id (gitea.gitea_commit_trees with files): the listing a package's digest is computed from.
|
||||
// Nil publishes no package.
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The module graph (novox/hq ADR 0603 §2): at every recompute the indexer writes, in its state graph
|
||||
// (delivery.GraphBucket), one key per module it holds the inputs of, its build and provision edges as that recompute
|
||||
// computed them, and deletes the key of a module the modules state no longer holds. A key is written only when its value
|
||||
// changes, so its history of 5 is five changes of the module's edges. The delivery view, the checker and the seat
|
||||
// mesh-catalog's verbs read it; nothing stores an order (ADR 0603 §1).
|
||||
|
||||
// GraphMost is the most modules the graph holds (ADR 0603 §2): past it, the modules after the first GraphMost in name
|
||||
// order are not written, and it is said in the log. The cap is the indexer's code, not a manifest field, until a judge
|
||||
// that knows such a field runs (the lead's decision of L3).
|
||||
const GraphMost = 2000
|
||||
|
||||
// graphEdges is what a recompute knows of one module for its graph node.
|
||||
type graphEdges struct {
|
||||
module string
|
||||
cur delivery.Module
|
||||
manifest catalogue.Manifest
|
||||
bases []string
|
||||
fp, none string
|
||||
tcs map[string]delivery.ToolchainUsed
|
||||
pkgs delivery.PackageVersions
|
||||
}
|
||||
|
||||
// graphNode is the graph node of e: its edges, the awaited provisions judged against providers (catalogue.Awaited, as
|
||||
// the assigner judges them).
|
||||
func graphNode(e graphEdges, providers func(string) []catalogue.Manifest) delivery.GraphNode {
|
||||
if e.fp == "" && e.none == "" {
|
||||
e.none = "its fingerprint was not computed"
|
||||
}
|
||||
n := delivery.GraphNode{FP: e.fp, None: e.none, Repository: e.cur.Repository, Path: e.cur.Path, Bases: e.bases,
|
||||
Packages: slices.Sorted(maps.Keys(e.pkgs))}
|
||||
tcs := e.tcs
|
||||
if len(tcs) == 0 {
|
||||
tcs = e.cur.Toolchains // no fingerprint now: the toolchain it was last computed with still builds it
|
||||
}
|
||||
for language, t := range tcs {
|
||||
builder, known := ToolchainModules[language]
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
if n.Toolchains == nil {
|
||||
n.Toolchains = map[string]delivery.GraphToolchain{}
|
||||
}
|
||||
n.Toolchains[language] = delivery.GraphToolchain{Module: builder, Version: t.Version, Recipe: t.Recipe}
|
||||
}
|
||||
for _, o := range e.manifest.Provides {
|
||||
n.Provides = append(n.Provides, o.Name)
|
||||
}
|
||||
for _, r := range e.manifest.Requires {
|
||||
if !e.manifest.OptionalRequires[r] {
|
||||
n.Requires = append(n.Requires, r)
|
||||
}
|
||||
}
|
||||
n.Awaits = catalogue.Awaited(e.manifest, providers)
|
||||
return n
|
||||
}
|
||||
|
||||
// writeGraph writes the graph node of every module of edges whose value changed, at most GraphMost of them, then
|
||||
// deletes each graph key whose module the modules state no longer holds. Nil Graph writes nothing.
|
||||
func (ix *Indexer) writeGraph(ctx context.Context, edges map[string]graphEdges) error {
|
||||
if ix.Graph == nil {
|
||||
return nil
|
||||
}
|
||||
names := slices.Sorted(maps.Keys(edges))
|
||||
if len(names) > GraphMost {
|
||||
ix.logf("the module graph holds at most %d modules: %d are known, so %s and the %d after it are not written",
|
||||
GraphMost, len(names), names[GraphMost], len(names)-GraphMost-1)
|
||||
names = names[:GraphMost]
|
||||
}
|
||||
providers := func(provision string) []catalogue.Manifest {
|
||||
var out []catalogue.Manifest
|
||||
for _, name := range slices.Sorted(maps.Keys(edges)) {
|
||||
if slices.Contains(edges[name].manifest.Offers(), provision) {
|
||||
out = append(out, edges[name].manifest)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
for _, name := range names {
|
||||
key, err := delivery.GraphKey(name)
|
||||
if err != nil {
|
||||
ix.logf("graph.%s is not written: %v", name, err)
|
||||
continue
|
||||
}
|
||||
next := graphNode(edges[name], providers)
|
||||
err = write(ctx, ix.Graph, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeGraphNode(value); err == nil && delivery.SameGraphNode(cur, next) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
data, err := delivery.EncodeGraphNode(next)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("graph.%s: %w", key, err)
|
||||
}
|
||||
return data, false, nil
|
||||
})
|
||||
if err != nil {
|
||||
ix.logf("graph.%s is not written: %v", key, err)
|
||||
}
|
||||
}
|
||||
held, err := ix.Modules.Keys(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing the modules for the graph: %w", err)
|
||||
}
|
||||
keys, err := ix.Graph.Keys(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing the graph: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if slices.Contains(held, key) {
|
||||
continue
|
||||
}
|
||||
if err := write(ctx, ix.Graph, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
|
||||
return fmt.Errorf("graph.%s: %w", key, err)
|
||||
}
|
||||
ix.logf("graph.%s deleted: the modules state holds no module %s", key, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -33,7 +33,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt
|
||||
t.Helper()
|
||||
ix, mods := modulesIndexer(t, recs...)
|
||||
manifests := newMem()
|
||||
ix.Manifests = manifests
|
||||
ix.Manifests, ix.Graph = manifests, newMem()
|
||||
return ix, mods, manifests
|
||||
}
|
||||
|
||||
|
||||
@@ -111,6 +111,7 @@ type toolchainOrNone struct {
|
||||
type known struct {
|
||||
key string
|
||||
cur delivery.Module
|
||||
manifest catalogue.Manifest
|
||||
languages map[string]string
|
||||
bases []string
|
||||
contexts bool
|
||||
@@ -196,7 +197,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
cur, stale[key] = again, true
|
||||
held[key] = cur
|
||||
}
|
||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
|
||||
all[key] = &known{key: key, cur: cur, manifest: m, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
|
||||
}
|
||||
tools := ix.agreed()
|
||||
published := ix.publishedAll(ctx)
|
||||
@@ -278,9 +279,11 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
return r
|
||||
}
|
||||
finals := map[string]delivery.Module{}
|
||||
edges := map[string]graphEdges{} // the module graph's nodes of this recompute (graph.go)
|
||||
for _, key := range slices.Sorted(maps.Keys(all)) {
|
||||
k := all[key]
|
||||
r := of(key, nil)
|
||||
edges[key] = graphEdges{module: key, cur: k.cur, manifest: k.manifest, bases: k.bases, fp: r.fp, none: r.none, tcs: r.tcs, pkgs: r.pkgs}
|
||||
if _, merged := overlay[key]; merged {
|
||||
final := overlay[key]
|
||||
final.FP2, final.None2, final.Inputs2, final.Toolchains, final.Packages = r.fp, r.none, k.cur.Inputs2, r.tcs, r.pkgs
|
||||
@@ -332,7 +335,10 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
w.wrote(key)
|
||||
}
|
||||
}
|
||||
return ix.writeOverlay(ctx, overlay, finals, w)
|
||||
if err := ix.writeOverlay(ctx, overlay, finals, w); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.writeGraph(ctx, edges)
|
||||
}
|
||||
|
||||
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
||||
|
||||
@@ -60,6 +60,20 @@ type mod struct {
|
||||
|
||||
// merged applies one merge of the catalogue carrying these modules (every one a Go bundle's go.sum, a TypeScript bundle's
|
||||
// package-lock.json) and returns the indexer.
|
||||
// graphOf is the graph node the indexer holds of module.
|
||||
func graphOf(t *testing.T, ix *Indexer, module string) delivery.GraphNode {
|
||||
t.Helper()
|
||||
v, ok := ix.Graph.(*memStore).vals[module]
|
||||
if !ok {
|
||||
t.Fatalf("graph.%s is not held", module)
|
||||
}
|
||||
g, err := delivery.DecodeGraphNode(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) {
|
||||
t.Helper()
|
||||
ix, store, f := newFormIndexerUnmerged(t, mods...)
|
||||
@@ -122,9 +136,24 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Src != 7 || m.FPSrc != 7 {
|
||||
t.Fatalf("before: src %d fp-src %d, want 7 and 7", m.Src, m.FPSrc)
|
||||
}
|
||||
if g := graphOf(t, ix, "avahi"); g.None != delivery.ToolchainNoReport || g.Toolchains != nil {
|
||||
t.Errorf("before: graph.avahi = %+v, want no toolchain edge and the reason", g)
|
||||
}
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
// L3 (ADR 0603 §2, §3): the agreement moving rewrites the module's toolchain edge, and again when it moves on.
|
||||
if g := graphOf(t, ix, "avahi"); g.FP != m.FP2 || g.Toolchains["go"].Version != "go1.27.1" || g.Toolchains["go"].Module != "mesh-build-go" {
|
||||
t.Errorf("after: graph.avahi = %+v, want the toolchain edge of go1.27.1", g)
|
||||
}
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.28.0")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.28.0")
|
||||
if g := graphOf(t, ix, "avahi"); g.Toolchains["go"].Version != "go1.28.0" {
|
||||
t.Errorf("moved on: graph.avahi = %+v, want the toolchain edge of go1.28.0", g)
|
||||
}
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
|
||||
m, _ = moduleOf(t, mods, "avahi")
|
||||
if m.None2 != "" || m.FP2 == "" {
|
||||
t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.FP2)
|
||||
}
|
||||
@@ -202,6 +231,31 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
|
||||
t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2)
|
||||
}
|
||||
// L3 (ADR 0603 §2): the module graph holds each module's edges as this recompute computed them.
|
||||
goTool := map[string]delivery.GraphToolchain{"go": {Module: "mesh-build-go", Version: "go1.27.1", Recipe: "1"}}
|
||||
if g := graphOf(t, ix, "avahi"); g.FP != want["avahi"] || !reflect.DeepEqual(g.Bases, []string{"mesh-tools"}) ||
|
||||
!reflect.DeepEqual(g.Toolchains, goTool) || g.Repository != catalogueRepo || g.Path != "modules/avahi" {
|
||||
t.Errorf("graph.avahi = %+v", g)
|
||||
}
|
||||
if g := graphOf(t, ix, "other"); g.FP != "" || !strings.Contains(g.None, "ts-base") || !reflect.DeepEqual(g.Bases, []string{"ts-base"}) {
|
||||
t.Errorf("graph.other = %+v, want its base and its reason", g)
|
||||
}
|
||||
graph := ix.Graph.(*memStore)
|
||||
written := graph.writes
|
||||
if err := ix.Recompute(context.Background()); err != nil || graph.writes != written {
|
||||
t.Errorf("a recompute that moves nothing wrote the graph %d times (%v)", graph.writes-written, err)
|
||||
}
|
||||
// A module the modules state no longer holds (a merge dropped it, issue 1900) leaves the graph at the next recompute.
|
||||
delete(mods.vals, "other")
|
||||
if err := ix.Recompute(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, held := graph.vals["other"]; held {
|
||||
t.Error("graph.other is still held after its module went")
|
||||
}
|
||||
if _, held := graph.vals["avahi"]; !held {
|
||||
t.Error("graph.avahi went with another module")
|
||||
}
|
||||
a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z")
|
||||
_ = ix.Builds.Put(a[1].([]byte))
|
||||
if err := ix.Recompute(context.Background()); err != nil {
|
||||
|
||||
@@ -90,7 +90,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits)
|
||||
}
|
||||
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory},
|
||||
{Name: delivery.GraphState, History: delivery.GraphHistory}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
|
||||
@@ -25,7 +25,8 @@
|
||||
{ "name": "takeins", "history": 1 },
|
||||
{ "name": "verdicts", "history": 5 },
|
||||
{ "name": "oldsections", "history": 1 },
|
||||
{ "name": "packages", "history": 5 }
|
||||
{ "name": "packages", "history": 5 },
|
||||
{ "name": "graph", "history": 5 }
|
||||
],
|
||||
"emits": [
|
||||
"manifest-refused",
|
||||
|
||||
Reference in New Issue
Block a user