From dd2e7c8e64fef53ae2b95e2427f32ff54555c5e2 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 3 Nov 2026 13:34:54 +0100 Subject: [PATCH] The indexer counts the modules on each old section, a tool reads it, and the judge refuses a section at 0 outside the foundation (issue 1966 D4) --- .../cmd/mesh-delivery-indexer-tools/main.go | 31 +++++++- .../cmd/mesh-delivery-indexer/main.go | 5 +- .../internal/index/candidates.go | 3 + .../internal/index/oldsections.go | 78 +++++++++++++++++++ .../internal/index/verdicts.go | 17 +++- .../internal/index/verdicts_test.go | 33 ++++++++ modules/mesh-delivery-indexer/judge/judge.go | 10 +++ .../judge/oldsections.go | 69 ++++++++++++++++ .../mesh-delivery-indexer/manifest_test.go | 7 +- modules/mesh-delivery-indexer/module.json | 4 +- 10 files changed, 249 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/oldsections.go create mode 100644 modules/mesh-delivery-indexer/judge/oldsections.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index ff97616b..5529d02f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -22,9 +22,22 @@ import ( type asker func(key string, value []byte) (index.TakeInAsk, error) // Tools are the indexer's tools. -func Tools(ask asker, now func() time.Time) []stdio.Tool { +func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCount, error)) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ + {Name: index.OldSectionsTool, + Description: "How many accepted modules use each old section of a manifest (state, logs, streams, objects), the foundation " + + "(nats, mesh-grants, the indexer) left out: the count the judge refuses a use of a section at 0 by, as modules move " + + "their bus state into `requires` by template. Read from the indexer's oldsections state, kept as the indexer judges " + + "the manifests; an empty answer means the indexer has not judged since it started.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + c, err := counts() + if err != nil { + return nil, err + } + return c.Counts, nil + }}, {Name: index.TakeInTool, Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " + "build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " + @@ -105,8 +118,22 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker { } } +// heldCounts reads the count of the old sections the indexer keeps. +func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) { + return func() (index.OldSectionsCount, error) { + e, err := s.Get(index.OldSectionsKey) + if err != nil { + return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsBucket, index.OldSectionsKey, err) + } + if e == nil { + return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsBucket, index.OldSectionsKey) + } + return index.DecodeOldSections(e.Value) + } +} + func main() { - if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil { + if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, heldCounts(stdio.State(index.OldSectionsState)))); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 04bb7252..ee6e8f37 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -296,7 +296,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every // fingerprint takes the version line. packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf) - ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, + // How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then + // the judge refuses no use of an old section. + oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf) + ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 95b4dde3..09f613a6 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -54,6 +54,9 @@ type Indexer struct { Taken Store // Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing. Verdicts Store + // OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no + // count and the judge refuses no use of an old section. + OldSections Store // JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's. JudgeID string // SayRefused says the event manifest-refused for a refusal newly written; nil says nothing. diff --git a/modules/mesh-delivery-indexer/internal/index/oldsections.go b/modules/mesh-delivery-indexer/internal/index/oldsections.go new file mode 100644 index 00000000..14a6db51 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/oldsections.go @@ -0,0 +1,78 @@ +package index + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "slices" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/judge" +) + +// How many accepted modules use each old section (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts them, the +// foundation left out, as it judges, keeps the count in its state oldsections under one key, and the tool +// indexer_old_sections reads it. The judge refuses a manifest of another module that uses a section whose count is 0. +const ( + // OldSectionsTool is the tool that reads the count. + OldSectionsTool = "indexer_old_sections" + // OldSectionsState is the indexer's state of the count: the bucket mesh-delivery-indexer_oldsections. + OldSectionsState = "oldsections" + OldSectionsBucket = delivery.IndexerModule + "_" + OldSectionsState + // OldSectionsKey is the one key it holds. + OldSectionsKey = "counts" +) + +// OldSectionsCount is the value of the oldsections state: per old section, the accepted modules outside the foundation using it. +type OldSectionsCount struct { + V int `json:"v"` + Counts map[string]int `json:"counts"` +} + +// DecodeOldSections reads the oldsections value. +func DecodeOldSections(value []byte) (OldSectionsCount, error) { + var c OldSectionsCount + if err := json.Unmarshal(value, &c); err != nil { + return c, fmt.Errorf("the count of the old sections cannot be read: %w", err) + } + if c.V != 1 { + return c, fmt.Errorf("a count of the old sections of version %d, not 1", c.V) + } + return c, nil +} + +// countBefore counts the accepted modules using each old section as the pass starts: those whose verdict held says accepted, +// or which have none and are not fresh (acceptedBefore). A manifest is read leniently, since a module the judge now finds +// a problem in is still accepted (ADR 0579 §1) and still counts. +func (ix *Indexer) countBefore(ctx context.Context, raws map[string][]byte, digests map[string]string, fresh map[string]bool) judge.OldSectionCounts { + var accepted []catalogue.Manifest + for _, name := range slices.Sorted(maps.Keys(raws)) { + if !ix.acceptedBefore(ctx, digests[name], fresh) { + continue + } + if m, err := catalogue.Decode(raws[name]); err == nil { + m.Module = name + accepted = append(accepted, m) + } + } + return judge.CountOldSections(accepted) +} + +// keepOldSections writes the counts under OldSectionsKey when they differ from the ones held. +func (ix *Indexer) keepOldSections(ctx context.Context, counts judge.OldSectionCounts) error { + if ix.OldSections == nil { + return nil + } + return write(ctx, ix.OldSections, OldSectionsKey, func(value []byte, found bool) ([]byte, bool, error) { + if found { + if cur, err := DecodeOldSections(value); err == nil && maps.Equal(cur.Counts, counts) { + return nil, false, nil + } + } + next, err := json.Marshal(OldSectionsCount{V: 1, Counts: counts}) + return next, false, err + }) +} diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 383dd65e..03138bd2 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -75,7 +75,12 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, digests[name] = digest } predefined := ix.predefinedList(ctx) - refused, unnamed := judge.Shelf(raws, predefined, 0) + // The old sections end when no module is left in them (issue 1966 D4): counted from the verdicts held before this pass. + var counts judge.OldSectionCounts + if ix.OldSections != nil { + counts = ix.countBefore(ctx, raws, digests, fresh) + } + refused, unnamed := judge.ShelfCounting(raws, predefined, 0, counts) for _, p := range unnamed { ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p) } @@ -84,6 +89,7 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, if id == "" { id = judge.Identity() } + var accepted []catalogue.Manifest for _, name := range slices.Sorted(maps.Keys(digests)) { out.Judged++ v := delivery.ManifestVerdict{Module: name, State: delivery.VerdictAccepted, Judge: id, At: ix.now()} @@ -95,6 +101,12 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, v.State, v.Problems = delivery.VerdictRefused, problems } } + if !v.Refused() { + if m, err := catalogue.Decode(raws[name]); err == nil { + m.Module = name + accepted = append(accepted, m) + } + } wrote, err := ix.writeVerdict(ctx, digests[name], v) if errors.Is(err, process.ErrNotYet) { return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsBucket, err) @@ -121,6 +133,9 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, ix.logf("judge: modules.%s does not carry its verdict: %v", name, err) } } + if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted)); err != nil { + ix.logf("judge: the count of the old sections is not kept: %v", err) + } return out, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go index d90b852d..380ca073 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go @@ -135,6 +135,39 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { string(accepted[0].BusState[0].Options["history"]) != "8" { t.Fatalf("said accepted %+v", accepted) } + // 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another + // module using logs is refused and the foundation's is accepted. + ix.OldSections = newMem() + hold("user", `{"module":"user","version":"1","state":["x"]}`) + hold("mesh-grants", `{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`) + if _, err := ix.JudgeAll(ctx, nil); err != nil { + t.Fatal(err) + } + counts := func() map[string]int { + value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey) + if err != nil || !found { + t.Fatalf("no count of the old sections: %v", err) + } + c, err := DecodeOldSections(value) + if err != nil { + t.Fatal(err) + } + return c.Counts + } + if c := counts(); c["state"] != 1 || c["logs"] != 0 || c["streams"] != 0 || c["objects"] != 0 { + t.Fatalf("the old sections count %v, want state 1 and the rest 0", c) + } + late := hold("late", `{"module":"late","version":"1","logs":[{"name":"l"}]}`) + grants := hold("mesh-grants", `{"module":"mesh-grants","version":"2","logs":[{"name":"l"},{"name":"m"}]}`) + if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil { + t.Fatal(err) + } + if v := verdict(late); !v.Refused() || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") { + t.Fatalf("late: %+v", v) + } + if v := verdict(grants); v.Refused() { + t.Fatalf("the foundation's manifest was refused: %+v", v) + } // Without the verdicts state the indexer judges nothing. ix.Verdicts = nil if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 { diff --git a/modules/mesh-delivery-indexer/judge/judge.go b/modules/mesh-delivery-indexer/judge/judge.go index 3add6332..9ebb2101 100644 --- a/modules/mesh-delivery-indexer/judge/judge.go +++ b/modules/mesh-delivery-indexer/judge/judge.go @@ -69,6 +69,12 @@ func problemsOf(err error) []string { // whose removal from the catalogue ends it (a rule two modules break together refuses both); one no removal ends is in // unnamed, for the caller to say. A module not in the answer is accepted. func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int) (refused map[string][]string, unnamed []string) { + return ShelfCounting(raws, predefined, longestMachine, nil) +} + +// ShelfCounting is Shelf also refusing a manifest of a module outside the foundation that uses an old section whose count +// is 0 (OldSectionProblems; issue 1966 D4); nil counts refuse none. +func ShelfCounting(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int, counts OldSectionCounts) (refused map[string][]string, unnamed []string) { refused = map[string][]string{} shelf := catalogue.Shelf{} for name, raw := range raws { @@ -81,6 +87,10 @@ func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestM refused[name] = []string{fmt.Sprintf("the manifest is of module %q, not %q", m.Module, name)} continue } + if problems := OldSectionProblems(m, counts); len(problems) > 0 { + refused[name] = problems + continue + } shelf[name] = m } between := Between(shelf, longestMachine) diff --git a/modules/mesh-delivery-indexer/judge/oldsections.go b/modules/mesh-delivery-indexer/judge/oldsections.go new file mode 100644 index 00000000..47181176 --- /dev/null +++ b/modules/mesh-delivery-indexer/judge/oldsections.go @@ -0,0 +1,69 @@ +package judge + +import ( + "fmt" + "slices" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The old sections end when no module is left in them (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts, per old +// section, the accepted modules using it, the foundation left out, and the judge refuses a manifest of another module +// that uses a section whose count is 0. The foundation (the bus module, the grants part and the indexer itself) is +// created before the bus module can create anything, so it keeps the old sections for good and is always accepted. + +// OldSections are the sections of a manifest that keep bus state in the old forms. +var OldSections = []string{"state", "logs", "streams", "objects"} + +// Foundation are the modules that keep using the old sections for good. +var Foundation = []string{"nats", "mesh-grants", "mesh-delivery-indexer"} + +// OldSectionCounts is, per old section, how many accepted modules outside the foundation use it. Nil is no count: the +// judge then refuses no use of a section. +type OldSectionCounts map[string]int + +// SectionsUsed are the old sections a manifest uses, in OldSections' order. +func SectionsUsed(m catalogue.Manifest) []string { + var used []string + for _, s := range OldSections { + if (s == "state" && len(m.State) > 0) || (s == "logs" && len(m.Logs) > 0) || + (s == "streams" && len(m.Streams) > 0) || (s == "objects" && len(m.Objects) > 0) { + used = append(used, s) + } + } + return used +} + +// CountOldSections counts, per old section, the manifests using it, the foundation's left out; every section is in the +// answer, with 0 where none uses it. +func CountOldSections(manifests []catalogue.Manifest) OldSectionCounts { + counts := OldSectionCounts{} + for _, s := range OldSections { + counts[s] = 0 + } + for _, m := range manifests { + if slices.Contains(Foundation, m.Module) { + continue + } + for _, s := range SectionsUsed(m) { + counts[s]++ + } + } + return counts +} + +// OldSectionProblems are the old sections a manifest uses that no accepted module outside the foundation uses any more. +func OldSectionProblems(m catalogue.Manifest, counts OldSectionCounts) []string { + if counts == nil || slices.Contains(Foundation, m.Module) { + return nil + } + var problems []string + for _, s := range SectionsUsed(m) { + if counts[s] == 0 { + problems = append(problems, fmt.Sprintf( + "%s uses the old section %s, which no other module uses any more (0 modules count); ask for it in "+ + "`requires` by template (novox/hq ADR 0595 §11)", m.Module, s)) + } + } + return problems +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 2a3545da..c6bbbf90 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -90,7 +90,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { @@ -113,8 +113,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } - // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. - if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) { + // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11: + // indexer_old_sections reads the count of the old sections (state oldsections). + if want := []string{"indexer_take_in", "indexer_old_sections"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // No streams, no bus, no seat (ADR 0437 §4); it says no event. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 350693cf..313afc81 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -3,7 +3,8 @@ "version": "1", "slug": "indexer", "tools": [ - "indexer_take_in" + "indexer_take_in", + "indexer_old_sections" ], "consumes": [ "mesh-build-go.toolchain.reported", @@ -21,6 +22,7 @@ { "name": "taken", "history": 1 }, { "name": "takeins", "history": 1 }, { "name": "verdicts", "history": 5 }, + { "name": "oldsections", "history": 1 }, { "name": "packages", "history": 5 } ], "emits": [