The indexer counts the modules on each old section, a tool reads it, and the judge refuses a section at 0 outside the foundation (issue 1966 D4)

modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go

@@ -22,9 +22,22 @@ import (
 type asker func(key string, value []byte) (index.TakeInAsk, error)
 
 // Tools are the indexer's tools.
-func Tools(ask asker, now func() time.Time) []stdio.Tool {
+func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCount, error)) []stdio.Tool {
 	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
 	return []stdio.Tool{
+		{Name: index.OldSectionsTool,
+			Description: "How many accepted modules use each old section of a manifest (state, logs, streams, objects), the foundation " +
+				"(nats, mesh-grants, the indexer) left out: the count the judge refuses a use of a section at 0 by, as modules move " +
+				"their bus state into `requires` by template. Read from the indexer's oldsections state, kept as the indexer judges " +
+				"the manifests; an empty answer means the indexer has not judged since it started.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{}},
+			Run: func(map[string]any) (any, error) {
+				c, err := counts()
+				if err != nil {
+					return nil, err
+				}
+				return c.Counts, nil
+			}},
 		{Name: index.TakeInTool,
 			Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
 				"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
@@ -105,8 +118,22 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
 	}
 }
 
+// heldCounts reads the count of the old sections the indexer keeps.
+func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
+	return func() (index.OldSectionsCount, error) {
+		e, err := s.Get(index.OldSectionsKey)
+		if err != nil {
+			return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsBucket, index.OldSectionsKey, err)
+		}
+		if e == nil {
+			return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsBucket, index.OldSectionsKey)
+		}
+		return index.DecodeOldSections(e.Value)
+	}
+}
+
 func main() {
-	if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
+	if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, heldCounts(stdio.State(index.OldSectionsState)))); err != nil {
 		fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
 		os.Exit(1)
 	}

modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go

@@ -296,7 +296,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
 	// fingerprint takes the version line.
 	packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
-	ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
+	// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
+	// the judge refuses no use of an old section.
+	oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
+	ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
 		Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
 		Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
 		Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),

modules/mesh-delivery-indexer/internal/index/candidates.go

@@ -54,6 +54,9 @@ type Indexer struct {
 	Taken Store
 	// Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing.
 	Verdicts Store
+	// OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no
+	// count and the judge refuses no use of an old section.
+	OldSections Store
 	// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
 	JudgeID string
 	// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.

modules/mesh-delivery-indexer/internal/index/oldsections.go

@@ -0,0 +1,78 @@
+package index
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"slices"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"mesh-delivery-indexer/judge"
+)
+
+// How many accepted modules use each old section (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts them, the
+// foundation left out, as it judges, keeps the count in its state oldsections under one key, and the tool
+// indexer_old_sections reads it. The judge refuses a manifest of another module that uses a section whose count is 0.
+const (
+	// OldSectionsTool is the tool that reads the count.
+	OldSectionsTool = "indexer_old_sections"
+	// OldSectionsState is the indexer's state of the count: the bucket mesh-delivery-indexer_oldsections.
+	OldSectionsState  = "oldsections"
+	OldSectionsBucket = delivery.IndexerModule + "_" + OldSectionsState
+	// OldSectionsKey is the one key it holds.
+	OldSectionsKey = "counts"
+)
+
+// OldSectionsCount is the value of the oldsections state: per old section, the accepted modules outside the foundation using it.
+type OldSectionsCount struct {
+	V      int            `json:"v"`
+	Counts map[string]int `json:"counts"`
+}
+
+// DecodeOldSections reads the oldsections value.
+func DecodeOldSections(value []byte) (OldSectionsCount, error) {
+	var c OldSectionsCount
+	if err := json.Unmarshal(value, &c); err != nil {
+		return c, fmt.Errorf("the count of the old sections cannot be read: %w", err)
+	}
+	if c.V != 1 {
+		return c, fmt.Errorf("a count of the old sections of version %d, not 1", c.V)
+	}
+	return c, nil
+}
+
+// countBefore counts the accepted modules using each old section as the pass starts: those whose verdict held says accepted,
+// or which have none and are not fresh (acceptedBefore). A manifest is read leniently, since a module the judge now finds
+// a problem in is still accepted (ADR 0579 §1) and still counts.
+func (ix *Indexer) countBefore(ctx context.Context, raws map[string][]byte, digests map[string]string, fresh map[string]bool) judge.OldSectionCounts {
+	var accepted []catalogue.Manifest
+	for _, name := range slices.Sorted(maps.Keys(raws)) {
+		if !ix.acceptedBefore(ctx, digests[name], fresh) {
+			continue
+		}
+		if m, err := catalogue.Decode(raws[name]); err == nil {
+			m.Module = name
+			accepted = append(accepted, m)
+		}
+	}
+	return judge.CountOldSections(accepted)
+}
+
+// keepOldSections writes the counts under OldSectionsKey when they differ from the ones held.
+func (ix *Indexer) keepOldSections(ctx context.Context, counts judge.OldSectionCounts) error {
+	if ix.OldSections == nil {
+		return nil
+	}
+	return write(ctx, ix.OldSections, OldSectionsKey, func(value []byte, found bool) ([]byte, bool, error) {
+		if found {
+			if cur, err := DecodeOldSections(value); err == nil && maps.Equal(cur.Counts, counts) {
+				return nil, false, nil
+			}
+		}
+		next, err := json.Marshal(OldSectionsCount{V: 1, Counts: counts})
+		return next, false, err
+	})
+}

modules/mesh-delivery-indexer/internal/index/verdicts.go

@@ -75,7 +75,12 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
 		digests[name] = digest
 	}
 	predefined := ix.predefinedList(ctx)
-	refused, unnamed := judge.Shelf(raws, predefined, 0)
+	// The old sections end when no module is left in them (issue 1966 D4): counted from the verdicts held before this pass.
+	var counts judge.OldSectionCounts
+	if ix.OldSections != nil {
+		counts = ix.countBefore(ctx, raws, digests, fresh)
+	}
+	refused, unnamed := judge.ShelfCounting(raws, predefined, 0, counts)
 	for _, p := range unnamed {
 		ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p)
 	}
@@ -84,6 +89,7 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
 	if id == "" {
 		id = judge.Identity()
 	}
+	var accepted []catalogue.Manifest
 	for _, name := range slices.Sorted(maps.Keys(digests)) {
 		out.Judged++
 		v := delivery.ManifestVerdict{Module: name, State: delivery.VerdictAccepted, Judge: id, At: ix.now()}
@@ -95,6 +101,12 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
 				v.State, v.Problems = delivery.VerdictRefused, problems
 			}
 		}
+		if !v.Refused() {
+			if m, err := catalogue.Decode(raws[name]); err == nil {
+				m.Module = name
+				accepted = append(accepted, m)
+			}
+		}
 		wrote, err := ix.writeVerdict(ctx, digests[name], v)
 		if errors.Is(err, process.ErrNotYet) {
 			return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsBucket, err)
@@ -121,6 +133,9 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
 			ix.logf("judge: modules.%s does not carry its verdict: %v", name, err)
 		}
 	}
+	if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted)); err != nil {
+		ix.logf("judge: the count of the old sections is not kept: %v", err)
+	}
 	return out, nil
 }
 

modules/mesh-delivery-indexer/internal/index/verdicts_test.go

@@ -135,6 +135,39 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
 		string(accepted[0].BusState[0].Options["history"]) != "8" {
 		t.Fatalf("said accepted %+v", accepted)
 	}
+	// 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another
+	// module using logs is refused and the foundation's is accepted.
+	ix.OldSections = newMem()
+	hold("user", `{"module":"user","version":"1","state":["x"]}`)
+	hold("mesh-grants", `{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`)
+	if _, err := ix.JudgeAll(ctx, nil); err != nil {
+		t.Fatal(err)
+	}
+	counts := func() map[string]int {
+		value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey)
+		if err != nil || !found {
+			t.Fatalf("no count of the old sections: %v", err)
+		}
+		c, err := DecodeOldSections(value)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return c.Counts
+	}
+	if c := counts(); c["state"] != 1 || c["logs"] != 0 || c["streams"] != 0 || c["objects"] != 0 {
+		t.Fatalf("the old sections count %v, want state 1 and the rest 0", c)
+	}
+	late := hold("late", `{"module":"late","version":"1","logs":[{"name":"l"}]}`)
+	grants := hold("mesh-grants", `{"module":"mesh-grants","version":"2","logs":[{"name":"l"},{"name":"m"}]}`)
+	if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil {
+		t.Fatal(err)
+	}
+	if v := verdict(late); !v.Refused() || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") {
+		t.Fatalf("late: %+v", v)
+	}
+	if v := verdict(grants); v.Refused() {
+		t.Fatalf("the foundation's manifest was refused: %+v", v)
+	}
 	// Without the verdicts state the indexer judges nothing.
 	ix.Verdicts = nil
 	if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 {

modules/mesh-delivery-indexer/judge/judge.go

@@ -69,6 +69,12 @@ func problemsOf(err error) []string {
 // whose removal from the catalogue ends it (a rule two modules break together refuses both); one no removal ends is in
 // unnamed, for the caller to say. A module not in the answer is accepted.
 func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int) (refused map[string][]string, unnamed []string) {
+	return ShelfCounting(raws, predefined, longestMachine, nil)
+}
+
+// ShelfCounting is Shelf also refusing a manifest of a module outside the foundation that uses an old section whose count
+// is 0 (OldSectionProblems; issue 1966 D4); nil counts refuse none.
+func ShelfCounting(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int, counts OldSectionCounts) (refused map[string][]string, unnamed []string) {
 	refused = map[string][]string{}
 	shelf := catalogue.Shelf{}
 	for name, raw := range raws {
@@ -81,6 +87,10 @@ func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestM
 			refused[name] = []string{fmt.Sprintf("the manifest is of module %q, not %q", m.Module, name)}
 			continue
 		}
+		if problems := OldSectionProblems(m, counts); len(problems) > 0 {
+			refused[name] = problems
+			continue
+		}
 		shelf[name] = m
 	}
 	between := Between(shelf, longestMachine)

modules/mesh-delivery-indexer/judge/oldsections.go

@@ -0,0 +1,69 @@
+package judge
+
+import (
+	"fmt"
+	"slices"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+)
+
+// The old sections end when no module is left in them (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts, per old
+// section, the accepted modules using it, the foundation left out, and the judge refuses a manifest of another module
+// that uses a section whose count is 0. The foundation (the bus module, the grants part and the indexer itself) is
+// created before the bus module can create anything, so it keeps the old sections for good and is always accepted.
+
+// OldSections are the sections of a manifest that keep bus state in the old forms.
+var OldSections = []string{"state", "logs", "streams", "objects"}
+
+// Foundation are the modules that keep using the old sections for good.
+var Foundation = []string{"nats", "mesh-grants", "mesh-delivery-indexer"}
+
+// OldSectionCounts is, per old section, how many accepted modules outside the foundation use it. Nil is no count: the
+// judge then refuses no use of a section.
+type OldSectionCounts map[string]int
+
+// SectionsUsed are the old sections a manifest uses, in OldSections' order.
+func SectionsUsed(m catalogue.Manifest) []string {
+	var used []string
+	for _, s := range OldSections {
+		if (s == "state" && len(m.State) > 0) || (s == "logs" && len(m.Logs) > 0) ||
+			(s == "streams" && len(m.Streams) > 0) || (s == "objects" && len(m.Objects) > 0) {
+			used = append(used, s)
+		}
+	}
+	return used
+}
+
+// CountOldSections counts, per old section, the manifests using it, the foundation's left out; every section is in the
+// answer, with 0 where none uses it.
+func CountOldSections(manifests []catalogue.Manifest) OldSectionCounts {
+	counts := OldSectionCounts{}
+	for _, s := range OldSections {
+		counts[s] = 0
+	}
+	for _, m := range manifests {
+		if slices.Contains(Foundation, m.Module) {
+			continue
+		}
+		for _, s := range SectionsUsed(m) {
+			counts[s]++
+		}
+	}
+	return counts
+}
+
+// OldSectionProblems are the old sections a manifest uses that no accepted module outside the foundation uses any more.
+func OldSectionProblems(m catalogue.Manifest, counts OldSectionCounts) []string {
+	if counts == nil || slices.Contains(Foundation, m.Module) {
+		return nil
+	}
+	var problems []string
+	for _, s := range SectionsUsed(m) {
+		if counts[s] == 0 {
+			problems = append(problems, fmt.Sprintf(
+				"%s uses the old section %s, which no other module uses any more (0 modules count); ask for it in "+
+					"`requires` by template (novox/hq ADR 0595 §11)", m.Module, s))
+		}
+	}
+	return problems
+}

modules/mesh-delivery-indexer/manifest_test.go

@@ -90,7 +90,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
 		t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits)
 	}
 	// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
-	if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
+	if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
 		t.Errorf("state = %+v, want %+v", m.State, want)
 	}
 	if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
@@ -113,8 +113,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
 	if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) {
 		t.Errorf("invokes = %v, want %v", m.Invokes, want)
 	}
-	// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none.
+	// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11:
-	if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) {
+	// indexer_old_sections reads the count of the old sections (state oldsections).
+	if want := []string{"indexer_take_in", "indexer_old_sections"}; !reflect.DeepEqual(m.Tools, want) {
 		t.Errorf("tools = %v, want %v", m.Tools, want)
 	}
 	// No streams, no bus, no seat (ADR 0437 §4); it says no event.

modules/mesh-delivery-indexer/module.json

@@ -3,7 +3,8 @@
   "version": "1",
   "slug": "indexer",
   "tools": [
-    "indexer_take_in"
+    "indexer_take_in",
+    "indexer_old_sections"
   ],
   "consumes": [
     "mesh-build-go.toolchain.reported",
@@ -21,6 +22,7 @@
     { "name": "taken", "history": 1 },
     { "name": "takeins", "history": 1 },
     { "name": "verdicts", "history": 5 },
+    { "name": "oldsections", "history": 1 },
     { "name": "packages", "history": 5 }
   ],
   "emits": [