Merge pull request 'The manifest reader reads and checks a kept data's form and catch-up tool' (#180) from fix/manifest-form-reader into main

go/catalogue/busstate.go

@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"slices"
-	"sort"
 	"strings"
 )
 
@@ -144,6 +142,8 @@ type BusStateEntry struct {
 	Configuration map[string]json.RawMessage `json:"configuration,omitempty"`
 	// Types are a ledger's types (ADR 0597 §1), for the storage type ledger only.
 	Types []LedgerType `json:"types,omitempty"`
+	// KeptLayout is the layout its holder says it writes: its form and its catch-up tool (design 47).
+	KeptLayout
 }
 
 // isBusStateEntry is whether a `requires` item in its long form asks for bus state: it names a storage type.
@@ -179,6 +179,9 @@ func readBusStateEntry(raw json.RawMessage) (BusStateEntry, []string, error) {
 		var e BusStateEntry
 		err := strict(&e)
 		if err == nil {
+			if err := e.KeptLayout.refuse(fmt.Sprintf("requires entry %q", e.Name)); err != nil {
+				return BusStateEntry{}, nil, err
+			}
 			return e, aside, nil
 		}
 		unknown := asUnknownField(err)
@@ -214,14 +217,10 @@ func setAsideAlso(unknown string, paths []string) string {
 // contribution's state name one of these, in either form.
 func KeptStates(m Manifest) []string {
 	var out []string
-	for _, s := range m.State {
+	for _, k := range KeptDeclarations(m) {
-		out = append(out, s.Name)
+		if k.Kind == KeptKindState {
-	}
+			out = append(out, k.Name)
-	for _, e := range m.BusState {
-		if e.Storage == StorageKeyValue {
-			out = append(out, e.Name)
 		}
 	}
-	sort.Strings(out)
+	return out
-	return slices.Compact(out)
 }

go/catalogue/kept_form.go

@@ -0,0 +1,146 @@
+package catalogue
+
+import (
+	"fmt"
+	"sort"
+)
+
+// A kept data's form (novox/hq design 47, *A module's kept data on the bus, backed up and restored with its delivery*):
+// every kept data a manifest declares, in an old section (`state`, `logs`, `streams`, `objects`) or in `requires` by
+// template (the five kinds: state, stream, objects, ledger, log), names a `form`, a number its holder raises whenever it
+// changes how it writes that data, or stops declaring it; a declaration naming none reads as form 0. Beside it, the
+// catch-up tool: the module's own tool that brings the data forward once a restore point of it is restored, recorded in
+// the restore point when it is taken. The declaration alone cannot tell a changed layout of the same bucket, so the holder
+// says it. Which form is right, and that the catch-up tool is the module's own, is the indexer's judge's.
+
+// KeptLayout is what a kept data's declaration says of the layout its holder writes: its form and its catch-up tool.
+type KeptLayout struct {
+	// Form is the form of the kept data's layout, 0 where none is said.
+	Form int `json:"form,omitempty"`
+	// CatchUp is the module's own tool a restore of this kept data is followed by, named by the manifest that raised its
+	// form; empty where none is.
+	CatchUp string `json:"catch-up,omitempty"`
+}
+
+// refuse is the one reading of a layout's form every kept kind's decoder calls: a form is a whole number from 0, and one
+// below it is refused here, naming the declaration (`what` is its kind and name, such as `state "seen"`).
+func (k KeptLayout) refuse(what string) error {
+	if k.Form < 0 {
+		return fmt.Errorf("%s: its form is %d; a form is a whole number from 0, which its holder raises when it changes how "+
+			"it writes it", what, k.Form)
+	}
+	return nil
+}
+
+// The kinds of kept data, by the template each is on (ADR 0595 §2): the five a restore point is taken of.
+const (
+	KeptKindState   = "state"
+	KeptKindStream  = "stream"
+	KeptKindObjects = "objects"
+	KeptKindLedger  = "ledger"
+	KeptKindLog     = "log"
+)
+
+// KeptDeclared is one kept data a manifest declares: its local name, its kind and its layout.
+type KeptDeclared struct {
+	Name string
+	Kind string
+	KeptLayout
+}
+
+// KeptDeclarations are every kept data a manifest declares, in the old sections and in `requires` by template, sorted by
+// name and kind, once each: the one reader of what a module keeps (KeptStates is its states). An entry in `requires` is
+// of its storage type's kind, a stream on the template log a log.
+func KeptDeclarations(m Manifest) []KeptDeclared {
+	var out []KeptDeclared
+	for _, s := range m.State {
+		out = append(out, KeptDeclared{s.Name, KeptKindState, s.KeptLayout})
+	}
+	for _, l := range m.Logs {
+		out = append(out, KeptDeclared{l.Name, KeptKindLog, l.KeptLayout})
+	}
+	for _, s := range m.Streams {
+		out = append(out, KeptDeclared{s.Name, KeptKindStream, s.KeptLayout})
+	}
+	for _, o := range m.Objects {
+		out = append(out, KeptDeclared{o.Name, KeptKindObjects, o.KeptLayout})
+	}
+	for _, e := range m.BusState {
+		kind := KeptKindStream
+		switch {
+		case e.Storage == StorageKeyValue:
+			kind = KeptKindState
+		case e.Storage == StorageObjectStore:
+			kind = KeptKindObjects
+		case e.Storage == StorageLedger:
+			kind = KeptKindLedger
+		case e.Template == TemplateLog:
+			kind = KeptKindLog
+		}
+		out = append(out, KeptDeclared{e.Name, kind, e.KeptLayout})
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		if out[i].Name != out[j].Name {
+			return out[i].Name < out[j].Name
+		}
+		return out[i].Kind < out[j].Kind
+	})
+	once := out[:0]
+	for i, k := range out {
+		if i > 0 && k.Name == out[i-1].Name && k.Kind == out[i-1].Kind {
+			continue
+		}
+		once = append(once, k)
+	}
+	return once
+}
+
+// FormChange is one kept data whose form differs between the build a node runs and the one a want moves it to, by its
+// name and kind: Running is its declaration in the running manifest and Applied in the one being applied, nil where that
+// manifest declares none.
+type FormChange struct {
+	Name    string
+	Kind    string
+	Running *KeptLayout
+	Applied *KeptLayout
+}
+
+// FormChanges are the kept data whose form differs between the running manifest and the one being applied, sorted by name
+// and kind: declared by both at another form, or by one of them only (a form-0 declaration of either side included, so
+// a revert to a manifest that declared it before is seen). Which of them is backed up before the move, and which restored
+// at a revert, is the node agent's applying it (design 47); a kept data whose form is unchanged is never one.
+func FormChanges(running, applied Manifest) []FormChange {
+	type key struct{ name, kind string }
+	before := map[key]KeptLayout{}
+	for _, k := range KeptDeclarations(running) {
+		before[key{k.Name, k.Kind}] = k.KeptLayout
+	}
+	after := map[key]KeptLayout{}
+	for _, k := range KeptDeclarations(applied) {
+		after[key{k.Name, k.Kind}] = k.KeptLayout
+	}
+	var out []FormChange
+	for k, r := range before {
+		a, has := after[k]
+		if has && a.Form == r.Form {
+			continue
+		}
+		c := FormChange{Name: k.name, Kind: k.kind, Running: &r}
+		if has {
+			c.Applied = &a
+		}
+		out = append(out, c)
+	}
+	for k, a := range after {
+		if _, has := before[k]; !has {
+			out = append(out, FormChange{Name: k.name, Kind: k.kind, Applied: &a})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Name != out[j].Name {
+			return out[i].Name < out[j].Name
+		}
+		return out[i].Kind < out[j].Kind
+	})
+	return out
+}

go/catalogue/kept_form_test.go

@@ -0,0 +1,89 @@
+package catalogue
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// W1.5 (design 47, *Its form is declared*): every kept data, in an old section or in `requires` by template, says its
+// form and its catch-up tool; a declaration naming none reads as form 0. KeptDeclarations reads them in one walk, and
+// FormChanges is what differs between a running manifest and the one a want applies.
+func TestAKeptDataSaysItsFormAndItsCatchUp(t *testing.T) {
+	m, err := decodeForTest([]byte(`{"module":"x","version":"1","tools":["x_take_in"],
+		"state": ["seen", {"name":"modules","history":20,"form":1,"catch-up":"x_take_in"}],
+		"logs": [{"name":"acts","form":2}],
+		"streams": [{"name":"forge","events":["merged"],"form":1,"catch-up":"x_take_in"}],
+		"objects": [{"name":"revisions","max-mib":64,"form":1,"catch-up":"x_take_in"}],
+		"requires": [
+			{"name":"records","storage":"ledger","template":"ledger","form":3,"catch-up":"x_take_in"},
+			{"name":"holds","storage":"key-value","template":"state"},
+			{"name":"history","storage":"stream","template":"log","form":1},
+			{"name":"marks","storage":"stream","template":"events"},
+			{"name":"blobs","storage":"object-store","template":"objects","form":4}]}`))
+	if err != nil || m.SetAside() != "" {
+		t.Fatalf("refused or set aside: %v %q", err, m.SetAside())
+	}
+	want := []KeptDeclared{
+		{"acts", KeptKindLog, KeptLayout{Form: 2}},
+		{"blobs", KeptKindObjects, KeptLayout{Form: 4}},
+		{"forge", KeptKindStream, KeptLayout{1, "x_take_in"}},
+		{"history", KeptKindLog, KeptLayout{Form: 1}},
+		{"holds", KeptKindState, KeptLayout{}},
+		{"marks", KeptKindStream, KeptLayout{}},
+		{"modules", KeptKindState, KeptLayout{1, "x_take_in"}},
+		{"records", KeptKindLedger, KeptLayout{3, "x_take_in"}},
+		{"revisions", KeptKindObjects, KeptLayout{1, "x_take_in"}},
+		{"seen", KeptKindState, KeptLayout{}},
+	}
+	if got := KeptDeclarations(m); !reflect.DeepEqual(got, want) {
+		t.Errorf("kept data read as\n%+v\nwant\n%+v", got, want)
+	}
+	if got := KeptStates(m); !reflect.DeepEqual(got, []string{"holds", "modules", "seen"}) {
+		t.Errorf("kept states %v", got)
+	}
+	// Stored and read back, each form and catch-up tool stays; a state with only a form keeps its long form.
+	if back := roundTrip(t, m); !reflect.DeepEqual(KeptDeclarations(back), want) {
+		t.Errorf("read back as %+v", KeptDeclarations(back))
+	}
+
+	// What differs between the build a node runs and the one applied: a form raised, a kept data no longer declared, one
+	// newly declared at form 0 (as a revert to a manifest from before sees it); an unchanged form is none.
+	running, err := decodeForTest([]byte(`{"module":"x","version":"1","state":["modules","seen"],"logs":["acts"]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	applied, err := decodeForTest([]byte(`{"module":"x","version":"1","state":["seen","fresh"],
+		"logs":[{"name":"acts","form":1,"catch-up":"x_take_in"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	changes := FormChanges(running, applied)
+	wantChanges := []FormChange{
+		{Name: "acts", Kind: KeptKindLog, Running: &KeptLayout{}, Applied: &KeptLayout{1, "x_take_in"}},
+		{Name: "fresh", Kind: KeptKindState, Applied: &KeptLayout{}},
+		{Name: "modules", Kind: KeptKindState, Running: &KeptLayout{}},
+	}
+	if !reflect.DeepEqual(changes, wantChanges) {
+		t.Errorf("form changes %+v, want %+v", changes, wantChanges)
+	}
+	if got := FormChanges(applied, applied); len(got) != 0 {
+		t.Errorf("a manifest applied over itself changes %+v", got)
+	}
+}
+
+// A form below 0 is refused where it is read, in every kept kind's declaration, naming it.
+func TestAKeptDataFormBelowZeroIsRefused(t *testing.T) {
+	for _, c := range []struct{ name, raw, says string }{
+		{"state", `"state":[{"name":"seen","form":-3}]`, `state "seen": its form is -3`},
+		{"log", `"logs":[{"name":"acts","form":-1}]`, `log "acts": its form is -1`},
+		{"stream", `"streams":[{"name":"forge","events":["merged"],"form":-2}]`, `stream "forge": its form is -2`},
+		{"objects", `"objects":[{"name":"revisions","max-mib":64,"form":-1}]`, `object bucket "revisions": its form is -1`},
+		{"requires", `"requires":[{"name":"holds","storage":"key-value","template":"state","form":-1}]`, `requires entry "holds": its form is -1`},
+	} {
+		_, err := decodeForTest([]byte(`{"module":"x","version":"1",` + c.raw + `}`))
+		if err == nil || !strings.Contains(err.Error(), c.says) {
+			t.Errorf("%s: read as %v, want a refusal holding %q", c.name, err, c.says)
+		}
+	}
+}

go/catalogue/logs.go

@@ -23,6 +23,8 @@ type LogDeclaration struct {
 	// entry leaves a log: the server ages it out, never a delete or a purge (novox/hq ADR 0565 §10: the hand-act log
 	// keeps every act 400 days; issue 1953 S-age).
 	MaxAgeDays int `json:"max-age-days,omitempty"`
+	// KeptLayout is the layout its holder says it writes: its form and its catch-up tool (design 47).
+	KeptLayout
 }
 
 // LogMostAgeDays is the longest a log may keep an entry by age: past it, a log keeps its entries as long as it is.
@@ -36,7 +38,7 @@ func (l LogDeclaration) Cap() int {
 	return l.MaxMiB
 }
 
-// UnmarshalJSON reads a log as its bare name, or as {name, max-mib, max-age-days}.
+// UnmarshalJSON reads a log as its bare name, or as {name, max-mib, max-age-days, form, catch-up}.
 func (l *LogDeclaration) UnmarshalJSON(raw []byte) error {
 	trimmed := bytes.TrimSpace(raw)
 	if len(trimmed) > 0 && trimmed[0] == '"' {
@@ -46,13 +48,18 @@ func (l *LogDeclaration) UnmarshalJSON(raw []byte) error {
 		Name       string `json:"name"`
 		MaxMiB     *int   `json:"max-mib"`
 		MaxAgeDays *int   `json:"max-age-days"`
+		KeptLayout
 	}
 	dec := json.NewDecoder(bytes.NewReader(trimmed))
 	dec.DisallowUnknownFields()
 	if err := dec.Decode(&full); err != nil {
-		return fmt.Errorf("a log is either a name or {name, max-mib, max-age-days}: %w", typedUnknown(err))
+		return fmt.Errorf("a log is either a name or {name, max-mib, max-age-days, form, catch-up}: %w", typedUnknown(err))
+	}
+	if err := full.KeptLayout.refuse(fmt.Sprintf("log %q", full.Name)); err != nil {
+		return err
 	}
 	l.Name = full.Name
+	l.KeptLayout = full.KeptLayout
 	l.MaxMiB = 0
 	if full.MaxMiB != nil {
 		// Said, and said as nothing: refused rather than read as the default, which it did not say.
@@ -73,7 +80,7 @@ func (l *LogDeclaration) UnmarshalJSON(raw []byte) error {
 
 // MarshalJSON writes back the short form when there is nothing else to say.
 func (l LogDeclaration) MarshalJSON() ([]byte, error) {
-	if l.MaxMiB == 0 && l.MaxAgeDays == 0 {
+	if l.MaxMiB == 0 && l.MaxAgeDays == 0 && l.KeptLayout == (KeptLayout{}) {
 		return json.Marshal(l.Name)
 	}
 	type plain LogDeclaration

go/catalogue/objects.go

@@ -26,6 +26,8 @@ type ObjectsDeclaration struct {
 	// Grant names the trusted config value of the module whose entries name the modules that may read it; absent,
 	// no other module reads it.
 	Grant *ObjectsGrant `json:"grant,omitempty"`
+	// KeptLayout is the layout its holder says it writes: its form and its catch-up tool (design 47).
+	KeptLayout
 }
 
 // ObjectsGrant says which trusted config value of the owning module grants another module the read of a bucket
@@ -45,7 +47,10 @@ func (o *ObjectsDeclaration) UnmarshalJSON(raw []byte) error {
 	dec := json.NewDecoder(bytes.NewReader(raw))
 	dec.DisallowUnknownFields()
 	if err := dec.Decode(&p); err != nil {
-		return fmt.Errorf("an object bucket is {name, max-mib, grant}: %w", typedUnknown(err))
+		return fmt.Errorf("an object bucket is {name, max-mib, grant, form, catch-up}: %w", typedUnknown(err))
+	}
+	if err := p.KeptLayout.refuse(fmt.Sprintf("object bucket %q", p.Name)); err != nil {
+		return err
 	}
 	*o = ObjectsDeclaration(p)
 	return nil

go/catalogue/state.go

@@ -35,9 +35,11 @@ type StateDeclaration struct {
 	// the tool runner's account, read it and never write it (the lead's ruling, 2026-11-04: nats' warrant spend record
 	// `spent`). The judge's rule and the grants follow from it.
 	ReadOnly bool `json:"read-only,omitempty"`
+	// KeptLayout is the layout its holder says it writes: its form and its catch-up tool (design 47).
+	KeptLayout
 }
 
-// UnmarshalJSON reads a bucket as its bare name, or as {name, history, ttl-seconds, per-machine, key-ttl, read-only}.
+// UnmarshalJSON reads a bucket as its bare name, or as {name, history, ttl-seconds, per-machine, key-ttl, read-only, form, catch-up}.
 func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
 	trimmed := bytes.TrimSpace(raw)
 	if len(trimmed) > 0 && trimmed[0] == '"' {
@@ -48,7 +50,10 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
 	dec := json.NewDecoder(bytes.NewReader(trimmed))
 	dec.DisallowUnknownFields()
 	if err := dec.Decode(&full); err != nil {
-		return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine, key-ttl, read-only}: %w", typedUnknown(err))
+		return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine, key-ttl, read-only, form, catch-up}: %w", typedUnknown(err))
+	}
+	if err := full.KeptLayout.refuse(fmt.Sprintf("state %q", full.Name)); err != nil {
+		return err
 	}
 	*s = StateDeclaration(full)
 	return nil
@@ -56,7 +61,7 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error {
 
 // MarshalJSON writes back the short form when there is nothing else to say.
 func (s StateDeclaration) MarshalJSON() ([]byte, error) {
-	if s.History == 0 && s.TTLSeconds == 0 && !s.PerMachine && !s.KeyTTL && !s.ReadOnly {
+	if s.History == 0 && s.TTLSeconds == 0 && !s.PerMachine && !s.KeyTTL && !s.ReadOnly && s.KeptLayout == (KeptLayout{}) {
 		return json.Marshal(s.Name)
 	}
 	type plain StateDeclaration

go/catalogue/streams.go

@@ -64,6 +64,8 @@ type StreamDeclaration struct {
 	// Duplicates is a work-queue stream's duplicate window in hours, at most its maximum age; refused
 	// without retention work-queue (novox/hq ADR 0436 §6).
 	Duplicates int `json:"duplicates,omitempty"`
+	// KeptLayout is the layout its holder says it writes: its form and its catch-up tool (design 47).
+	KeptLayout
 }
 
 // StreamGrant says which trusted config value of the module grants a consumer a part of the stream, and
@@ -84,7 +86,10 @@ func (s *StreamDeclaration) UnmarshalJSON(raw []byte) error {
 	dec.DisallowUnknownFields()
 	if err := dec.Decode(&p); err != nil {
 		return fmt.Errorf("a stream is {name, events, max-age-days, max-mib, grant, retention, max-deliveries, "+
-			"duplicates}: %w", typedUnknown(err))
+			"duplicates, form, catch-up}: %w", typedUnknown(err))
+	}
+	if err := p.KeptLayout.refuse(fmt.Sprintf("stream %q", p.Name)); err != nil {
+		return err
 	}
 	*s = StreamDeclaration(p)
 	return nil