A requires entry naming a storage type asks for bus state, read leniently beside the plain names (issue 1966 S1)

ADR 0595 §1: the typed long form {name, storage, template, options,
configuration} decodes into Manifest.BusState, its name also on Requires;
a key it does not know is set aside and named, never refused (ADR 0578 §3).
The requirements tests are one table.

go/catalogue/busstate.go

@@ -0,0 +1,96 @@
+package catalogue
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// Bus state is what a module keeps on the bus, asked for as a provision of the bus module in `requires` (novox/hq ADR
+// 0595 §1, design 32 § 4): an entry of `requires` in its typed long form, the local name, the storage type, the template
+// and the template's options. This is the contract only (ADR 0578 §7, ADR 0593 §1): the entry's shape, decoded
+// leniently. Every rule that judges an entry (a template known, an option within its bound, a fixed setting not given)
+// is the indexer's judge's.
+
+// BusStateField is where a stored manifest keeps its bus-state entries: beside `requires`, which stays a plain list of
+// names, as OptionalRequiresKey keeps the optional ones.
+const BusStateField = "bus-state"
+
+// BusStateEntry is one piece of a module's bus state, as its manifest asks for it.
+type BusStateEntry struct {
+	// Name is its local name; the mesh, never the module, names it on the bus (ADR 0201 §4).
+	Name string `json:"name"`
+	// Storage is its storage type (design 25 § 12.1): how the bus keeps it.
+	Storage string `json:"storage"`
+	// Template is the predefined configuration of that storage type it uses; empty for a bare stream, which gives
+	// Configuration instead.
+	Template string `json:"template,omitempty"`
+	// Options are the template's settings the module changes, by option name; each value as the manifest wrote it.
+	Options map[string]json.RawMessage `json:"options,omitempty"`
+	// Configuration is a configuration of the module's own, where no template fits: a bare stream's, or a ledger's
+	// whose history expires (ADR 0595 §2).
+	Configuration map[string]json.RawMessage `json:"configuration,omitempty"`
+}
+
+// isBusStateEntry is whether a `requires` item in its long form asks for bus state: it names a storage type.
+func isBusStateEntry(raw json.RawMessage) bool {
+	var keys map[string]json.RawMessage
+	if json.Unmarshal(raw, &keys) != nil {
+		return false
+	}
+	_, has := keys["storage"]
+	return has
+}
+
+// readBusStateEntry reads one bus-state entry leniently (ADR 0578 §3): a key it does not know, at any depth, is set
+// aside and its path named, never refused. An error is only an entry of the wrong shape.
+func readBusStateEntry(raw json.RawMessage) (BusStateEntry, []string, error) {
+	var tree any
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.UseNumber()
+	if err := dec.Decode(&tree); err != nil {
+		return BusStateEntry{}, nil, err
+	}
+	var aside []string
+	strict := func(e *BusStateEntry) error {
+		b, err := json.Marshal(tree)
+		if err != nil {
+			return err
+		}
+		d := json.NewDecoder(bytes.NewReader(b))
+		d.DisallowUnknownFields()
+		return typedUnknown(d.Decode(e))
+	}
+	for tries := 0; ; tries++ {
+		var e BusStateEntry
+		err := strict(&e)
+		if err == nil {
+			return e, aside, nil
+		}
+		unknown := asUnknownField(err)
+		if unknown == nil || tries > 64 {
+			return BusStateEntry{}, nil, err
+		}
+		path, ok := removalThatHelps(tree, unknown.Field, err.Error(), func() error { var e BusStateEntry; return strict(&e) })
+		if !ok {
+			if removeEvery(tree, unknown.Field) == 0 {
+				return BusStateEntry{}, nil, err
+			}
+			path = "…." + unknown.Field
+		}
+		aside = append(aside, path)
+	}
+}
+
+// setAsideAlso adds paths read without to what a manifest set aside (SetAside), in its words.
+func setAsideAlso(unknown string, paths []string) string {
+	if len(paths) == 0 {
+		return unknown
+	}
+	if unknown == "" {
+		field := paths[0][strings.LastIndex(paths[0], ".")+1:]
+		unknown = fmt.Sprintf("json: unknown field %q", field)
+	}
+	return unknown + " (read without " + strings.Join(paths, ", ") + ")"
+}

go/catalogue/manifest.go

@@ -478,6 +478,10 @@ type Manifest struct {
 	// RequireShares is who else may use the bucket a module requires, by requirement (BucketShare, novox/hq ADR 0468
 	// §8): filled from the long form's `shares`, or from the stored form's `requires-shares`; never written by hand.
 	RequireShares map[string][]BucketShare `json:"-"`
+	// BusState is what the module keeps on the bus, each entry asked for in `requires` in its typed long form
+	// (BusStateEntry, novox/hq ADR 0595 §1), whose name is also in Requires. Stored beside `requires`
+	// (BusStateField), which stays a plain list of names.
+	BusState []BusStateEntry `json:"bus-state,omitempty"`
 
 	// Emits are the events this module publishes, named **locally**: `order.placed`, not a subject
 	// and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the
@@ -1556,18 +1560,21 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
 	}
 	var optionalRequires map[string]bool
 	var requireShares map[string][]BucketShare
+	var busState []BusStateEntry
+	var busStateAside []string
 	if requires, ok := keys["requires"]; ok && string(requires) != "null" {
-		names, optional, shares, err := readRequires(requires)
+		read, err := readRequires(requires)
 		if err != nil {
 			return err
 		}
-		plainNames, err := json.Marshal(names)
+		plainNames, err := json.Marshal(read.names)
 		if err != nil {
 			return err
 		}
 		keys["requires"] = plainNames
-		optionalRequires = optional
-		requireShares = shares
+		optionalRequires = read.optional
+		requireShares = read.shares
+		busState, busStateAside = read.busState, read.aside
 	}
 	// The stored form's shares (RequireSharesKey).
 	if stored, ok := keys[RequireSharesKey]; ok {
@@ -1631,7 +1638,9 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
 		}
 	}
 	*m = Manifest(fields)
-	m.unknown = unknown
+	m.unknown = setAsideAlso(unknown, busStateAside)
+	// A bus-state entry of `requires`, beside those of the stored form (BusStateField).
+	m.BusState = append(m.BusState, busState...)
 	m.forms = forms
 	m.OptionalRequires = optionalRequires
 	m.RequireShares = requireShares

go/catalogue/optional_requirements.go

@@ -21,34 +21,59 @@ type requirementForm struct {
 	Shares []BucketShare `json:"shares,omitempty"`
 }
 
-// readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the
-// names and which of them are optional.
-func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][]BucketShare, error) {
+// requiresRead is `requires` as readRequires read it.
+type requiresRead struct {
+	names    []string
+	optional map[string]bool
+	shares   map[string][]BucketShare
+	// busState are the entries asking for bus state (BusStateEntry), each also named in names; aside the paths of the
+	// keys they had that this package does not know, set aside (ADR 0578 §3).
+	busState []BusStateEntry
+	aside    []string
+}
+
+// readRequires reads `requires` in its forms — a bare name, {name, optional}, or a bus-state entry (BusStateEntry,
+// novox/hq ADR 0595 §1) — into the names, which of them are optional, and the bus-state entries.
+func readRequires(raw json.RawMessage) (requiresRead, error) {
 	var entries []json.RawMessage
 	if err := json.Unmarshal(raw, &entries); err != nil {
-		return nil, nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
+		return requiresRead{}, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
 	}
 	var names []string
 	optional := map[string]bool{}
 	var shares map[string][]BucketShare
+	var busState []BusStateEntry
+	var aside []string
 	for i, e := range entries {
 		var plain string
 		if err := json.Unmarshal(e, &plain); err == nil {
 			names = append(names, plain)
 			continue
 		}
+		if isBusStateEntry(e) {
+			entry, unknown, err := readBusStateEntry(e)
+			if err != nil {
+				return requiresRead{}, fmt.Errorf("requires[%d]: a bus-state entry is {name, storage, template, options, configuration} (novox/hq ADR 0595 §1): %w", i, err)
+			}
+			names = append(names, entry.Name)
+			busState = append(busState, entry)
+			for _, path := range unknown {
+				aside = append(aside, fmt.Sprintf("requires[%d]%s", i, path))
+			}
+			continue
+		}
 		var full requirementForm
 		dec := json.NewDecoder(bytes.NewReader(e))
 		dec.DisallowUnknownFields()
 		if err := dec.Decode(&full); err != nil {
-			return nil, nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
+			return requiresRead{}, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
 		}
 		if strings.TrimSpace(full.Name) == "" {
-			return nil, nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
+			return requiresRead{}, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
 		}
 		names = append(names, full.Name)
 		if err := checkBucketShares(full.Name, full.Shares); err != nil {
-			return nil, nil, nil, fmt.Errorf("requires[%d]: %w", i, err)
+			return requiresRead{}, fmt.Errorf("requires[%d]: %w", i, err)
 		}
 		if len(full.Shares) > 0 {
 			if shares == nil {
@@ -63,7 +88,7 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][]
 	if len(optional) == 0 {
 		optional = nil
 	}
-	return names, optional, shares, nil
+	return requiresRead{names: names, optional: optional, shares: shares, busState: busState, aside: aside}, nil
 }
 
 // OptionalRequiresKey is where a stored manifest keeps which of its requirements are optional: beside

go/catalogue/optional_requirements_test.go

@@ -2,6 +2,7 @@ package catalogue
 
 import (
 	"encoding/json"
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -9,9 +10,8 @@ import (
 // A requirement a module can run without (novox/hq ADR 0330, issue 166). The shape of the issue:
 // grafana runs without a data source and requires `influxdb-api` softly; a mesh without influxdb
 // could not assign grafana at all, and one with influxdb in its catalogue would have had it installed
-// for grafana.
-
-func TestARequirementIsHardUnlessMarkedOptional(t *testing.T) {
+// for grafana. And `requires` in its typed long form, a module's bus state (novox/hq ADR 0595 §1).
+func TestARequirementsLongForm(t *testing.T) {
 	m := mustParse(t, `{"module":"grafana","version":"1",
 		"requires":["route",{"name":"influxdb-api","optional":true},{"name":"oidc-client"}]}`)
 	if strings.Join(m.Requires, " ") != "route influxdb-api oidc-client" {
@@ -24,23 +24,12 @@ func TestARequirementIsHardUnlessMarkedOptional(t *testing.T) {
 		t.Error("a requirement marked optional read as hard")
 	}
 	// And written back as it was read, so a manifest through the store keeps its form.
-	raw, err := json.Marshal(m)
-	if err != nil {
-		t.Fatal(err)
-	}
-	var again Manifest
-	if err := json.Unmarshal(raw, &again); err != nil {
-		t.Fatal(err)
-	}
+	again := roundTrip(t, m)
 	if !again.IsOptional("influxdb-api") || again.IsOptional("route") || len(again.Requires) != 3 {
-		t.Fatalf("written and read back, it says %v optional %v (%s)", again.Requires, again.OptionalRequires, raw)
+		t.Fatalf("written and read back, it says %v optional %v", again.Requires, again.OptionalRequires)
 	}
-	if strings.Contains(string(raw), `"name":`) {
-		t.Errorf("a requirement was stored in the long form: %s", raw)
-	}
-}
 
-func TestARequirementsLongFormRefusesWhatItDoesNotKnow(t *testing.T) {
+	// The long form {name, optional} refuses what it does not know.
 	for _, raw := range []string{
 		`{"module":"x","version":"1","requires":[{"name":"influxdb-api","optinal":true}]}`,
 		`{"module":"x","version":"1","requires":[{"optional":true}]}`,
@@ -49,4 +38,73 @@ func TestARequirementsLongFormRefusesWhatItDoesNotKnow(t *testing.T) {
 			t.Errorf("accepted %s", raw)
 		}
 	}
+
+	records := BusStateEntry{Name: "records", Storage: "ledger", Template: "ledger",
+		Options: map[string]json.RawMessage{"max-mib": json.RawMessage(`1024`)}}
+	for _, c := range []struct {
+		name     string
+		requires string
+		names    string
+		busState []BusStateEntry
+		aside    []string
+	}{
+		// 1966/AC1: a plain name and a bus-state entry: Requires names both, BusState holds the one entry whole.
+		{"a plain name and a bus-state entry", `["postgres-database",
+			{"name":"records","storage":"ledger","template":"ledger","options":{"max-mib":1024}}]`,
+			"postgres-database records", []BusStateEntry{records}, nil},
+		// 1966/AC2: an unknown key is set aside and named, never refused; the entry decodes.
+		{"a bus-state entry with a key it does not know", `[
+			{"name":"records","storage":"ledger","template":"ledger","colour":"red","options":{"max-mib":1024}}]`,
+			"records", []BusStateEntry{records}, []string{"requires[0].colour"}},
+	} {
+		got, err := decodeForTest([]byte(`{"module":"x","version":"1","requires":` + c.requires + `}`))
+		if err != nil {
+			t.Errorf("%s: refused: %v", c.name, err)
+			continue
+		}
+		if strings.Join(got.Requires, " ") != c.names {
+			t.Errorf("%s: requires read as %v, want %s", c.name, got.Requires, c.names)
+		}
+		if !reflect.DeepEqual(got.BusState, c.busState) {
+			t.Errorf("%s: bus state read as %+v, want %+v", c.name, got.BusState, c.busState)
+		}
+		for _, path := range c.aside {
+			if !strings.Contains(got.SetAside(), path) {
+				t.Errorf("%s: %s is not named among what was set aside: %q", c.name, path, got.SetAside())
+			}
+		}
+		if len(c.aside) == 0 && got.SetAside() != "" {
+			t.Errorf("%s: set aside %q", c.name, got.SetAside())
+		}
+		// Stored and read back, the entries stay whole and `requires` stays plain names.
+		back := roundTrip(t, got)
+		if !reflect.DeepEqual(back.BusState, c.busState) || strings.Join(back.Requires, " ") != c.names {
+			t.Errorf("%s: read back as %v %+v", c.name, back.Requires, back.BusState)
+		}
+	}
+}
+
+// roundTrip is m written as the store keeps it and read again; the stored form keeps `requires` plain names.
+func roundTrip(t *testing.T, m Manifest) Manifest {
+	t.Helper()
+	raw, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var stored struct {
+		Requires []any `json:"requires"`
+	}
+	if err := json.Unmarshal(raw, &stored); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range stored.Requires {
+		if _, plain := r.(string); !plain {
+			t.Errorf("a requirement was stored in the long form: %s", raw)
+		}
+	}
+	var again Manifest
+	if err := json.Unmarshal(raw, &again); err != nil {
+		t.Fatalf("%v: %s", err, raw)
+	}
+	return again
 }

go/catalogue/unknown_field.go

@@ -75,7 +75,7 @@ func prunedFields(keys map[string]json.RawMessage) (manifestFields, []string, er
 				removed = append(removed, k)
 				break
 			}
-			path, ok := removalThatHelps(k, tree[k], unknown.Field, err.Error())
+			path, ok := removalThatHelps(tree[k], unknown.Field, err.Error(), func() error { return decodesAlone(k, tree[k]) })
 			if !ok {
 				// The same key unknown in two entries alike: no one removal changes the words.
 				// Every occurrence goes, and the field is judged again.
@@ -113,8 +113,8 @@ func decodesAlone(k string, v any) error {
 }
 
 // removalThatHelps removes, from v, the one occurrence of key whose removal changes what the strict
-// decoder says of field k, and says where it was. Every other occurrence is left as it was.
-func removalThatHelps(k string, v any, key, said string) (string, bool) {
+// decoder says (decodes, which decodes v strictly), and says where it was. Every other occurrence is left as it was.
+func removalThatHelps(v any, key, said string, decodes func() error) (string, bool) {
 	var found bool
 	var where string
 	var walk func(node any, at string) bool
@@ -126,7 +126,7 @@ func removalThatHelps(k string, v any, key, said string) (string, bool) {
 				// Accepted only when the decoder is past it: nothing left, or an unknown key said
 				// elsewhere. A different kind of error means the removal broke the entry; the same
 				// words mean this was not the occurrence it refused.
-				err := decodesAlone(k, v)
+				err := decodes()
 				if err == nil || (asUnknownField(err) != nil && err.Error() != said) {
 					found, where = true, at+"."+key
 					return true