diff --git a/go/catalogue/busstate.go b/go/catalogue/busstate.go new file mode 100644 index 0000000..865465c --- /dev/null +++ b/go/catalogue/busstate.go @@ -0,0 +1,96 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +// Bus state is what a module keeps on the bus, asked for as a provision of the bus module in `requires` (novox/hq ADR +// 0595 §1, design 32 § 4): an entry of `requires` in its typed long form, the local name, the storage type, the template +// and the template's options. This is the contract only (ADR 0578 §7, ADR 0593 §1): the entry's shape, decoded +// leniently. Every rule that judges an entry (a template known, an option within its bound, a fixed setting not given) +// is the indexer's judge's. + +// BusStateField is where a stored manifest keeps its bus-state entries: beside `requires`, which stays a plain list of +// names, as OptionalRequiresKey keeps the optional ones. +const BusStateField = "bus-state" + +// BusStateEntry is one piece of a module's bus state, as its manifest asks for it. +type BusStateEntry struct { + // Name is its local name; the mesh, never the module, names it on the bus (ADR 0201 §4). + Name string `json:"name"` + // Storage is its storage type (design 25 § 12.1): how the bus keeps it. + Storage string `json:"storage"` + // Template is the predefined configuration of that storage type it uses; empty for a bare stream, which gives + // Configuration instead. + Template string `json:"template,omitempty"` + // Options are the template's settings the module changes, by option name; each value as the manifest wrote it. + Options map[string]json.RawMessage `json:"options,omitempty"` + // Configuration is a configuration of the module's own, where no template fits: a bare stream's, or a ledger's + // whose history expires (ADR 0595 §2). + Configuration map[string]json.RawMessage `json:"configuration,omitempty"` +} + +// isBusStateEntry is whether a `requires` item in its long form asks for bus state: it names a storage type. +func isBusStateEntry(raw json.RawMessage) bool { + var keys map[string]json.RawMessage + if json.Unmarshal(raw, &keys) != nil { + return false + } + _, has := keys["storage"] + return has +} + +// readBusStateEntry reads one bus-state entry leniently (ADR 0578 §3): a key it does not know, at any depth, is set +// aside and its path named, never refused. An error is only an entry of the wrong shape. +func readBusStateEntry(raw json.RawMessage) (BusStateEntry, []string, error) { + var tree any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&tree); err != nil { + return BusStateEntry{}, nil, err + } + var aside []string + strict := func(e *BusStateEntry) error { + b, err := json.Marshal(tree) + if err != nil { + return err + } + d := json.NewDecoder(bytes.NewReader(b)) + d.DisallowUnknownFields() + return typedUnknown(d.Decode(e)) + } + for tries := 0; ; tries++ { + var e BusStateEntry + err := strict(&e) + if err == nil { + return e, aside, nil + } + unknown := asUnknownField(err) + if unknown == nil || tries > 64 { + return BusStateEntry{}, nil, err + } + path, ok := removalThatHelps(tree, unknown.Field, err.Error(), func() error { var e BusStateEntry; return strict(&e) }) + if !ok { + if removeEvery(tree, unknown.Field) == 0 { + return BusStateEntry{}, nil, err + } + path = "…." + unknown.Field + } + aside = append(aside, path) + } +} + +// setAsideAlso adds paths read without to what a manifest set aside (SetAside), in its words. +func setAsideAlso(unknown string, paths []string) string { + if len(paths) == 0 { + return unknown + } + if unknown == "" { + field := paths[0][strings.LastIndex(paths[0], ".")+1:] + unknown = fmt.Sprintf("json: unknown field %q", field) + } + return unknown + " (read without " + strings.Join(paths, ", ") + ")" +} diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index 322afde..c9e2ccc 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -478,6 +478,10 @@ type Manifest struct { // RequireShares is who else may use the bucket a module requires, by requirement (BucketShare, novox/hq ADR 0468 // §8): filled from the long form's `shares`, or from the stored form's `requires-shares`; never written by hand. RequireShares map[string][]BucketShare `json:"-"` + // BusState is what the module keeps on the bus, each entry asked for in `requires` in its typed long form + // (BusStateEntry, novox/hq ADR 0595 §1), whose name is also in Requires. Stored beside `requires` + // (BusStateField), which stays a plain list of names. + BusState []BusStateEntry `json:"bus-state,omitempty"` // Emits are the events this module publishes, named **locally**: `order.placed`, not a subject // and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the @@ -1556,18 +1560,21 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { } var optionalRequires map[string]bool var requireShares map[string][]BucketShare + var busState []BusStateEntry + var busStateAside []string if requires, ok := keys["requires"]; ok && string(requires) != "null" { - names, optional, shares, err := readRequires(requires) + read, err := readRequires(requires) if err != nil { return err } - plainNames, err := json.Marshal(names) + plainNames, err := json.Marshal(read.names) if err != nil { return err } keys["requires"] = plainNames - optionalRequires = optional - requireShares = shares + optionalRequires = read.optional + requireShares = read.shares + busState, busStateAside = read.busState, read.aside } // The stored form's shares (RequireSharesKey). if stored, ok := keys[RequireSharesKey]; ok { @@ -1631,7 +1638,9 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { } } *m = Manifest(fields) - m.unknown = unknown + m.unknown = setAsideAlso(unknown, busStateAside) + // A bus-state entry of `requires`, beside those of the stored form (BusStateField). + m.BusState = append(m.BusState, busState...) m.forms = forms m.OptionalRequires = optionalRequires m.RequireShares = requireShares diff --git a/go/catalogue/optional_requirements.go b/go/catalogue/optional_requirements.go index 48fcef5..55505b4 100644 --- a/go/catalogue/optional_requirements.go +++ b/go/catalogue/optional_requirements.go @@ -21,34 +21,59 @@ type requirementForm struct { Shares []BucketShare `json:"shares,omitempty"` } -// readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the -// names and which of them are optional. -func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][]BucketShare, error) { +// requiresRead is `requires` as readRequires read it. +type requiresRead struct { + names []string + optional map[string]bool + shares map[string][]BucketShare + // busState are the entries asking for bus state (BusStateEntry), each also named in names; aside the paths of the + // keys they had that this package does not know, set aside (ADR 0578 §3). + busState []BusStateEntry + aside []string +} + +// readRequires reads `requires` in its forms — a bare name, {name, optional}, or a bus-state entry (BusStateEntry, +// novox/hq ADR 0595 §1) — into the names, which of them are optional, and the bus-state entries. +func readRequires(raw json.RawMessage) (requiresRead, error) { var entries []json.RawMessage if err := json.Unmarshal(raw, &entries); err != nil { - return nil, nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err) + return requiresRead{}, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err) } var names []string optional := map[string]bool{} var shares map[string][]BucketShare + var busState []BusStateEntry + var aside []string for i, e := range entries { var plain string if err := json.Unmarshal(e, &plain); err == nil { names = append(names, plain) continue } + if isBusStateEntry(e) { + entry, unknown, err := readBusStateEntry(e) + if err != nil { + return requiresRead{}, fmt.Errorf("requires[%d]: a bus-state entry is {name, storage, template, options, configuration} (novox/hq ADR 0595 §1): %w", i, err) + } + names = append(names, entry.Name) + busState = append(busState, entry) + for _, path := range unknown { + aside = append(aside, fmt.Sprintf("requires[%d]%s", i, path)) + } + continue + } var full requirementForm dec := json.NewDecoder(bytes.NewReader(e)) dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { - return nil, nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err) + return requiresRead{}, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err) } if strings.TrimSpace(full.Name) == "" { - return nil, nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i) + return requiresRead{}, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i) } names = append(names, full.Name) if err := checkBucketShares(full.Name, full.Shares); err != nil { - return nil, nil, nil, fmt.Errorf("requires[%d]: %w", i, err) + return requiresRead{}, fmt.Errorf("requires[%d]: %w", i, err) } if len(full.Shares) > 0 { if shares == nil { @@ -63,7 +88,7 @@ func readRequires(raw json.RawMessage) ([]string, map[string]bool, map[string][] if len(optional) == 0 { optional = nil } - return names, optional, shares, nil + return requiresRead{names: names, optional: optional, shares: shares, busState: busState, aside: aside}, nil } // OptionalRequiresKey is where a stored manifest keeps which of its requirements are optional: beside diff --git a/go/catalogue/optional_requirements_test.go b/go/catalogue/optional_requirements_test.go index 9f0aa04..1475676 100644 --- a/go/catalogue/optional_requirements_test.go +++ b/go/catalogue/optional_requirements_test.go @@ -2,6 +2,7 @@ package catalogue import ( "encoding/json" + "reflect" "strings" "testing" ) @@ -9,9 +10,8 @@ import ( // A requirement a module can run without (novox/hq ADR 0330, issue 166). The shape of the issue: // grafana runs without a data source and requires `influxdb-api` softly; a mesh without influxdb // could not assign grafana at all, and one with influxdb in its catalogue would have had it installed -// for grafana. - -func TestARequirementIsHardUnlessMarkedOptional(t *testing.T) { +// for grafana. And `requires` in its typed long form, a module's bus state (novox/hq ADR 0595 §1). +func TestARequirementsLongForm(t *testing.T) { m := mustParse(t, `{"module":"grafana","version":"1", "requires":["route",{"name":"influxdb-api","optional":true},{"name":"oidc-client"}]}`) if strings.Join(m.Requires, " ") != "route influxdb-api oidc-client" { @@ -24,23 +24,12 @@ func TestARequirementIsHardUnlessMarkedOptional(t *testing.T) { t.Error("a requirement marked optional read as hard") } // And written back as it was read, so a manifest through the store keeps its form. - raw, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - var again Manifest - if err := json.Unmarshal(raw, &again); err != nil { - t.Fatal(err) - } + again := roundTrip(t, m) if !again.IsOptional("influxdb-api") || again.IsOptional("route") || len(again.Requires) != 3 { - t.Fatalf("written and read back, it says %v optional %v (%s)", again.Requires, again.OptionalRequires, raw) + t.Fatalf("written and read back, it says %v optional %v", again.Requires, again.OptionalRequires) } - if strings.Contains(string(raw), `"name":`) { - t.Errorf("a requirement was stored in the long form: %s", raw) - } -} -func TestARequirementsLongFormRefusesWhatItDoesNotKnow(t *testing.T) { + // The long form {name, optional} refuses what it does not know. for _, raw := range []string{ `{"module":"x","version":"1","requires":[{"name":"influxdb-api","optinal":true}]}`, `{"module":"x","version":"1","requires":[{"optional":true}]}`, @@ -49,4 +38,73 @@ func TestARequirementsLongFormRefusesWhatItDoesNotKnow(t *testing.T) { t.Errorf("accepted %s", raw) } } + + records := BusStateEntry{Name: "records", Storage: "ledger", Template: "ledger", + Options: map[string]json.RawMessage{"max-mib": json.RawMessage(`1024`)}} + for _, c := range []struct { + name string + requires string + names string + busState []BusStateEntry + aside []string + }{ + // 1966/AC1: a plain name and a bus-state entry: Requires names both, BusState holds the one entry whole. + {"a plain name and a bus-state entry", `["postgres-database", + {"name":"records","storage":"ledger","template":"ledger","options":{"max-mib":1024}}]`, + "postgres-database records", []BusStateEntry{records}, nil}, + // 1966/AC2: an unknown key is set aside and named, never refused; the entry decodes. + {"a bus-state entry with a key it does not know", `[ + {"name":"records","storage":"ledger","template":"ledger","colour":"red","options":{"max-mib":1024}}]`, + "records", []BusStateEntry{records}, []string{"requires[0].colour"}}, + } { + got, err := decodeForTest([]byte(`{"module":"x","version":"1","requires":` + c.requires + `}`)) + if err != nil { + t.Errorf("%s: refused: %v", c.name, err) + continue + } + if strings.Join(got.Requires, " ") != c.names { + t.Errorf("%s: requires read as %v, want %s", c.name, got.Requires, c.names) + } + if !reflect.DeepEqual(got.BusState, c.busState) { + t.Errorf("%s: bus state read as %+v, want %+v", c.name, got.BusState, c.busState) + } + for _, path := range c.aside { + if !strings.Contains(got.SetAside(), path) { + t.Errorf("%s: %s is not named among what was set aside: %q", c.name, path, got.SetAside()) + } + } + if len(c.aside) == 0 && got.SetAside() != "" { + t.Errorf("%s: set aside %q", c.name, got.SetAside()) + } + // Stored and read back, the entries stay whole and `requires` stays plain names. + back := roundTrip(t, got) + if !reflect.DeepEqual(back.BusState, c.busState) || strings.Join(back.Requires, " ") != c.names { + t.Errorf("%s: read back as %v %+v", c.name, back.Requires, back.BusState) + } + } +} + +// roundTrip is m written as the store keeps it and read again; the stored form keeps `requires` plain names. +func roundTrip(t *testing.T, m Manifest) Manifest { + t.Helper() + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + var stored struct { + Requires []any `json:"requires"` + } + if err := json.Unmarshal(raw, &stored); err != nil { + t.Fatal(err) + } + for _, r := range stored.Requires { + if _, plain := r.(string); !plain { + t.Errorf("a requirement was stored in the long form: %s", raw) + } + } + var again Manifest + if err := json.Unmarshal(raw, &again); err != nil { + t.Fatalf("%v: %s", err, raw) + } + return again } diff --git a/go/catalogue/unknown_field.go b/go/catalogue/unknown_field.go index 5721847..8c2ffe7 100644 --- a/go/catalogue/unknown_field.go +++ b/go/catalogue/unknown_field.go @@ -75,7 +75,7 @@ func prunedFields(keys map[string]json.RawMessage) (manifestFields, []string, er removed = append(removed, k) break } - path, ok := removalThatHelps(k, tree[k], unknown.Field, err.Error()) + path, ok := removalThatHelps(tree[k], unknown.Field, err.Error(), func() error { return decodesAlone(k, tree[k]) }) if !ok { // The same key unknown in two entries alike: no one removal changes the words. // Every occurrence goes, and the field is judged again. @@ -113,8 +113,8 @@ func decodesAlone(k string, v any) error { } // removalThatHelps removes, from v, the one occurrence of key whose removal changes what the strict -// decoder says of field k, and says where it was. Every other occurrence is left as it was. -func removalThatHelps(k string, v any, key, said string) (string, bool) { +// decoder says (decodes, which decodes v strictly), and says where it was. Every other occurrence is left as it was. +func removalThatHelps(v any, key, said string, decodes func() error) (string, bool) { var found bool var where string var walk func(node any, at string) bool @@ -126,7 +126,7 @@ func removalThatHelps(k string, v any, key, said string) (string, bool) { // Accepted only when the decoder is past it: nothing left, or an unknown key said // elsewhere. A different kind of error means the removal broke the entry; the same // words mean this was not the occurrence it refused. - err := decodesAlone(k, v) + err := decodes() if err == nil || (asUnknownField(err) != nil && err.Error() != said) { found, where = true, at+"."+key return true