From af63b233dbd5609bf46f93dd62fddc8b078fd6f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 17:34:53 +0200 Subject: [PATCH] Leave out a module whose stored manifest has an unknown field, and raise it (hq ADR 0262 review) A key dropped silently ran a module without what its manifest says, and a key inside a block still failed the whole catalogue. Judge a key by what it is about, and narrow the listing to one machine. --- cmd/mesh-controller/modules.go | 24 ++++++- cmd/mesh-controller/pending.go | 3 + cmd/mesh-controller/plan.go | 5 +- cmd/mesh-controller/seatverbs.go | 6 ++ .../settings_effective_test.go | 38 ++++++++++ cmd/mesh-controller/unknown_fields.go | 64 +++++++++++++++++ cmd/mesh-controller/unknown_fields_test.go | 50 +++++++++++++ internal/catalogue/data.go | 2 +- internal/catalogue/declaration.go | 4 ++ internal/catalogue/manifest.go | 25 ++++--- internal/catalogue/setting_defaults.go | 48 +++++++++---- internal/catalogue/setting_defaults_test.go | 72 ++++++++++++++----- internal/catalogue/state.go | 2 +- internal/catalogue/unknown_field.go | 58 +++++++++++++++ internal/catalogue/verbs.go | 2 +- internal/inventory/catalogue.go | 10 +-- internal/inventory/unknown_field_test.go | 34 +++++++++ 17 files changed, 395 insertions(+), 52 deletions(-) create mode 100644 cmd/mesh-controller/unknown_fields.go create mode 100644 cmd/mesh-controller/unknown_fields_test.go create mode 100644 internal/catalogue/unknown_field.go create mode 100644 internal/inventory/unknown_field_test.go diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 0c9070ca..9a8e3c8b 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -547,6 +547,12 @@ func settingsCommand(ctx context.Context, args []string) error { if len(positionals) == 1 { only = positionals[0] } + if *node != "" { + // A machine the mesh does not know is refused, never answered with an empty listing. + if _, err := inv.NodeByName(ctx, *node); err != nil { + return err + } + } entries, err := inv.Catalogued(ctx) if err != nil { return err @@ -557,6 +563,10 @@ func settingsCommand(ctx context.Context, args []string) error { if len(m.Settings) == 0 || (only != "" && m.Module != only) { continue } + if *node != "" && !containsString(e.On, *node) { + // Asked for one machine: a module not on it has no value there to say. + continue + } p := preferencesOf{Manifest: m, On: map[string][]catalogue.SettingSource{}} for _, n := range e.On { if *node != "" && n != *node { @@ -572,7 +582,11 @@ func settingsCommand(ctx context.Context, args []string) error { listed = append(listed, p) } if only != "" && len(listed) == 0 { - fmt.Printf("%s declares no preferences\n", only) + fmt.Printf("%s declares no preferences%s\n", only, onNode(*node)) + return nil + } + if len(listed) == 0 && *node != "" { + fmt.Printf("no module on %s declares a preference\n", *node) return nil } fmt.Print(describePreferences(listed)) @@ -610,6 +624,14 @@ func describeEffective(module, where string, values []catalogue.SettingSource) s return b.String() } +// onNode is ` on ` for one machine, nothing for the whole mesh. +func onNode(node string) string { + if node == "" { + return "" + } + return " on " + node +} + // preferencesOf is one module's preferences and its value on each machine it is assigned to. type preferencesOf struct { Manifest catalogue.Manifest diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index 3627eb68..7743f165 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -478,6 +478,9 @@ func settlingPending(ctx context.Context, open *stores) { for _, line := range settlePending(ctx, open, time.Now()) { fmt.Println(line) } + for _, line := range raiseUnknownFields(ctx, open.inventory) { + fmt.Println(line) + } select { case <-ctx.Done(): return diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 26dddca4..55586d33 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -515,9 +515,8 @@ func sortedKeysOf(m map[string]string) []string { // 0163, rule 6), one line each: the machine is told everything else, and is told it was left out. func reportLeftOut(node string, declared sendable) { for _, m := range declared.LeftOut { - fmt.Printf("%s: %s left out — a setting stored for it cannot compose with its definition; "+ - "what the machine holds for it is kept and its containers are untouched. %s\n", - node, m, declared.leftOutWhy[m]) + fmt.Printf("%s: %s left out — what the machine holds for it is kept and its containers are "+ + "untouched. %s\n", node, m, declared.leftOutWhy[m]) } // And whom it serves nothing, because their identity overflows what the provision keeps (ADR // 0225): the machine is sent everything else, and the consumer is named. diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 0a233da3..e134cf3e 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -764,6 +764,12 @@ func (a *verbArguments) commandLine() ([]string, error) { // Every module's preferences, their defaults and each machine's value (novox/hq ADR 0262): // the one interface for them, so no module builds a settings tool of its own. Asked for by // name, or by naming no module, since a layer is always some module's. + if str("module") == "" && str("values") != "" { + return nil, errors.New("settings: a module is needed to set values; name it with module") + } + if str("module") == "" && on("clear") { + return nil, errors.New("settings: a module is needed to clear a layer; name it with module") + } if list := str("list"); list != "" || str("module") == "" { if list != "" && list != "preferences" { return nil, fmt.Errorf("settings lists %q only; %q is not a listing", "preferences", list) diff --git a/cmd/mesh-controller/settings_effective_test.go b/cmd/mesh-controller/settings_effective_test.go index 2f2ddf1a..f248e2b4 100644 --- a/cmd/mesh-controller/settings_effective_test.go +++ b/cmd/mesh-controller/settings_effective_test.go @@ -37,6 +37,14 @@ func TestSettingsListPreferences(t *testing.T) { t.Errorf("%v: %v %v, want %s", c.args, argv, err, c.want) } } + for args, want := range map[string]map[string]any{ + "a module is needed to set values": {"values": `{"width": 300}`}, + "a module is needed to clear a layer": {"clear": "true"}, + } { + if _, err := argvFor("settings", want); err == nil || !strings.Contains(err.Error(), args) { + t.Errorf("%v: %v, want %q", want, err, args) + } + } if _, err := argvFor("settings", map[string]any{"list": "everything"}); err == nil { t.Error("a listing other than preferences was taken") } @@ -69,3 +77,33 @@ func TestPreferencesSayEachMachinesValueAndItsSource(t *testing.T) { t.Fatal("an empty listing") } } + +// The listing over the real stores: each machine's value with its source; a machine names only the +// modules on it; a machine the mesh does not know is refused (novox/hq ADR 0262). +func TestPreferencesListedFromTheStores(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Settings: map[string]catalogue.SettingDeclaration{ + "font-size": {Kind: catalogue.KindPreference, Default: float64(10), Why: "readable at 100 DPI"}, + }, + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", + "content": "font = ${setting:font-size}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "laptop", "notes", map[string]any{"font-size": float64(16)}); err != nil { + t.Fatal(err) + } + all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) }) + if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") || + !strings.Contains(all, "font-size, default 10: readable at 100 DPI") { + t.Fatalf("the listing:\n%s", all) + } + if got := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences", "--node", "anchor"}) }); got != "no module on anchor declares a preference\n" { + t.Fatalf("a machine without the module:\n%s", got) + } + if err := settingsCommand(ctx, []string{"preferences", "--node", "nowhere"}); err == nil { + t.Fatal("a machine the mesh does not know was answered") + } +} diff --git a/cmd/mesh-controller/unknown_fields.go b/cmd/mesh-controller/unknown_fields.go new file mode 100644 index 00000000..020da9b1 --- /dev/null +++ b/cmd/mesh-controller/unknown_fields.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "fmt" + "sort" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A stored manifest with a key this controller does not know (novox/hq ADR 0262). The module is left out +// of every machine's declaration by name; this is the loud half: a condition per module until the +// controller is updated, or the module is registered again in a shape this controller reads. + +const ( + sourceUnknownFields = "the catalogue" + kindUnknownField = "unknown-field" +) + +// unknownFieldObservations is one condition for each module of the catalogue whose stored manifest has +// a key this controller does not know. +func unknownFieldObservations(known map[string]catalogue.Manifest) []conditions.Observation { + names := make([]string, 0, len(known)) + for name, m := range known { + if m.UnknownField() != "" { + names = append(names, name) + } + } + sort.Strings(names) + var out []conditions.Observation + for _, name := range names { + m := known[name] + out = append(out, conditions.Observation{ + Scope: conditions.ScopeMesh, ID: name, Kind: kindUnknownField, Severity: conditions.Warning, + Resolver: conditions.ResolverOperator, Source: sourceUnknownFields, + Summary: catalogue.UnknownFieldReason(m), + Said: m.UnknownField(), + Headline: name + " is left out until the controller is updated", + Explanation: name + " uses a field this controller does not know, so it is left out of every machine it is on, and nothing of it changes there until the controller is updated.", + Needs: "update the controller, or register " + name + " again at a version this controller knows.", + Resolved: "the controller reads " + name + " again", + }) + } + return out +} + +// raiseUnknownFields raises those conditions and clears the ones no longer true, on the controller's +// tick. A catalogue that could not be read raises and clears nothing: "none" is not said for "could not +// tell" (ADR 0227 rule 4). +func raiseUnknownFields(ctx context.Context, inv *inventory.Inventory) []string { + if conditionsFrom == nil { + return nil + } + known, err := inv.Catalogue(ctx) + if err != nil { + return []string{fmt.Sprintf("the catalogue could not be read to say which modules it cannot read: %v", err)} + } + if err := conditionsFrom.Reconcile(ctx, sourceUnknownFields, unknownFieldObservations(known)); err != nil { + return []string{fmt.Sprintf("the modules with a field this controller does not know could not be kept as conditions: %v", err)} + } + return nil +} diff --git a/cmd/mesh-controller/unknown_fields_test.go b/cmd/mesh-controller/unknown_fields_test.go new file mode 100644 index 00000000..9776331c --- /dev/null +++ b/cmd/mesh-controller/unknown_fields_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" +) + +// A module whose stored manifest has a key this controller does not know is a condition, in plain +// words, until it is read again; every other module raises nothing (novox/hq ADR 0262). +func TestAModuleWithAnUnknownFieldIsACondition(t *testing.T) { + var later, now catalogue.Manifest + if err := json.Unmarshal([]byte(`{"module": "dunst", "version": "2", "settings": {}, "a-field-from-later": 1}`), &later); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(`{"module": "xorg", "version": "1"}`), &now); err != nil { + t.Fatal(err) + } + observed := unknownFieldObservations(map[string]catalogue.Manifest{"dunst": later, "xorg": now}) + if len(observed) != 1 || observed[0].ID != "dunst" || observed[0].Kind != kindUnknownField { + t.Fatalf("observed: %+v", observed) + } + o := observed[0] + if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, + Resolved: o.Resolved, Needs: o.Needs}); !ok { + t.Fatalf("not plain: %s", why) + } + + k, _ := withConditionsInMemory(t) + if err := k.Reconcile(t.Context(), sourceUnknownFields, observed); err != nil { + t.Fatal(err) + } + if _, open, _ := k.Get(t.Context(), o.Key()); !open { + t.Fatal("not raised") + } + if err := k.Reconcile(t.Context(), sourceUnknownFields, unknownFieldObservations(map[string]catalogue.Manifest{"xorg": now})); err != nil { + t.Fatal(err) + } + still, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, c := range still { + if c.Key == o.Key() { + t.Fatalf("not cleared once read again: %+v", c) + } + } +} diff --git a/internal/catalogue/data.go b/internal/catalogue/data.go index e44ac75a..5a43203c 100644 --- a/internal/catalogue/data.go +++ b/internal/catalogue/data.go @@ -159,7 +159,7 @@ func (b *DataBackup) UnmarshalJSON(raw []byte) error { dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { - return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", err) + return fmt.Errorf("a data item's backup is \"copy\", \"none\" or {dump, into}: %w", typedUnknown(err)) } *b = DataBackup{Dump: full.Dump, Into: full.Into} return nil diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 2cc13ba3..dac7a6f7 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -318,6 +318,10 @@ func (e *NotMadeError) Error() string { func (r Resolution) LeftOut(settings SettingsBy, adopted bool) map[string]string { out := map[string]string{} for _, m := range r.Modules { + if why := UnknownFieldReason(m); why != "" { + out[m.Module] = why + continue + } if err := JudgeSettings(m, settings[m.Module], adopted); err != nil { out[m.Module] = err.Error() } diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index e8cdea05..c0c0fe8e 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -197,7 +197,7 @@ func (i *OfferIdentity) UnmarshalJSON(raw []byte) error { dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { - return fmt.Errorf("an offer's identity is false or {max, in}: %w", err) + return fmt.Errorf("an offer's identity is false or {max, in}: %w", typedUnknown(err)) } *i = OfferIdentity{Max: full.Max, In: full.In} return nil @@ -316,7 +316,7 @@ func (o *Offer) UnmarshalJSON(raw []byte) error { dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity, "+ - "keeps-consumer-data}: %w", err) + "keeps-consumer-data}: %w", typedUnknown(err)) } o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity o.KeepsConsumerData = full.Keeps @@ -465,9 +465,10 @@ type Manifest struct { // stays refused by name until a layer sets it. The mesh's layer, then the node's, override it. Settings map[string]SettingDeclaration `json:"settings,omitempty"` - // unknown is the first key this manifest has that this controller does not know, when it was read - // leniently (novox/hq ADR 0262): a stored manifest written for a newer controller. ParseManifest - // refuses it; reading the stored catalogue keeps the rest of the manifest. + // unknown is the first key this manifest has that this controller does not know, at any depth, when + // it was read from the store (novox/hq ADR 0262): a manifest a newer controller registered. + // ParseManifest refuses it; the store's catalogue still loads, and the module is left out of every + // machine's declaration by name until the controller is updated (LeftOut). unknown string // Data is every kind of data this module keeps — its own, by directory, and what it keeps for @@ -1279,13 +1280,19 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { var fields manifestFields unknown := "" if err := decoder.Decode(&fields); err != nil { - if !strings.HasPrefix(err.Error(), "json: unknown field ") { + if asUnknownField(err) == nil { return err } + // Read without it where the key is at the top; where it is inside a block, the block's own + // decoder refuses it again, and the manifest keeps its name and version alone. Either way the + // module is left out of every declaration by name (LeftOut), so nothing runs on a part-read + // manifest. unknown = err.Error() fields = manifestFields{} - if err := json.Unmarshal(rest, &fields); err != nil { - return err + if json.Unmarshal(rest, &fields) != nil { + fields = manifestFields{} + _ = json.Unmarshal(keys["module"], &fields.Module) + _ = json.Unmarshal(keys["version"], &fields.Version) } } *m = Manifest(fields) @@ -2479,7 +2486,7 @@ func (o *OwnSecrets) UnmarshalJSON(raw []byte) error { dec := json.NewDecoder(bytes.NewReader(body)) dec.DisallowUnknownFields() if err := dec.Decode(&long); err != nil { - return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, err) + return fmt.Errorf("own-secrets.%s: a path, or {\"path\", \"taken\", \"issued-by\"}: %w", name, typedUnknown(err)) } out[name] = OwnSecret{Path: long.Path, Taken: long.Taken, IssuedBy: long.IssuedBy} } diff --git a/internal/catalogue/setting_defaults.go b/internal/catalogue/setting_defaults.go index d28912d4..2297c17b 100644 --- a/internal/catalogue/setting_defaults.go +++ b/internal/catalogue/setting_defaults.go @@ -46,32 +46,54 @@ var meshWords = map[string]bool{ PlacesSetting: true, AccessesSetting: true, NetworksSetting: true, } -// operatorsOwn are words that, as a whole word of a key's name, say its value is the operator's and -// never a preference: a default for one would be the literal ADR 0112 removed from definitions. Whole -// words, split at dashes, underscores and dots, so `max-tokens`, `show-hostname` and `mailbox-size` are -// preferences and `api-token`, `host` and `mail-domain` are not. +// operatorsOwn are the words that, as what a key's name is about, say its value is the operator's and +// never a preference: a default for one would be the literal ADR 0112 removed from definitions. var operatorsOwn = map[string]bool{ - "domain": true, "host": true, "fqdn": true, "zone": true, "realm": true, "tenant": true, - "issuer": true, "url": true, "uri": true, "webhook": true, "origin": true, "dsn": true, "ip": true, + "domain": true, "host": true, "hostname": true, "servername": true, "fqdn": true, "zone": true, + "realm": true, "tenant": true, "site": true, "timezone": true, + "issuer": true, "url": true, "uri": true, "webhook": true, "origin": true, "dsn": true, + "ip": true, "ipv4": true, "ipv6": true, "email": true, "mail": true, "phone": true, "address": true, "identity": true, "login": true, "user": true, "username": true, "account": true, "owner": true, "uid": true, "gid": true, "puid": true, "pgid": true, "password": true, "pass": true, "passwd": true, "passphrase": true, "secret": true, "token": true, - "key": true, "credential": true, + "key": true, "apikey": true, "bearer": true, "cert": true, "credential": true, } -// operatorsWord is the word of a key's name that says its value is the operator's, or "". +// operatorsCompounds are names of two words that are the operator's though neither word alone says so +// at the end of a key: a client's identifier, and a name the world knows a site or server by. +var operatorsCompounds = map[string]bool{ + "client-id": true, "site-name": true, "server-name": true, "public-name": true, "smtp-relay": true, +} + +// aboutAnAmount are first words that make a key about how many or whether, never about whom: +// `max-tokens` is a number, `show-hostname` a switch. +var aboutAnAmount = map[string]bool{ + "max": true, "min": true, "num": true, "count": true, "show": true, "hide": true, "enable": true, + "disable": true, "use": true, "allow": true, +} + +// operatorsWord is what in a key's name says its value is the operator's, or "". A key is about its +// last word — `url-timeout` is a timeout, `user-agent` an agent, `mail-domain` a domain — or its last two +// as one of operatorsCompounds. A plural is read as its singular. func operatorsWord(key string) string { words := strings.FieldsFunc(key, func(r rune) bool { return r == '-' || r == '_' || r == '.' }) + if len(words) == 0 || (len(words) > 1 && aboutAnAmount[words[0]]) { + return "" + } for i, w := range words { - if operatorsOwn[w] { - return w + if !operatorsOwn[w] && strings.HasSuffix(w, "s") && operatorsOwn[strings.TrimSuffix(w, "s")] { + words[i] = strings.TrimSuffix(w, "s") } - // An identifier a client is known by: `client-id`, `oauth-client-id`. - if w == "client" && i+1 < len(words) && words[i+1] == "id" { - return "client-id" + } + if n := len(words); n > 1 { + if pair := words[n-2] + "-" + words[n-1]; operatorsCompounds[pair] { + return pair } } + if last := words[len(words)-1]; operatorsOwn[last] { + return last + } return "" } diff --git a/internal/catalogue/setting_defaults_test.go b/internal/catalogue/setting_defaults_test.go index fe68c915..cb3be6d8 100644 --- a/internal/catalogue/setting_defaults_test.go +++ b/internal/catalogue/setting_defaults_test.go @@ -218,18 +218,24 @@ func TestANodeCalledDefaultIsANodesLayer(t *testing.T) { } } -// The operator's own value is told by a whole word of the key's name, not by a part of one. -func TestAKeyIsTheOperatorsByAWholeWord(t *testing.T) { +// The operator's own value is told by what a key's name is about: its last word, or its last two as +// a known compound; a plural as its singular; and never a number or a switch. +func TestAKeyIsTheOperatorsByWhatItIsAbout(t *testing.T) { for _, key := range []string{"max-tokens", "show-hostname", "ghost-opacity", "users-per-page", "mailbox-size", - "font-size", "width", "keyboard-delay", "ipv6-preferred", "client-width"} { + "font-size", "width", "keyboard-delay", "ipv6-preferred", "client-width", "user-agent", "url-timeout", + "site-title", "cert-renewal-days", "name", "font-name"} { if w := operatorsWord(key); w != "" { t.Errorf("%s read as the operator's (%s)", key, w) } } - for key, word := range map[string]string{"mail-domain": "mail", "site-domain": "domain", "api-key": "key", "admin-password": "password", + for key, word := range map[string]string{"mail-domain": "domain", "api-key": "key", "admin-password": "password", "oauth-client-id": "client-id", "db-dsn": "dsn", "public-ip": "ip", "puid": "puid", "dns-zone": "zone", "webhook": "webhook", "notify_phone": "phone", "cors.origin": "origin", "smtp-pass": "pass", - "backup-passphrase": "passphrase", "data-owner": "owner", "fqdn": "fqdn", "tenant": "tenant", "host": "host"} { + "backup-passphrase": "passphrase", "data-owner": "owner", "fqdn": "fqdn", "tenant": "tenant", "host": "host", + "allowed-hosts": "host", "admin-emails": "email", "tokens": "token", "hostname": "hostname", + "apikey": "apikey", "servername": "servername", "tls-cert": "cert", "site": "site", "timezone": "timezone", + "bearer": "bearer", "bind-ipv4": "ipv4", "listen-ipv6": "ipv6", "site-name": "site-name", + "server-name": "server-name", "public-name": "public-name", "smtp-relay": "smtp-relay"} { if w := operatorsWord(key); w != word { t.Errorf("%s: read %q, want %q", key, w, word) } @@ -279,22 +285,50 @@ func TestADefaultFillsAContributionAndAServedFactButNotALiteral(t *testing.T) { } } -// A stored manifest with a key this controller does not know is read without it, and said; the module -// check still refuses it. -func TestAStoredManifestWithAnUnknownKeyIsReadAndRegistrationRefusesIt(t *testing.T) { - raw := []byte(`{"module": "later", "version": "1", "tools": ["later_x"], "a-field-from-later": {"x": 1}}`) - var m Manifest - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatalf("a stored manifest with an unknown key was not read: %v", err) - } - if m.Module != "later" || len(m.Tools) != 1 || !strings.Contains(m.UnknownField(), `"a-field-from-later"`) { - t.Fatalf("read as %+v, unknown %q", m, m.UnknownField()) - } - if _, err := ParseManifest(raw); err == nil || !strings.Contains(err.Error(), `unknown field "a-field-from-later"`) { - t.Fatalf("registration took it: %v", err) +// A stored manifest with a key this controller does not know, at the top or inside any block, is read +// and its module is left out of every declaration by name; the rest of the catalogue is read; the module +// check refuses it. One case per block whose decoder wraps the decoder's words in its own. +func TestAStoredManifestWithAnUnknownKeyIsLeftOutAndRegistrationRefusesIt(t *testing.T) { + for where, raw := range map[string]string{ + "the top": `{"module": "later", "version": "2", "a-field-from-later": {"x": 1}}`, + "a state": `{"module": "later", "version": "2", "state": [{"name": "s", "a-field-from-later": 1}]}`, + "a provided name": `{"module": "later", "version": "2", "provides": [{"name": "p", "a-field-from-later": 1}]}`, + "an offer's identity": `{"module": "later", "version": "2", "provides": [{"name": "p", "identity": {"in": "x", "a-field-from-later": 1}}]}`, + "a backup": `{"module": "later", "version": "2", "data": {"own": [{"id": "d", "path": "${dir:d}", "class": "valuable", "backup": {"dump": "x", "into": "y", "a-field-from-later": 1}}]}}`, + "an own secret": `{"module": "later", "version": "2", "own-secrets": {"s": {"path": "/x", "a-field-from-later": 1}}}`, + "a seat's verb": `{"module": "later", "version": "2", "seats": [{"name": "later-seat", "serves": [{"name": "v", "a-field-from-later": 1}]}]}`, + } { + var m Manifest + if err := json.Unmarshal([]byte(raw), &m); err != nil { + t.Errorf("%s: the stored manifest was not read: %v", where, err) + continue + } + if m.Module != "later" || m.Version != "2" || !strings.Contains(m.UnknownField(), `"a-field-from-later"`) { + t.Errorf("%s: read as %q %q, unknown %q", where, m.Module, m.Version, m.UnknownField()) + continue + } + if where != "the top" && !strings.Contains(m.UnknownField(), ": json: unknown field") { + t.Errorf("%s: the block's decoder did not say it: %q", where, m.UnknownField()) + } + left := Resolution{Node: "laptop", Modules: []Manifest{m, notifier()}}.LeftOut(nil, false) + if why := left["later"]; !strings.Contains(why, "uses a field this controller does not know") || + !strings.Contains(why, "a-field-from-later") { + t.Errorf("%s: not left out by name: %v", where, left) + } + if _, notifierLeft := left["notifier"]; notifierLeft { + t.Errorf("%s: another module was left out with it: %v", where, left) + } + if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "a-field-from-later") { + t.Errorf("%s: the module check took it: %v", where, err) + } } var known Manifest - if err := json.Unmarshal([]byte(`{"module": "now"}`), &known); err != nil || known.UnknownField() != "" { + if err := json.Unmarshal([]byte(`{"module": "now", "state": [{"name": "s"}]}`), &known); err != nil || known.UnknownField() != "" { t.Fatalf("a known manifest: %v %q", err, known.UnknownField()) } + // A malformed manifest is still refused: only an unknown key is read past. + var bad Manifest + if err := json.Unmarshal([]byte(`{"module": "bad", "state": [{"name": 3}]}`), &bad); err == nil { + t.Fatal("a malformed stored manifest was read") + } } diff --git a/internal/catalogue/state.go b/internal/catalogue/state.go index d21297cd..1944b778 100644 --- a/internal/catalogue/state.go +++ b/internal/catalogue/state.go @@ -49,7 +49,7 @@ func (s *StateDeclaration) UnmarshalJSON(raw []byte) error { dec := json.NewDecoder(bytes.NewReader(trimmed)) dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { - return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine}: %w", err) + return fmt.Errorf("a state is either a name or {name, history, ttl-seconds, per-machine}: %w", typedUnknown(err)) } *s = StateDeclaration(full) return nil diff --git a/internal/catalogue/unknown_field.go b/internal/catalogue/unknown_field.go new file mode 100644 index 00000000..ed2d3319 --- /dev/null +++ b/internal/catalogue/unknown_field.go @@ -0,0 +1,58 @@ +package catalogue + +import ( + "errors" + "regexp" +) + +// A key this controller does not know, in a manifest (novox/hq ADR 0262). +// +// Registration refuses it, as it always has. A manifest the store already holds was registered by a newer +// controller, and is read by this one after a rollback: refusing it there failed the whole catalogue, and +// with it every plan and every send. Dropping the key silently would be worse — a module running without +// something its manifest says. So the manifest is read, the module is left out of every machine's +// declaration by name, and the controller raises a condition until it is updated. + +// UnknownFieldError is a key a manifest has that this controller does not know, wherever it is: at the +// top of the manifest or inside a block (a state, an offer, a data item's backup, an own secret, a verb). +// Typed, because the blocks' decoders wrap the decoder's words in their own. +type UnknownFieldError struct { + Field string + err error +} + +func (e *UnknownFieldError) Error() string { return e.err.Error() } +func (e *UnknownFieldError) Unwrap() error { return e.err } + +// unknownFieldText is how the JSON decoder words a key its target does not have. +var unknownFieldText = regexp.MustCompile(`^json: unknown field "([^"]*)"$`) + +// typedUnknown is err as an UnknownFieldError when it is the decoder refusing an unknown key, else err. +func typedUnknown(err error) error { + if err == nil { + return nil + } + if m := unknownFieldText.FindStringSubmatch(err.Error()); m != nil { + return &UnknownFieldError{Field: m[1], err: err} + } + return err +} + +// asUnknownField is the unknown key err is about, at any depth, or nil. +func asUnknownField(err error) *UnknownFieldError { + var u *UnknownFieldError + if errors.As(typedUnknown(err), &u) { + return u + } + return nil +} + +// UnknownFieldReason is why a module whose stored manifest has a key this controller does not know is +// left out of a machine's declaration, or "" when it has none. +func UnknownFieldReason(m Manifest) string { + if m.unknown == "" { + return "" + } + return m.Module + " uses a field this controller does not know (" + m.unknown + "); it is left out " + + "until the controller is updated (novox/hq ADR 0262)" +} diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index e326b231..de1faf8a 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -56,7 +56,7 @@ func (v *Verb) UnmarshalJSON(raw []byte) error { decoder := json.NewDecoder(bytes.NewReader(trimmed)) decoder.DisallowUnknownFields() if err := decoder.Decode(&p); err != nil { - return fmt.Errorf("a served verb is a name or {name, description, input, output}: %w", err) + return fmt.Errorf("a served verb is a name or {name, description, input, output}: %w", typedUnknown(err)) } if p.Name == "" { return fmt.Errorf("a served verb has no name: %s", trimmed) diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index a82d4368..3112962c 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -20,8 +20,9 @@ import ( var noted sync.Map // noteUnknown says, once per module and key, that a stored manifest has a key this controller does not -// know and that it was read without it (novox/hq ADR 0262). A manifest registered under a newer -// controller is read by an older one after a rollback; refusing it here failed the whole catalogue. +// know (novox/hq ADR 0262). A manifest registered under a newer controller is read by an older one after +// a rollback; refusing it here failed the whole catalogue. The module is left out of every machine by name +// (catalogue.LeftOut), and the controller's tick raises a condition for it. func noteUnknown(m catalogue.Manifest) { u := m.UnknownField() if u == "" { @@ -30,8 +31,9 @@ func noteUnknown(m catalogue.Manifest) { if _, said := noted.LoadOrStore(m.Module+"\x00"+u, true); said { return } - log.Printf("the stored manifest of %s has a key this controller does not know (%s); read without it — "+ - "a newer controller registered it (novox/hq ADR 0262)", m.Module, u) + log.Printf("the stored manifest of %s has a key this controller does not know (%s): a newer controller "+ + "registered it, and %s is left out of every machine until this controller is updated (novox/hq ADR 0262)", + m.Module, u, m.Module) } // ErrNoSuchModule is what the mesh says about a module it has never been told about. diff --git a/internal/inventory/unknown_field_test.go b/internal/inventory/unknown_field_test.go new file mode 100644 index 00000000..fd163f86 --- /dev/null +++ b/internal/inventory/unknown_field_test.go @@ -0,0 +1,34 @@ +package inventory + +import ( + "strings" + "testing" +) + +// A manifest a newer controller registered, with a key this one does not know, does not stop the +// catalogue from loading: it is read, marked, and every other module is read as before (novox/hq ADR 0262). +func TestTheCatalogueLoadsAroundAManifestWithAnUnknownKey(t *testing.T) { + inv := fresh(t) + if err := inv.RegisterModule(t.Context(), manifest("thing", nil, nil), Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.store.Pool().Exec(t.Context(), + `insert into module (name, manifest) values ($1, $2)`, "later", + `{"module": "later", "version": "2", "state": [{"name": "s", "a-field-from-later": 1}]}`); err != nil { + t.Fatal(err) + } + known, err := inv.Catalogue(t.Context()) + if err != nil { + t.Fatalf("one manifest with an unknown key failed the whole catalogue: %v", err) + } + if known["thing"].Module != "thing" || known["thing"].UnknownField() != "" { + t.Fatalf("the other module: %+v", known["thing"]) + } + if !strings.Contains(known["later"].UnknownField(), "a-field-from-later") { + t.Fatalf("the newer manifest is not marked: %q", known["later"].UnknownField()) + } + entries, err := inv.Catalogued(t.Context()) + if err != nil || len(entries) < 2 { + t.Fatalf("the listing: %v %d", err, len(entries)) + } +}