diff --git a/modules/mesh-conditions/cmd/mesh-conditions/homes.go b/modules/mesh-conditions/cmd/mesh-conditions/homes.go index 86072e9f..96671fb7 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/homes.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/homes.go @@ -93,6 +93,15 @@ var Homes = map[string]Home{ "asks-undelivered": {Source: "asks", Controller: true, Observer: messenger}, // The seat mesh-artifact-sweep (L7): an image a want names is missing from the store, artifact-store-tools' finding. "image-missing": {Source: "S25", Controller: true, Observer: artifactStore}, + // mesh-config's own (design 32): a rename it refused, a rename window or a retirement held seven days by a node, a + // retirement asked twice and not answered. + "rename-refused": {Source: "config", Observer: configModule}, + "rename-window-held": {Source: "config", Observer: configModule}, + "retirement-held": {Source: "config", Observer: configModule}, + "retiring-unanswered": {Source: "config", Observer: configModule}, + // A reader whose owner does not answer its first view past two minutes (ADR 0615 §6): written by each reader in its own + // findings (mesh-config, mesh-vault), so the kind has no one observer. + "first-view-unanswered": {Source: "views", Why: "every reader of the indexer's or the assigner's state writes its own"}, // The kinds that end with the controller and are carried until its lease write (clauses 4, 9). "controller-deaf": {Source: "S4", Controller: true, Carried: true}, "lease-lost": {Source: "S12", Controller: true, Carried: true}, @@ -155,6 +164,8 @@ const ( provisioner = "mesh-provisioner" messenger = "messenger" artifactStore = "artifact-store-tools" + configModule = "mesh-config" + vaultModule = "mesh-vault" ) // homeOf is the home of a condition's kind, and whether the table has one. diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index e72cd426..ec3719b3 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -40,7 +40,7 @@ const HistoryKeptFor = 90 * 24 * time.Hour // table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder, // which observes S25, image-missing). var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule, - "mesh-provisioner", "messenger", artifactStore} + "mesh-provisioner", "messenger", artifactStore, configModule, vaultModule} // The other states it reads, as `.`. const ( diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index c2d6b3bb..b84caac2 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -294,7 +294,7 @@ func TestEveryControllerKindHasItsHome(t *testing.T) { t.Errorf("the ended probe's kind %s is %+v", kind, h) } } - if len(Homes) != 84 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled + if len(Homes) != 89 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled, mesh-config's four (design 32) and the readers' unanswered first view t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried) } } diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index ce6ecbae..793b7cdb 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -21,6 +21,8 @@ "messenger.findings", "mesh-provisioner.findings", "artifact-store-tools.findings", + "mesh-config.findings", + "mesh-vault.findings", "mesh-healers.attempts", "mesh-delivery-assigner.wants", "node-engine.have", diff --git a/modules/mesh-config/README.md b/modules/mesh-config/README.md index e5fffe1a..db8e61a7 100644 --- a/modules/mesh-config/README.md +++ b/modules/mesh-config/README.md @@ -56,6 +56,26 @@ value as a set does, keeps who approved it beside the value (`approved`), and as warrant in its hand-act log. A Decline, an expiry or the router's refusal sets nothing. `action` `proposals` lists them, newest first; with `proposal`, one read whole, with whether the router's ask is the one composed from it. +## A value renamed, a value no longer read (design 32, novox/hq ADR 0609) + +- A declaration's `renamed-from` moves the value set under the old name to the new one on each node, with its trust (it + moves whole, and a trusted new name over a value set without the Approve is refused), saying `renamed` + (module, from, to, at) once. A **rename window** opens per node and keeps the old name answering the new value in + `mesh-config_resolved` until that node's node-engine runs a build that reads the new name: judged at start, on the + status view's `delivered`, and for a returning node-engine on the node-engine seat's `manifest-reads-changed` after + `manifest-reads` answers (one that does not serve it states nothing, and its window stays open). State `renames`. +- The seat verb `renames` answers the open renames and the nodes holding each open (the checker asks it); the tools + `config_renames` (all, with the closed windows), `config_rename_again` (move by hand after a refusal) and + `config_rename_close` (drop a node by hand, a hand act). +- On the indexer's `manifest-accepted` it asks the indexer's `indexer_config_readers`; an indexer that does not serve + it answers "not known" and nothing retires. A value (or secret) nobody reads is **retiring** once no node runs a + build that reads it: `retiring` is said and the operator is asked once; the value is removed only on the Approve + (a secret through the seat mesh-vault's `retire`, with a hand act naming node, module, name and ask), `retired` + said per node. State `retiring`; the tool `config_retiring` lists, asks again and cancels. +- Findings (state `findings`, read by the condition keeper): `rename-refused`, `rename-window-held` and + `retirement-held` (seven days, naming the node), `retiring-unanswered` (asked twice). +- The events `renamed`, `retiring` and `retired` are carried by the stream `config-events`, kept seven days. + ## Files it does not know Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index 1db9388f..a745180e 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -93,6 +93,10 @@ type Config struct { Asks AskBus // Forge is the forge as pulls are asked of it (pulls.go); nil, a merge or a close is heard from its events alone. Forge PullForge + // Delivery is what the delivery parts answer (renames.go, retiring.go); nil, no window closes and nothing retires. + Delivery Delivery + // Vault is the seat mesh-vault, for the retiring of a secret; nil, a secret does not retire. + Vault Vault // RouterWithin and RouterEvery bound how long a proposal waits for the router's word on its ask, and how often // it looks; zero is RouterAnswersWithin and RouterAnswersEvery. RouterWithin, RouterEvery time.Duration diff --git a/modules/mesh-config/cmd/mesh-config/config_test.go b/modules/mesh-config/cmd/mesh-config/config_test.go index 7201384a..5b4334d0 100644 --- a/modules/mesh-config/cmd/mesh-config/config_test.go +++ b/modules/mesh-config/cmd/mesh-config/config_test.go @@ -20,6 +20,7 @@ type memStore struct { states map[string]map[string][]byte logs map[string][]LogEntry events []Change + said []saidEvent handActs []string // failHandAct makes the hand-act log refuse. failHandAct bool @@ -89,15 +90,38 @@ func (m *memStore) Read(log string) ([]LogEntry, error) { } func (m *memStore) Emit(event string, body any) error { - if event != ConfigChanged { - return fmt.Errorf("mesh-config emits no %s", event) - } m.mu.Lock() defer m.mu.Unlock() - m.events = append(m.events, body.(Change)) + switch event { + case ConfigChanged: + m.events = append(m.events, body.(Change)) + case EventRenamed, EventRetiring, EventRetired: + m.said = append(m.said, saidEvent{event, body}) + default: + return fmt.Errorf("mesh-config emits no %s", event) + } return nil } +// saidEvent is an event of the family config.events as it was emitted. +type saidEvent struct { + Event string + Body any +} + +// saidOf is the events of one name emitted so far. +func (m *memStore) saidOf(event string) []any { + m.mu.Lock() + defer m.mu.Unlock() + var out []any + for _, e := range m.said { + if e.Event == event { + out = append(out, e.Body) + } + } + return out +} + func (m *memStore) HandAct(what, why, cause string) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/modules/mesh-config/cmd/mesh-config/declared.go b/modules/mesh-config/cmd/mesh-config/declared.go index 205c215d..b4c0d340 100644 --- a/modules/mesh-config/cmd/mesh-config/declared.go +++ b/modules/mesh-config/cmd/mesh-config/declared.go @@ -41,6 +41,9 @@ type Declaration struct { Predefined string `json:"predefined,omitempty"` Required bool `json:"required,omitempty"` Runtime bool `json:"runtime,omitempty"` + // RenamedFrom is the key's old name when it was renamed (novox/hq ADR 0609 §4, design 32): on the merge the value + // set under the old name moves to this key (renames.go). + RenamedFrom string `json:"renamed-from,omitempty"` } // HasDefault is whether the declaration gives a default: present and not null. diff --git a/modules/mesh-config/cmd/mesh-config/delivery.go b/modules/mesh-config/cmd/mesh-config/delivery.go new file mode 100644 index 00000000..a2c3a5cd --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/delivery.go @@ -0,0 +1,132 @@ +package main + +// What mesh-config asks of the delivery parts and the vault, through their tools and seat verbs on the bus: never a +// bucket of theirs (novox/hq ADR 0596). Each key is the address a module's `invokes` names (mesh-sdk calladdress). + +import ( + "encoding/json" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// The addresses asked. +const ( + // HavesAsk is the status view's read of the node-engines' haves, narrowed to one node and module. + HavesAsk = "mesh-delivery-view.node_haves" + // ManifestReadsAsk is the node-engine seat's verb saying what manifest forms a node-engine reads (ADR 0618 §3), asked + // of one node by the address's `@`. + ManifestReadsAsk = "seat:node-engine.manifest-reads" + // ConfigReadersAsk is the indexer's own tool saying who reads a module's config keys (design 32). + ConfigReadersAsk = "mesh-delivery-indexer.indexer_config_readers" + // VaultRetireAsk is the seat mesh-vault's verb retiring one module's own secret on a node (design 32). + VaultRetireAsk = "seat:mesh-vault.retire" +) + +// toolDelivery answers Delivery through the bus. +type toolDelivery struct { + ask func(key string, body any) (json.RawMessage, error) +} + +// Have asks node_haves for one node and module and reads the fingerprint the node runs. +func (d toolDelivery) Have(node, module string) (string, bool, error) { + raw, err := d.ask(HavesAsk, map[string]any{"node": node, "module": module}) + if err != nil { + return "", false, err + } + answer, output, ok := unwrapAnswer(raw) + if !ok { + return "", false, fmt.Errorf("node_haves refused: %s", output) + } + var kept struct { + Keys map[string]json.RawMessage `json:"keys"` + } + if err := json.Unmarshal(answer, &kept); err != nil { + return "", false, fmt.Errorf("node_haves answered what is not its listing: %s", output) + } + key, err := delivery.HaveKey(node, module) + if err != nil { + return "", false, err + } + have, ok := kept.Keys[key] + if !ok { + return "", false, nil + } + var h delivery.Have + if err := json.Unmarshal(have, &h); err != nil { + return "", false, fmt.Errorf("the have of %s on %s is not readable: %w", module, node, err) + } + return h.FP, true, nil +} + +// Reads asks a node's node-engine what it reads; an answer means it states it. +func (d toolDelivery) Reads(node string) error { + _, err := d.ask(ManifestReadsAsk+"@"+node, map[string]any{}) + return err +} + +// Readers asks the indexer who reads a module's config keys; an indexer that does not serve the tool yet is "not known". +func (d toolDelivery) Readers(module string) ([]ReaderRow, bool, error) { + raw, err := d.ask(ConfigReadersAsk, map[string]any{"module": module}) + if err != nil { + if notServed(err) { + return nil, false, nil + } + return nil, false, err + } + answer, output, ok := unwrapAnswer(raw) + if !ok { + if notServed(fmt.Errorf("%s", output)) { + return nil, false, nil + } + return nil, false, fmt.Errorf("indexer_config_readers refused: %s", output) + } + var a struct { + Readers []ReaderRow `json:"readers"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return nil, false, fmt.Errorf("indexer_config_readers answered what is not its listing: %s", output) + } + return a.Readers, true, nil +} + +// notServed is whether an error says the address has no holder serving it. +func notServed(err error) bool { + m := strings.ToLower(err.Error()) + for _, w := range []string{"no responders", "not served", "unknown tool", "no such tool", "serves no"} { + if strings.Contains(m, w) { + return true + } + } + return false +} + +// toolVault asks the seat mesh-vault to retire a secret: the hand act that warrants it is recorded first, by this +// module, naming the node, module, secret and the ask of the Approve; the vault reads it back by its id (`warrant`). +type toolVault struct { + ask func(key string, body any) (json.RawMessage, error) + recorder handacts.Recorder +} + +// Retire records the warrant and calls the vault's verb. +func (v toolVault) Retire(r RetireRequest) error { + taken, err := v.recorder.Record(handacts.Request{What: r.Node + " " + r.Module + " " + r.Name + " " + r.Ask, + Why: "the operator approved retiring " + r.Module + "'s secret " + r.Name + " on " + r.Node + " (ask " + r.Ask + ")", + Cause: "the operator's Approve of ask " + r.Ask, Verb: "mesh-vault.retire"}) + if err != nil { + return err + } + answer, _, _ := unwrapAnswer(taken.Answer) + var rec struct { + Act struct { + ID string `json:"id"` + } `json:"act"` + } + if json.Unmarshal(answer, &rec) != nil || rec.Act.ID == "" { + return fmt.Errorf("mesh-hand-acts recorded the warrant and named no act") + } + _, err = v.ask(VaultRetireAsk, map[string]any{"node": r.Node, "module": r.Module, "name": r.Name, "warrant": rec.Act.ID, "ask": r.Ask}) + return err +} diff --git a/modules/mesh-config/cmd/mesh-config/findings.go b/modules/mesh-config/cmd/mesh-config/findings.go new file mode 100644 index 00000000..fcccb214 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/findings.go @@ -0,0 +1,138 @@ +package main + +// mesh-config's findings (novox/hq ADR 0564 clause 2): what it sees wrong about a rename or a retirement, kept in its +// state `findings` as the condition keeper reads them, written when what they say changes and deleted when the fault +// is gone. The keeper raises and clears the condition of the same key (mesh-conditions' Observers and Homes). + +import ( + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// FindingsState is the state, health.FindingsState. +const FindingsState = health.FindingsState + +// The kinds of finding mesh-config writes. +const ( + // FindingRenameRefused is a move mesh-config refused; cleared once the value moved or `renamed-from` left the + // trunk's manifest. + FindingRenameRefused = "rename-refused" + // FindingWindowHeld is a rename window held open seven days by a node; cleared when the window closes. + FindingWindowHeld = "rename-window-held" + // FindingRetireUnanswered is a retirement's ask unanswered after it was asked once more; cleared by an answer, a + // cancel, or the value read again. + FindingRetireUnanswered = "retiring-unanswered" + // FindingRetireHeld is a retirement waiting seven days for a node still running a build that reads the value. + FindingRetireHeld = "retirement-held" +) + +// findingKey is the key of a finding of kind about a module on a node: the condition's key. +func findingKey(kind, module, node, ref string) string { + id := module + if node != "" { + id += "." + node + } + return health.Key(health.ScopeModule, id, kind+"-"+ref) +} + +func finding(kind, module, node, ref, summary, headline, explanation, needs, resolved, said string, at time.Time) health.Finding { + id := module + if node != "" { + id += "." + node + } + return health.Finding{V: health.FindingVersion, Key: findingKey(kind, module, node, ref), Kind: kind, Ref: kind + "-" + ref, + Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node}, + Severity: health.Warning, Resolver: health.ResolverOperator, Summary: summary, Headline: headline, + Explanation: explanation, Needs: needs, Resolved: resolved, + Evidence: []health.Evidence{{At: at.UTC(), Said: said}}, Source: "config", Observer: Module, At: at.UTC()} +} + +// renameRefused is the finding of a move mesh-config refused. +func renameRefused(module, to, from, why string, at time.Time) health.Finding { + return finding(FindingRenameRefused, module, "", to, + fmt.Sprintf("the rename of %s's config value %s to %s is refused: %s", module, from, to, why), + fmt.Sprintf("%s's value %s is not moved to %s", module, from, to), + fmt.Sprintf("The new version of %s names %s as the new name of its config value %s, and mesh-config refused to move the value: %s. Nothing was moved.", module, to, from, why), + fmt.Sprintf("put the cause right, then move it with config_rename_again (module %s, to %s); the details name it.", module, to), + fmt.Sprintf("Resolved: %s's value %s is under %s", module, from, to), why, at) +} + +// windowHeld is the finding of a rename window held open past HeldFinding by a node. +func windowHeld(r Rename, node string, at time.Time) health.Finding { + return finding(FindingWindowHeld, r.Module, node, r.To, + fmt.Sprintf("%s on %s has held the old config name %s open for %d days", r.Module, node, r.From, int(HeldFinding.Hours()/24)), + fmt.Sprintf("%s on %s still runs the old config name", r.Module, node), + fmt.Sprintf("The rename of %s's config value %s to %s is not complete on %s: its node-engine has not reported a build that reads %s (it may be away).", r.Module, r.From, r.To, node, r.To), + fmt.Sprintf("bring %s back, or drop it from the window with config_rename_close.", node), + fmt.Sprintf("Resolved: %s reads %s on %s", r.Module, r.To, node), + fmt.Sprintf("the window opened %s and %s has not reported a build reading %s", r.Nodes[node].Since.UTC().Format(time.RFC3339), node, r.To), at) +} + +// setFinding writes a finding when it differs from the one kept. +func (c *Config) setFinding(f health.Finding) { + if _, err := health.EncodeFinding(f); err != nil { + c.logf("[mesh-config] a finding is not valid: %v", err) + return + } + if e, err := c.Store.Get(FindingsState, f.Key); err == nil && e != nil { + if old, derr := health.DecodeFinding(e.Value); derr == nil && old.Summary == f.Summary { + return // what it says has not changed + } + } + if err := c.Store.Put(FindingsState, f.Key, f); err != nil { + c.logf("[mesh-config] the finding %s is not written: %v", f.Key, err) + } +} + +// clearFinding deletes the findings of a kind and ref about a module, on whatever node but those in keep. +func (c *Config) clearFinding(kind, module, ref string, keep ...string) { + keys, err := c.Store.Keys(FindingsState) + if err != nil { + return + } + want := kind + "-" + ref + for _, k := range keys { + e, err := c.Store.Get(FindingsState, k) + if err != nil || e == nil { + continue + } + f, err := health.DecodeFinding(e.Value) + if err != nil || f.Ref != want || !(f.Subject.ID == module || strings.HasPrefix(f.Subject.ID, module+".")) || + (f.Subject.Machine != "" && slices.Contains(keep, f.Subject.Machine)) { + continue + } + if err := c.Store.Delete(FindingsState, k); err != nil { + c.logf("[mesh-config] the finding %s is not cleared: %v", k, err) + } + } +} + +// retirementHeld is the finding of a retirement that waits seven days for nodes that still run a build reading the value. +func retirementHeld(e Retiring, at time.Time) health.Finding { + node := "" + if len(e.Nodes) > 0 { + node = e.Nodes[0] + } + return finding(FindingRetireHeld, e.Module, node, e.Key, + fmt.Sprintf("%s's %s has waited %d days to retire: %s still run a build that reads it", e.Module, e.Key, int(HeldFinding.Hours()/24), strings.Join(e.Nodes, ", ")), + fmt.Sprintf("%s's %s cannot be retired yet", e.Module, e.Key), + fmt.Sprintf("No manifest reads %s's %s any more, but %s still run a build that does, so the operator is not asked to remove it.", e.Module, e.Key, strings.Join(e.Nodes, ", ")), + "bring those nodes up to date, or cancel the retirement with config_retiring.", + fmt.Sprintf("Resolved: no node reads %s's %s", e.Module, e.Key), + "retiring since "+e.Since.UTC().Format(time.RFC3339)+", still read on "+strings.Join(e.Nodes, ", "), at) +} + +// retireUnanswered is the finding of a retirement asked twice and not answered. +func retireUnanswered(e Retiring, at time.Time) health.Finding { + return finding(FindingRetireUnanswered, e.Module, "", e.Key, + fmt.Sprintf("the operator did not answer the ask to remove %s's %s, asked twice", e.Module, e.Key), + fmt.Sprintf("%s's %s waits for an answer", e.Module, e.Key), + fmt.Sprintf("No manifest reads %s's %s any more. The operator was asked twice whether to remove it and did not answer, so it stays.", e.Module, e.Key), + "answer the ask again with config_retiring ask-again, or cancel it.", + fmt.Sprintf("Resolved: %s's %s is answered or read again", e.Module, e.Key), + "asked "+e.Asked.UTC().Format(time.RFC3339)+" and once more", at) +} diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 93b271a8..454126b5 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -10,6 +10,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -30,10 +31,19 @@ const ( func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } -// runtimeStore is the bus, through the runtime that launched this bundle. +// theViews are the pictures of the delivery owners' state, made when the module runs (nil in a test of the tools). +var theViews *ownerViews + +// runtimeStore is the bus, through the runtime that launched this bundle. A state of the delivery owners that the +// owners' tools answer is read from the picture those keep (ownerviews.go), the rest from the bus. type runtimeStore struct{} func (runtimeStore) Get(state, key string) (*Entry, error) { + if theViews != nil { + if e, followed, err := theViews.get(state, key); followed { + return e, err + } + } e, err := stdio.State(state).Get(key) if err != nil || e == nil { return nil, err @@ -48,7 +58,14 @@ func (runtimeStore) Put(state, key string, value any) error { func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } -func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +func (runtimeStore) Keys(state string) ([]string, error) { + if theViews != nil { + if keys, followed, err := theViews.keys(state); followed { + return keys, err + } + } + return stdio.State(state).Keys() +} func (runtimeStore) Append(log, key string, value any) error { _, err := stdio.Log(log).Append(key, value) @@ -100,9 +117,23 @@ func (runtimeAsks) Warranted(ask string) error { } func main() { - c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}} - go resolving(c) + c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}, + Delivery: toolDelivery{ask: stdio.Ask}, Vault: toolVault{ask: stdio.Ask, recorder: recorder}} + moved := make(chan struct{}, 1) + theViews = newOwnerViews(stdio.Ask, func() { + select { + case moved <- struct{}{}: + default: + } + }, func(key string, v json.RawMessage) error { + _, err := stdio.State(FindingsState).Put(key, v) + return err + }, + func(key string) error { return stdio.State(FindingsState).Delete(key) }) + theViews.Start(context.Background()) + go resolving(c, moved) go hearing(c) + go hearingDelivery(c) if err := stdio.Serve("", Tools(c)); err != nil { logf("%v", err) os.Exit(1) @@ -112,22 +143,9 @@ func main() { // resolving resolves at start and again whenever a source moves, and every five minutes besides, so a change // heard nowhere is still resolved. The runtime answers only once Serve runs, so the watches are asked again // until they are taken. -func resolving(c *Config) { - moved := make(chan struct{}, 1) - poke := func(stdio.StateChange) error { - select { - case moved <- struct{}{}: - default: - } - return nil - } - // A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is - // every change of what a module declares where (issue 1953); the builds themselves are read, not watched. - for _, read := range []string{PlacementRead, WantsRead, ModulesRead, IndexerPredefinedRead, - CandidatesRead} { - // Asked again until the runtime takes it, each give-up said (mesh-sdk's Follow). - go stdio.State(read).Follow("", poke) - } +func resolving(c *Config, moved chan struct{}) { + // The owners' states reach `moved` through their views (ownerviews.go); the builder's builds, which has no tool yet, + // is read when a want moves, and nothing else is watched. tick := time.NewTicker(5 * time.Minute) defer tick.Stop() time.Sleep(time.Second) @@ -221,9 +239,90 @@ func hearing(c *Config) { } } +// hearingDelivery takes the delivery parts' words that close a rename window or retire a value, each asked again until +// the runtime takes the subscription: the status view's `delivered`, the node-engine seat's `manifest-reads-changed` +// (a returning node-engine; design 32) and the indexer's `manifest-accepted`. At its start it judges the windows once, +// since an event said while this module was down is not said again. +func hearingDelivery(c *Config) { + <-stdio.Served() + for _, h := range []struct { + pattern string + handle func(stdio.Envelope) []string + }{ + {"mesh-delivery-view.delivered", func(stdio.Envelope) []string { return c.Windows("") }}, + {"node-engine.manifest-reads-changed", func(e stdio.Envelope) []string { + var b struct { + Node string `json:"node"` + } + _ = json.Unmarshal(e.Body, &b) + if b.Node == "" { + // The event's subject ends with the node's token (ADR 0618 as built): mesh.seat.node-engine.event.manifest-reads-changed.. + b.Node = e.Key[strings.LastIndex(e.Key, ".")+1:] + } + return c.Windows(b.Node) + }}, + {"mesh-delivery-indexer.manifest-accepted", func(e stdio.Envelope) []string { + var b struct { + Module string `json:"module"` + } + if json.Unmarshal(e.Body, &b) != nil { + return nil + } + return c.Retirements(b.Module) + }}, + } { + go func(pattern string, handle func(stdio.Envelope) []string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(pattern, func(e stdio.Envelope) error { + for _, s := range handle(e) { + logf("[mesh-config] %s", s) + } + return nil + }) + if err == nil { + return + } + logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait) + time.Sleep(wait) + } + }(h.pattern, h.handle) + } + for _, s := range c.Windows("") { + logf("[mesh-config] %s", s) + } +} + +// seatRename is one open rename as the seat's definition answers it: module, from, to, at and the nodes holding it open. +type seatRename struct { + Module string `json:"module"` + From string `json:"from"` + To string `json:"to"` + At time.Time `json:"at"` + Open []string `json:"open"` +} + +func seatRenames(rows []RenameRow) []seatRename { + out := make([]seatRename, 0, len(rows)) + for _, r := range rows { + out = append(out, seatRename{Module: r.Module, From: r.From, To: r.To, At: r.At, Open: r.Open}) + } + return out +} + // Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them. func Tools(c *Config) []stdio.Tool { - return append(switchTools(c), stdio.Tool{ + renames := stdio.Tool{Name: Seat + "." + RenamesVerb, + Description: "The open renames of config values: for each, the module, the old and the new name, and the nodes " + + "still holding the old name open (the rename window). The checker asks it before it lets a manifest add or " + + "drop renamed-from (novox/hq design 32). Narrowed by module.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + rows, err := c.Renames(strings.TrimSpace(m), true) + return map[string]any{"renames": seatRenames(rows)}, err + }} + return append(append(switchTools(c), viewTools(c, func() *ownerViews { return theViews })...), renames, stdio.Tool{ Name: Seat + "." + Verb, Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " + "one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " + diff --git a/modules/mesh-config/cmd/mesh-config/manifest_test.go b/modules/mesh-config/cmd/mesh-config/manifest_test.go index 7c3b73a9..42fa4e6d 100644 --- a/modules/mesh-config/cmd/mesh-config/manifest_test.go +++ b/modules/mesh-config/cmd/mesh-config/manifest_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself @@ -35,6 +37,10 @@ func TestEveryStateAndActHasItsTool(t *testing.T) { PlacementRead: "config list", WantsRead: "config list", BuildsRead: "config list", ModulesRead: DeclaredTool, ManifestsRead: DeclaredTool, IndexerPredefinedRead: "config list", CandidatesRead: ResyncTool, "a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, + // design 32: each new state with the tool that reads it, each act with the tool that starts it. + RenamesState: RenamesTool, RetiringState: RetiringTool, FindingsState: "config_renames and config_retiring", + "a move refused": RenameAgainTool, "a window dropped": RenameCloseTool, "an ask again": RetiringTool, + "the owners' first views": FirstViewsTool, "a first view asked again": FirstViewAgainTool, } served := map[string]bool{} for _, tool := range Tools(&Config{}) { @@ -43,6 +49,21 @@ func TestEveryStateAndActHasItsTool(t *testing.T) { if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") { t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs) } + // The stream and its events are declared as the module emits them, and the manifest decodes whole. + if dm, err := catalogue.Decode(raw); err != nil { + t.Errorf("the manifest does not decode: %v", err) + } else { + emits := strings.Join(dm.Emits, ",") + for _, e := range []string{ConfigChanged, EventRenamed, EventRetiring, EventRetired} { + if !strings.Contains(emits, e) { + t.Errorf("the manifest does not declare the event %s", e) + } + } + if len(dm.Streams) != 1 || dm.Streams[0].Name != StreamName || dm.Streams[0].Days() != 7 || !dm.Streams[0].Carries(EventRenamed) || + !dm.Streams[0].Carries(EventRetiring) || !dm.Streams[0].Carries(EventRetired) || dm.Streams[0].Carries(ConfigChanged) { + t.Errorf("streams %+v: want %s, seven days, carrying renamed, retiring and retired but not config-changed", dm.Streams, StreamName) + } + } for _, verb := range SeatVerbs { if !served[Seat+"."+verb] { t.Errorf("the seat's verb %s is not served", verb) @@ -53,6 +74,11 @@ func TestEveryStateAndActHasItsTool(t *testing.T) { t.Errorf("the manifest lists the tool %s, which is not served", name) } } + // The delivery owners' state is read through their tools and views (ADR 0615 §1): only the builder's builds, which has + // no tool yet, is still a read. + if len(m.Reads) != 1 || m.Reads[0] != BuildsRead { + t.Errorf("reads %v, want the builder's builds alone", m.Reads) + } for _, list := range [][]string{m.State, m.Logs, m.Reads} { for _, s := range list { if _, ok := via[s]; !ok { diff --git a/modules/mesh-config/cmd/mesh-config/ownerviews.go b/modules/mesh-config/cmd/mesh-config/ownerviews.go new file mode 100644 index 00000000..17900b6c --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/ownerviews.go @@ -0,0 +1,249 @@ +package main + +// What mesh-config reads of the delivery owners' state, through their tools and views families (novox/hq ADR 0615 §1, +// §4 to §6, ADR 0596): never a bucket of theirs. For each of the six states it follows (the indexer's modules, manifests, +// predefined and config-candidates, the assigner's placement and wants) it asks the owner's read tool for the whole +// state at its start, keeps the picture from the owner's `views.` events, and asks again only on a gap. Started +// while an owner is down it keeps asking, and past two minutes writes a finding naming the owner and the tool. +// +// The ordinary read tools are asked, not the first-view tools: those answer a few named callers (mesh-grants, +// node-tools, the delivery parts), and mesh-config is not one. The builder's `builds` has no owner tool yet (A2 built the +// indexer's and the assigner's), so it is still read as a `reads` entry. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-delivery/sdk/views" +) + +// The tools of the owners that mesh-config asks, by the state they read. +const ( + IndexerModulesTool = "indexer_read_modules" + IndexerManifestsTool = "indexer_read_manifests" + IndexerPredefTool = "indexer_read_predefined" + IndexerCandidatesTool = "indexer_read_config_candidates" + AssignerPlacementTool = "assigner_read_placement" + AssignerWantsTool = "assigner_read_wants" + + FirstViewsTool = "config_first_views" + FirstViewAgainTool = "config_first_view_again" +) + +// followed is each state read through its owner: the `.` name this module's code uses, the owner, its +// state and its tool. +var followed = []struct{ Read, Owner, State, Tool string }{ + {ModulesRead, delivery.IndexerModule, "modules", IndexerModulesTool}, + {ManifestsRead, delivery.IndexerModule, "manifests", IndexerManifestsTool}, + {IndexerPredefinedRead, delivery.IndexerModule, delivery.IndexerPredefinedState, IndexerPredefTool}, + {CandidatesRead, delivery.IndexerModule, delivery.IndexerConfigCandidatesState, IndexerCandidatesTool}, + {PlacementRead, delivery.AssignerModule, "placement", AssignerPlacementTool}, + {WantsRead, delivery.AssignerModule, "wants", AssignerWantsTool}, +} + +// ownerViews are the pictures, by the read name. +type ownerViews struct { + by map[string]*views.Follower + // store keeps the findings a first view unanswered writes. + store func(key string, value json.RawMessage) error + drop func(key string) error +} + +// newOwnerViews makes a follower for each state; changed is told when any picture changes. +func newOwnerViews(ask func(key string, body any) (json.RawMessage, error), changed func(), put func(string, json.RawMessage) error, + del func(string) error) *ownerViews { + o := &ownerViews{by: map[string]*views.Follower{}, store: put, drop: del} + for _, f := range followed { + f := f + fol := &views.Follower{Owner: f.Owner, Tool: f.Owner + "." + f.Tool, State: f.State, Logf: logf, + Ask: func(_ context.Context, after string) (views.Answer, error) { + body := map[string]any{"values": true} + if after != "" { + body["after"] = after + } + raw, err := ask(f.Owner+"."+f.Tool, body) + if err != nil { + return views.Answer{}, err + } + answer, output, ok := unwrapAnswer(raw) + if !ok { + return views.Answer{}, fmt.Errorf("%s refused: %s", f.Tool, output) + } + return views.DecodeAnswer(answer) + }, + Changed: func(string, json.RawMessage, bool, time.Time) { changed() }} + fol.Raise = func(s views.Status) { + if k, v, err := views.UnansweredFinding(Module, s, time.Now()); err == nil { + _ = put(k, v) + } + } + fol.Clear = func(s views.Status) { + if k, _, err := views.UnansweredFinding(Module, s, time.Now()); err == nil { + _ = del(k) + } + } + o.by[f.Read] = fol + } + return o +} + +// Start asks every first view until answered (each on its goroutine) and subscribes to the owners' views events. +func (o *ownerViews) Start(ctx context.Context) { + for _, fol := range o.by { + go func(f *views.Follower) { + if err := f.Start(ctx); err != nil { + logf("[mesh-config] the first view of %s's %s ended unanswered: %v", f.Owner, f.State, err) + } + }(fol) + } + for _, f := range followed { + f := f + pattern := f.Owner + "." + views.EventName(f.State) + go func() { + <-stdio.Served() + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(pattern, func(e stdio.Envelope) error { + ev, err := views.DecodeEvent(e.Body) + if err != nil { + logf("[mesh-config] %s is not a views event: %v", pattern, err) + return nil + } + fol := o.by[f.Read] + if gap, why := fol.Apply(ev); gap { + logf("[mesh-config] a gap in %s's %s (%s): its first view is asked once more", f.Owner, f.State, why) + go func() { _ = fol.Start(ctx) }() + } + return nil + }) + if err == nil { + return + } + logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait) + time.Sleep(wait) + } + }() + } +} + +// read is the picture of a read name, and whether the name is one followed. +func (o *ownerViews) read(state string) (*views.Follower, bool) { + f, ok := o.by[state] + return f, ok +} + +// errUnanswered says a first view is not answered yet: a reader waits, and its next resolution reads it. +func errUnanswered(f *views.Follower) error { + s := f.Status() + return fmt.Errorf("the first view of %s's %s is not answered yet (%s asked %d times)", s.Owner, s.State, s.Tool, s.Asks) +} + +// get is one key of a followed state. +func (o *ownerViews) get(state, key string) (*Entry, bool, error) { + f, ok := o.read(state) + if !ok { + return nil, false, nil + } + select { + case <-f.Ready(): + default: + return nil, true, errUnanswered(f) + } + v, held := f.Get(key) + if !held { + return nil, true, nil + } + return &Entry{Key: key, Value: v}, true, nil +} + +// keys are the keys of a followed state, sorted. +func (o *ownerViews) keys(state string) ([]string, bool, error) { + f, ok := o.read(state) + if !ok { + return nil, false, nil + } + select { + case <-f.Ready(): + default: + return nil, true, errUnanswered(f) + } + var out []string + for k := range f.Values() { + out = append(out, k) + } + sort.Strings(out) + return out, true, nil +} + +// statuses are the first views' statuses, by owner and state. +func (o *ownerViews) statuses() []views.Status { + var out []views.Status + for _, f := range followed { + out = append(out, o.by[f.Read].Status()) + } + return out +} + +// again asks one first view once more by hand (".", or all when empty). +func (o *ownerViews) again(ctx context.Context, which, why string) ([]string, error) { + var done []string + for _, f := range followed { + if which != "" && which != f.Owner+"."+f.State { + continue + } + if err := o.by[f.Read].Again(ctx, why); err != nil { + return done, err + } + done = append(done, f.Owner+"."+f.State) + } + if len(done) == 0 { + return nil, errors.New("no first view is named " + which) + } + return done, nil +} + +// viewTools are the read tool of the first views and its re-read (design 25 § 12.2): the status of each, and a hand ask. +func viewTools(c *Config, o func() *ownerViews) []stdio.Tool { + return []stdio.Tool{ + {Name: FirstViewsTool, + Description: "mesh-config's first views of the delivery owners' state: for each of the indexer's modules, manifests, " + + "predefined and config-candidates and the assigner's placement and wants, the tool it asks, the revision its " + + "picture holds, when it asked and was answered, how many asks (and by hand), and whether its condition is raised.", + Run: func(map[string]any) (any, error) { + ov := o() + if ov == nil { + return nil, errors.New("mesh-config reads its owners through views only when it runs as the module") + } + return map[string]any{"first-views": ov.statuses()}, nil + }}, + {Name: FirstViewAgainTool, + Description: "Ask a first view once more by hand, `.` (all when not given), recorded as a hand act with " + + "its why.", + Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{ + "view": map[string]any{"type": "string", "description": "., such as mesh-delivery-assigner.wants"}, + "why": map[string]any{"type": "string"}}}, + Run: func(a map[string]any) (any, error) { + ov := o() + if ov == nil { + return nil, errors.New("mesh-config reads its owners through views only when it runs as the module") + } + why, _ := a["why"].(string) + view, _ := a["view"].(string) + if strings.TrimSpace(why) == "" { + return nil, errors.New("say why: a hand act keeps its why") + } + if err := c.Store.HandAct("config_first_view_again "+view, strings.TrimSpace(why), "config_first_view_again"); err != nil { + return nil, fmt.Errorf("the hand-act log does not hold it, so nothing is asked: %w", err) + } + done, err := ov.again(context.Background(), strings.TrimSpace(view), strings.TrimSpace(why)) + return map[string]any{"asked-again": done}, err + }}, + } +} diff --git a/modules/mesh-config/cmd/mesh-config/ownerviews_test.go b/modules/mesh-config/cmd/mesh-config/ownerviews_test.go new file mode 100644 index 00000000..8f04825f --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/ownerviews_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/sdk/views" +) + +// CAT-C11 (ADR 0615 §1): a followed state of the delivery owners is read from the picture of its owner's tool and events: +// a key asked before the first view is answered is an error (a reader waits), never a read of the owner's bucket; after +// it, the owner's value; an event at the next revision changes it, one past it is a gap and asks the first view again; a +// state that is no owner's falls through to the bus. +func TestAFollowedStateIsReadFromItsOwnersFirstViewAndEvents(t *testing.T) { + created := time.Date(2026, 11, 4, 9, 0, 0, 0, time.UTC) + var asks atomic.Int32 + hold := make(chan struct{}) + ask := func(key string, body any) (json.RawMessage, error) { + if key != "mesh-delivery-assigner.assigner_read_wants" || body.(map[string]any)["values"] != true { + return nil, errors.New("asked " + key) + } + <-hold + n := asks.Add(1) + a := views.Answer{V: 1, Owner: "mesh-delivery-assigner", State: "wants", Revision: uint64(n) * 10, Created: created, + Values: map[string]json.RawMessage{"want.g14.dunst": json.RawMessage(`{"fp":"a"}`)}} + raw, _ := json.Marshal(a) + return raw, nil + } + var changed atomic.Int32 + o := newOwnerViews(ask, func() { changed.Add(1) }, func(string, json.RawMessage) error { return nil }, func(string) error { return nil }) + if _, followed, err := o.get(WantsRead, "want.g14.dunst"); !followed || err == nil || !strings.Contains(err.Error(), "not answered yet") { + t.Fatalf("before the first view: followed %v, %v", followed, err) + } + if _, followed, _ := o.get(BuildsRead, "x"); followed { + t.Error("the builder's builds is followed, and it has no owner tool") + } + ctx := context.Background() + go func() { _ = o.by[WantsRead].Start(ctx) }() + hold <- struct{}{} + <-o.by[WantsRead].Ready() + e, isFollowed, err := o.get(WantsRead, "want.g14.dunst") + if err != nil || !isFollowed || e == nil || string(e.Value) != `{"fp":"a"}` { + t.Fatalf("after the first view: %+v, %v, %v", e, isFollowed, err) + } + ev := func(rev uint64, key string, deleted bool) views.Event { + return views.Event{V: 1, Owner: "mesh-delivery-assigner", State: "wants", Key: key, Revision: rev, Created: created, Deleted: deleted, + Value: json.RawMessage(`{"fp":"b"}`)} + } + if gap, why := o.by[WantsRead].Apply(ev(11, "want.novox.dunst", false)); gap { + t.Fatalf("the next revision is a gap: %s", why) + } + if keys, _, _ := o.keys(WantsRead); strings.Join(keys, ",") != "want.g14.dunst,want.novox.dunst" { + t.Errorf("keys %v", keys) + } + if gap, _ := o.by[WantsRead].Apply(ev(11, "want.novox.dunst", false)); gap { + t.Error("an event already in the picture is a gap") + } + if gap, why := o.by[WantsRead].Apply(ev(13, "want.x.y", false)); !gap || !strings.Contains(why, "changes missed") { + t.Errorf("a missed revision: gap %v, %s", gap, why) + } + o.by[WantsRead].Apply(ev(12, "want.g14.dunst", true)) + if e, _, _ := o.get(WantsRead, "want.g14.dunst"); e != nil { + t.Error("a deleted key is still held") + } + if changed.Load() < 3 { + t.Errorf("the resolution was told of %d changes", changed.Load()) + } + if got := o.statuses(); len(got) != len(followed) { + t.Errorf("statuses %d, want %d", len(got), len(followed)) + } +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index d6cb304d..d9224798 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -759,6 +759,8 @@ func (c *Config) Decide() ([]string, error) { said = append(said, c.decide(r.p, r.w)) } c.recordWarrants(now) + // The retirements' asks are taken from the router's record the same way (retiring.go). + said = append(said, c.decideRetirements()...) return said, nil } diff --git a/modules/mesh-config/cmd/mesh-config/renames.go b/modules/mesh-config/cmd/mesh-config/renames.go new file mode 100644 index 00000000..e82958fb --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/renames.go @@ -0,0 +1,494 @@ +package main + +// A config value renamed (novox/hq ADR 0609 §4, design 32 *A config value renamed*). +// +// A declaration naming `renamed-from` moves the value set under the old name to the new name, with the value's trust +// (its Value, whole), and opens a **rename window** per node: until that node's node-engine has applied a build of the +// module that reads the new name, the old name still answers with the new value (Resolve). A window closes on the +// status view's `delivered` event and, for a node that was away, on the node-engine seat's `manifest-reads-changed` +// (Windows). Its state is `renames`, key `.`; what it says is `renamed` on the stream `config-events`. + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" +) + +// The state, the stream's events and the bounds this part keeps. +const ( + RenamesState = "renames" + // EventRenamed is `renamed`, EventRetiring `retiring` and EventRetired `retired`, the family `config.events` of the + // seat's definition (design 26), under the holder's subjects: each carried by the stream StreamName, kept seven days. + EventRenamed = "renamed" + EventRetiring = "retiring" + EventRetired = "retired" + StreamName = "config-events" + // HeldFinding is how long a window or a retirement may stay open before it is a finding naming the node. + HeldFinding = 7 * 24 * time.Hour +) + +// Rename is one rename, as `renames` keeps it, key `.`. +type Rename struct { + Module string `json:"module"` + From string `json:"from"` + To string `json:"to"` + At time.Time `json:"at"` + // Moved are the nodes a value was moved on, by the last move of each. + Moved []string `json:"moved,omitempty"` + // Nodes are the windows, by node. + Nodes map[string]Window `json:"nodes"` + // Said says `renamed` was emitted; false until the bus took it. + Said bool `json:"said"` +} + +// Window is one node's rename window: open until Closed is set. +type Window struct { + Since time.Time `json:"since"` + Closed *time.Time `json:"closed,omitempty"` + // How is what closed it: "applied" (the node runs a build reading the new name), "unplaced" (the module left the + // node), or "by hand" (config_rename_close). + How string `json:"how,omitempty"` +} + +// Open is the nodes whose window is open, sorted. +func (r Rename) Open() []string { + var out []string + for node, w := range r.Nodes { + if w.Closed == nil { + out = append(out, node) + } + } + sort.Strings(out) + return out +} + +// renameKey is a rename's key in `renames`. +func renameKey(module, to string) string { return module + "." + to } + +// RenamedEvent is the body of `renamed`. +type RenamedEvent struct { + Module string `json:"module"` + From string `json:"from"` + To string `json:"to"` + At time.Time `json:"at"` +} + +// readRenames is every rename kept, by key. +func (c *Config) readRenames() (map[string]Rename, error) { + out := map[string]Rename{} + err := c.each(RenamesState, func(key string, raw []byte) error { + var r Rename + if json.Unmarshal(raw, &r) != nil || r.Module == "" || renameKey(r.Module, r.To) != key { + return nil // not one this module wrote + } + if r.Nodes == nil { + r.Nodes = map[string]Window{} + } + out[key] = r + return nil + }) + return out, err +} + +// moduleValueKey is the key of a module's own value on a node. +func moduleValueKey(node, module, key string) string { + return Level{Kind: LevelModule, Node: node, Module: module}.ValueKey(key) +} + +// applyRenames moves the values of every rename the trunk's manifests declare, opens the windows of the nodes the +// module is placed on, and writes `renamed` once: run on each resolution, under the lock, before the resolved values +// are written. s.Values and s.Renames are brought to what it did, so the same resolution sees it. A rename it refuses +// is a finding (findings.go), cleared once it moved or `renamed-from` left the manifest. It answers a line for what it +// did or could not do. +func (c *Config) applyRenames(s *Sources) []string { + held, err := c.readRenames() + if err != nil { + return []string{"the renames cannot be read: " + err.Error()} + } + now := c.now() + var said []string + seen := map[string]bool{} + for _, module := range modulesOf(s.Assignments) { + trunk, err := c.trunkDeclarations(module) + if err != nil { + continue // unresolved, said by the resolution + } + for _, to := range sortedKeys(trunk.Config) { + decl := trunk.Config[to] + if decl.RenamedFrom == "" { + continue + } + key := renameKey(module, to) + seen[key] = true + said = append(said, c.rename(s, held, trunk, to, decl, now)...) + } + } + // A rename whose `renamed-from` left the trunk's manifest goes once every window is closed. + for key, r := range held { + if seen[key] { + continue + } + if len(r.Open()) > 0 { + s.Renames[key] = r + continue + } + if err := c.Store.Delete(RenamesState, key); err != nil { + said = append(said, fmt.Sprintf("the rename %s is not removed: %v", key, err)) + continue + } + delete(s.Renames, key) + c.clearFinding(FindingRenameRefused, r.Module, r.To) + c.clearFinding(FindingWindowHeld, r.Module, r.To) + } + c.sweepRenameFindings(s, now) + return said +} + +// rename judges and applies one declared rename. +func (c *Config) rename(s *Sources, held map[string]Rename, trunk ModuleDeclarations, to string, decl Declaration, + now time.Time) []string { + module, from := trunk.Module, decl.RenamedFrom + key := renameKey(module, to) + nodes := nodesPlaced(s, module) + var said []string + refuse := func(why string) []string { + c.setFinding(renameRefused(module, to, from, why, now)) + if r, ok := held[key]; ok { + s.Renames[key] = r + } + return []string{fmt.Sprintf("the rename of %s's %s to %s is refused: %s", module, from, to, why)} + } + if _, both := trunk.Config[from]; both { + return refuse("the manifest declares both names") + } + if from == to { + return refuse("the new name is the old one") + } + trusted := decl.Trusted + if p, ok := s.Predefined[decl.Predefined]; ok && p.Trusted { + trusted = true + } + type move struct { + node string + old, new Value + hadNew bool + oldKey string + newKey string + } + var moves []move + for _, node := range nodes { + oldKey, newKey := moduleValueKey(node, module, from), moduleValueKey(node, module, to) + ov, hadOld := s.Values[oldKey] + if !hadOld { + continue + } + nv, hadNew := s.Values[newKey] + switch { + case hadNew && string(compact(nv.Value)) != string(compact(ov.Value)): + return refuse(fmt.Sprintf("on %s both names hold a value, and they differ (%s under the old name, %s under the new)", + node, ov.Value, nv.Value)) + case trusted && ov.Approved == "": + return refuse(fmt.Sprintf("on %s the value under the old name was set without the operator's Approve, and "+ + "the new name is trusted (a rename keeps a value's trust, novox/hq ADR 0348 §8)", node)) + } + moves = append(moves, move{node: node, old: ov, new: nv, hadNew: hadNew, oldKey: oldKey, newKey: newKey}) + } + r, had := held[key] + if !had { + r = Rename{Module: module, From: from, To: to, At: now, Nodes: map[string]Window{}} + } + changed := !had + for _, m := range moves { + if !m.hadNew { + if err := c.Store.Put(ValuesState, m.newKey, m.old); err != nil { + return refuse("the value on " + m.node + " is not written under the new name: " + err.Error()) + } + } + if err := c.Store.Delete(ValuesState, m.oldKey); err != nil { + return refuse("the old name's value on " + m.node + " is not removed: " + err.Error()) + } + s.Values[m.newKey] = m.old + delete(s.Values, m.oldKey) + lvl := Level{Kind: LevelModule, Node: m.node, Module: module} + _ = c.Store.Append(HistoryLog, m.newKey, Change{Action: "rename", Level: &lvl, Key: to, Value: m.old.Value, + Was: m.old.Value, By: Module, Via: Module, Why: "renamed from " + from + " (renamed-from)", At: now}) + if !slices.Contains(r.Moved, m.node) { + r.Moved = append(r.Moved, m.node) + sort.Strings(r.Moved) + } + changed = true + said = append(said, fmt.Sprintf("%s's %s on %s moved to %s", module, from, m.node, to)) + } + for _, node := range nodes { + if _, ok := r.Nodes[node]; !ok { + r.Nodes[node] = Window{Since: now} + changed = true + } + } + c.clearFinding(FindingRenameRefused, module, to) + if !r.Said { + if err := c.Store.Emit(EventRenamed, RenamedEvent{Module: module, From: from, To: to, At: r.At}); err != nil { + said = append(said, fmt.Sprintf("renamed for %s's %s is not said yet: %v", module, to, err)) + } else { + r.Said, changed = true, true + } + } + if changed { + if err := c.Store.Put(RenamesState, key, r); err != nil { + said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err)) + } + } + s.Renames[key] = r + return said +} + +// nodesPlaced is the nodes a module is placed on, sorted. +func nodesPlaced(s *Sources, module string) []string { + var out []string + for _, a := range s.Assignments { + if a.Module == module { + out = append(out, a.Node) + } + } + sort.Strings(out) + return slices.Compact(out) +} + +// resolveOldNames adds, for each rename with a window open on a node, the old name answering the new name's value on +// that node (design 32: until then mesh-config answers both names). +func resolveOldNames(s Sources, out map[string]Resolved) { + for _, key := range sortedKeys(s.Renames) { + r := s.Renames[key] + for _, node := range r.Open() { + nv, ok := out[ResolvedKey(node, r.Module, r.To)] + if !ok { + continue + } + old := nv + old.Key = r.From + out[ResolvedKey(node, r.Module, r.From)] = old + } + } +} + +// ---- the windows -------------------------------------------------------------------------------------------- + +// Windows closes the rename windows (and the retirements waiting for nodes) that the delivery says are done, at start +// and on each `delivered`; for node, the one a returning node-engine's `manifest-reads-changed` names, it first asks +// the node-engine seat's `manifest-reads`: a node-engine that does not serve it states nothing and its window stays +// open (design 32). It answers a line for each window closed. +func (c *Config) Windows(node string) []string { + c.mu.Lock() + defer c.mu.Unlock() + said := c.closeWindows(node) + said = append(said, c.drainRetirements(node)...) + if len(said) > 0 { + if _, err := c.reconcileLocked(); err != nil { + said = append(said, "resolving after the windows: "+err.Error()) + } + } + return said +} + +// closeWindows is Windows' rename half; the caller holds the lock. +func (c *Config) closeWindows(only string) []string { + renames, err := c.readRenames() + if err != nil { + return []string{"the renames cannot be read: " + err.Error()} + } + if c.Delivery == nil { + return nil + } + s, err := c.sources() + if err != nil { + return []string{"the windows cannot be judged: " + err.Error()} + } + now := c.now() + var said []string + for _, key := range sortedKeys(renames) { + r := renames[key] + changed := false + for _, node := range r.Open() { + if only != "" && only != node { + continue + } + how, why := c.windowShut(s, r, node, only != "") + if why != "" { + said = append(said, fmt.Sprintf("%s's window for %s stays open on %s: %s", r.Module, r.To, node, why)) + } + if how == "" { + continue + } + w := r.Nodes[node] + w.Closed, w.How = &now, how + r.Nodes[node] = w + changed = true + said = append(said, fmt.Sprintf("%s's window for %s is closed on %s (%s)", r.Module, r.To, node, how)) + } + if !changed { + continue + } + if err := c.Store.Put(RenamesState, key, r); err != nil { + said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err)) + } + if len(r.Open()) == 0 { + c.clearFinding(FindingWindowHeld, r.Module, r.To) + } + } + return said +} + +// windowShut says how a node's window may close now ("" when it may not) and, when it may not, why. +func (c *Config) windowShut(s Sources, r Rename, node string, returning bool) (how, why string) { + if !slices.Contains(nodesPlaced(&s, r.Module), node) { + return "unplaced", "" + } + if returning { + if err := c.Delivery.Reads(node); err != nil { + return "", "the node-engine states nothing yet (" + err.Error() + ")" + } + } + fp, found, err := c.Delivery.Have(node, r.Module) + switch { + case err != nil: + return "", "its have cannot be read: " + err.Error() + case !found || fp == "": + return "", "it reports no build of the module" + } + d, err := c.builtDeclarations(r.Module, fp) + if err != nil { + return "", "the build it runs cannot be read: " + err.Error() + } + if _, reads := d.Config[r.To]; !reads { + return "", "it still runs a build that reads " + r.From + } + return "applied", "" +} + +// ---- the tools --------------------------------------------------------------------------------------------- + +// RenameRow is one rename as the tools say it. +type RenameRow struct { + Module string `json:"module"` + From string `json:"from"` + To string `json:"to"` + At time.Time `json:"at"` + Moved []string `json:"moved,omitempty"` + // Open are the nodes holding the old name open; Closed the others, each with how. + Open []string `json:"open"` + Closed map[string]string `json:"closed,omitempty"` +} + +// Renames lists the renames, the open ones only for the seat's verb, narrowed by module. +func (c *Config) Renames(module string, openOnly bool) ([]RenameRow, error) { + renames, err := c.readRenames() + if err != nil { + return nil, err + } + rows := []RenameRow{} + for _, key := range sortedKeys(renames) { + r := renames[key] + if module != "" && r.Module != module { + continue + } + row := RenameRow{Module: r.Module, From: r.From, To: r.To, At: r.At, Moved: r.Moved, Open: r.Open()} + if row.Open == nil { + row.Open = []string{} + } + if openOnly && len(row.Open) == 0 { + continue + } + for node, w := range r.Nodes { + if w.Closed != nil { + if row.Closed == nil { + row.Closed = map[string]string{} + } + row.Closed[node] = w.How + } + } + rows = append(rows, row) + } + return rows, nil +} + +// RenameAgain moves a rename's values now, the way a resolution does, and answers what it did: the recovery for a +// refused move once its cause is put right. +func (c *Config) RenameAgain(module, to string) (any, error) { + c.mu.Lock() + defer c.mu.Unlock() + if module == "" || to == "" { + return nil, fmt.Errorf("name the module and the new name of the rename") + } + r, err := c.reconcileLocked() + if err != nil { + return r, err + } + renames, err := c.readRenames() + if err != nil { + return nil, err + } + if _, ok := renames[renameKey(module, to)]; !ok { + return nil, fmt.Errorf("no rename of %s to %s is kept: its trunk manifest declares no renamed-from for %s, or the rename is refused (config_renames and the findings say which)", module, to, to) + } + return map[string]any{"said": r.Renamed, "moved": r.Moved}, nil +} + +// RenameClose drops a node from a rename's window by hand, recorded as a hand act. +func (c *Config) RenameClose(module, to, node, why string) (any, error) { + if strings.TrimSpace(why) == "" { + return nil, fmt.Errorf("say why the window is dropped: a hand act keeps its why") + } + c.mu.Lock() + defer c.mu.Unlock() + key := renameKey(module, to) + renames, err := c.readRenames() + if err != nil { + return nil, err + } + r, ok := renames[key] + if !ok { + return nil, fmt.Errorf("no rename of %s to %s is kept", module, to) + } + w, ok := r.Nodes[node] + if !ok || w.Closed != nil { + return nil, fmt.Errorf("%s holds no open window for %s's %s; it holds: %s", node, module, to, strings.Join(r.Open(), ", ")) + } + now := c.now() + w.Closed, w.How = &now, "by hand" + r.Nodes[node] = w + if err := c.Store.HandAct(fmt.Sprintf("config_rename_close %s %s on %s", module, to, node), why, + "the operator dropped a node from the rename window of "+key); err != nil { + return nil, fmt.Errorf("the hand-act log does not hold it, so the window is not dropped: %w", err) + } + if err := c.Store.Put(RenamesState, key, r); err != nil { + return nil, fmt.Errorf("the window is not dropped: %w", err) + } + if len(r.Open()) == 0 { + c.clearFinding(FindingWindowHeld, r.Module, r.To) + } + if _, err := c.reconcileLocked(); err != nil { + return nil, err + } + return map[string]any{"closed": node, "open": r.Open()}, nil +} + +// sweepRenameFindings writes a finding for each window held open past HeldFinding, naming the node, and clears one +// whose window closed. +func (c *Config) sweepRenameFindings(s *Sources, now time.Time) { + for _, key := range sortedKeys(s.Renames) { + r := s.Renames[key] + var held []string + for _, node := range r.Open() { + if now.Sub(r.Nodes[node].Since) >= HeldFinding { + held = append(held, node) + } + } + c.clearFinding(FindingWindowHeld, r.Module, r.To, held...) + for _, node := range held { + c.setFinding(windowHeld(r, node, now)) + } + } +} diff --git a/modules/mesh-config/cmd/mesh-config/renames_test.go b/modules/mesh-config/cmd/mesh-config/renames_test.go new file mode 100644 index 00000000..d3e705b5 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/renames_test.go @@ -0,0 +1,436 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// fakeDelivery is the delivery parts as mesh-config asks them: each node's have, whether a node-engine states what it +// reads, and who reads a module's keys (nil rows and known false: the indexer does not serve the tool yet). +type fakeDelivery struct { + mu sync.Mutex + haves map[string]string // "." → fingerprint + notSays map[string]bool // node → its node-engine states nothing + rows []ReaderRow + known bool +} + +func (d *fakeDelivery) Have(node, module string) (string, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + fp, ok := d.haves[node+"."+module] + return fp, ok, nil +} + +func (d *fakeDelivery) Reads(node string) error { + d.mu.Lock() + defer d.mu.Unlock() + if d.notSays[node] { + return errors.New("the verb manifest-reads is not served") + } + return nil +} + +func (d *fakeDelivery) Readers(string) ([]ReaderRow, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + return d.rows, d.known, nil +} + +// fakeVault records the retirements asked of the seat mesh-vault. +type fakeVault struct{ asked []RetireRequest } + +func (v *fakeVault) Retire(r RetireRequest) error { v.asked = append(v.asked, r); return nil } + +// The fingerprints of the first, second and third build of player. +var ( + fp1 = "src2:" + strings.Repeat("1", 64) + fp2 = "src2:" + strings.Repeat("2", 64) + fp3 = "src2:" + strings.Repeat("3", 64) +) + +const ( + oldPlayer = `{"module":"player","config":{"volume":{"type":"number","default":5}}}` + newPlayer = `{"module":"player","config":{"level":{"type":"number","default":5,"renamed-from":"volume"}}}` + // donePlayer is the manifest once `renamed-from` left it. + donePlayer = `{"module":"player","config":{"level":{"type":"number","default":5}}}` +) + +// aRenamingMesh is a mesh with player on g14 and novox, both running its first build (reading `volume`), a clock the +// test moves, and the delivery parts faked. +func aRenamingMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *time.Time) { + t.Helper() + m := newStore() + m.declare(t, "player", oldPlayer) + indexerList(t, m, "aaaaaaaaaaaa", theList) + for _, node := range []string{"g14", "novox"} { + m.assign(t, node, "player") + m.want(t, node, "player", fp1, oldPlayer) + } + clock := time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC) + d := &fakeDelivery{haves: map[string]string{"g14.player": fp1, "novox.player": fp1}, notSays: map[string]bool{}} + c := &Config{Store: m, Now: func() time.Time { return clock }, Logf: t.Logf, Delivery: d} + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + return c, m, d, &clock +} + +func findingsOf(t *testing.T, m *memStore, kind string) []health.Finding { + t.Helper() + var out []health.Finding + for _, k := range m.keysOf(FindingsState) { + e, _ := m.Get(FindingsState, k) + f, err := health.DecodeFinding(e.Value) + if err != nil { + t.Fatalf("finding %s: %v", k, err) + } + if f.Kind == kind { + out = append(out, f) + } + } + return out +} + +// **A renamed value moves with its trust, both names answer until the node-engines run the new build, and the window +// closes per node** (design 32, C20): the value set under the old name is under the new one with no operator input, +// `renamed` is said once, the old name answers the new value on a node until that node's have runs a build that reads the +// new name (on `delivered`, or for a returning node on `manifest-reads-changed` once its node-engine states what it +// reads), and the rename goes once every window is closed and `renamed-from` left the manifest. +func TestARenamedValueMovesAndTheOldNameAnswersUntilEachNodeRunsTheNewBuild(t *testing.T) { + c, m, d, _ := aRenamingMesh(t) + set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"}) + m.declare(t, "player", newPlayer) + r, err := c.Reconcile() + if err != nil || len(r.Renamed) != 1 { + t.Fatalf("reconcile: %+v, %v", r, err) + } + if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" || v.By == "" { + t.Errorf("the value is %+v under the new name (set: %v)", v, ok) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); ok { + t.Error("the old name still holds a value") + } + for _, c := range []struct{ node, key, want string }{{"g14", "level", "8"}, {"g14", "volume", "8"}, {"novox", "level", "5"}, {"novox", "volume", "5"}} { + if r := m.resolved(t, c.node, "player", c.key); string(r.Value) != c.want { + t.Errorf("%s answers %s on %s as %s, want %s", c.key, "player", c.node, r.Value, c.want) + } + } + if got := m.saidOf(EventRenamed); len(got) != 1 || got[0].(RenamedEvent).From != "volume" || got[0].(RenamedEvent).To != "level" { + t.Errorf("renamed was said as %+v", got) + } + // The history keeps the move; a second pass says and moves nothing more. + if h, _ := c.History(Call{Module: "player", Node: "g14"}); len(h) == 0 || h[len(h)-1].Action != "resolve" && h[len(h)-1].Action != "rename" { + t.Errorf("history %+v", h) + } + if _, err := c.Reconcile(); err != nil || len(m.saidOf(EventRenamed)) != 1 { + t.Errorf("a second pass said renamed %d times: %v", len(m.saidOf(EventRenamed)), err) + } + rows, _ := c.Renames("", true) + if len(rows) != 1 || strings.Join(rows[0].Open, ",") != "g14,novox" || strings.Join(rows[0].Moved, ",") != "g14" { + t.Fatalf("renames %+v", rows) + } + + // A delivery while both nodes still run the first build closes nothing. + if said := c.Windows(""); len(said) != 2 || !strings.Contains(said[0], "still runs a build that reads volume") { + t.Errorf("windows said %v", said) + } + // g14 applies the new build. + m.want(t, "g14", "player", fp2, newPlayer) + d.haves["g14.player"] = fp2 + if said := c.Windows(""); len(said) == 0 { + t.Error("g14 applied the new build and its window did not close") + } + if _, err := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); err != nil { + t.Fatal(err) + } else if e, _ := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); e != nil { + t.Error("g14 still answers the old name after its window closed") + } + m.resolved(t, "novox", "player", "volume") // novox still answers it + // novox applies it, but its node-engine, returning, states nothing yet: the window stays open. + m.want(t, "novox", "player", fp2, newPlayer) + d.haves["novox.player"] = fp2 + d.notSays["novox"] = true + c.Windows("novox") + if rows, _ := c.Renames("", true); len(rows) != 1 || strings.Join(rows[0].Open, ",") != "novox" { + t.Errorf("a node-engine that states nothing closed its window: %+v", rows) + } + d.notSays["novox"] = false + c.Windows("novox") + if rows, _ := c.Renames("", true); len(rows) != 0 { + t.Errorf("windows still open: %+v", rows) + } + // The rename stays while `renamed-from` is in the manifest, and goes when it left. + if all, _ := c.Renames("", false); len(all) != 1 || all[0].Closed["g14"] != "applied" { + t.Errorf("renames %+v", all) + } + m.declare(t, "player", donePlayer) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + if all, _ := c.Renames("", false); len(all) != 0 { + t.Errorf("the rename stayed after renamed-from left the manifest: %+v", all) + } +} + +// A refused move is a finding, cleared once it moved (C4): both names declared, both names holding different values, a +// trusted new name over a value set without the operator's Approve. Nothing is moved, nothing is said. +func TestARefusedRenameIsAFindingAndIsClearedOnceItMoves(t *testing.T) { + both := `{"module":"player","config":{"volume":{"type":"number","default":5},"level":{"type":"number","default":5,"renamed-from":"volume"}}}` + trusted := `{"module":"player","config":{"level":{"type":"number","default":5,"trusted":true,"renamed-from":"volume"}}}` + for _, tc := range []struct { + name, manifest string + both bool + why string + fixed string + }{ + {"both names declared", both, false, "declares both names", newPlayer}, + {"both names hold a value", newPlayer, true, "both names hold a value", newPlayer}, + {"a trusted name over an unapproved value", trusted, false, "without the operator's Approve", newPlayer}, + } { + t.Run(tc.name, func(t *testing.T) { + c, m, _, _ := aRenamingMesh(t) + set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"}) + if tc.both { + // The new name is set meanwhile, to another value, in the state a faulty hand wrote it. + if err := m.Put(ValuesState, "module.g14.player.level", Value{Value: json.RawMessage("9"), By: "x", At: time.Now()}); err != nil { + t.Fatal(err) + } + } + m.declare(t, "player", tc.manifest) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + fs := findingsOf(t, m, FindingRenameRefused) + if len(fs) != 1 || !strings.Contains(fs[0].Summary, tc.why) || fs[0].Observer != Module { + t.Fatalf("findings %+v, want one saying %q", fs, tc.why) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok || len(m.saidOf(EventRenamed)) != 0 { + t.Errorf("a refused rename moved the value or said renamed") + } + // Put right, it moves by hand and the finding goes. + m.declare(t, "player", tc.fixed) + if tc.both { + _ = m.Delete(ValuesState, "module.g14.player.level") + } + if _, err := c.RenameAgain("player", "level"); err != nil { + t.Fatal(err) + } + if fs := findingsOf(t, m, FindingRenameRefused); len(fs) != 0 { + t.Errorf("the finding stayed after the move: %+v", fs) + } + if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" { + t.Errorf("the value is %+v after the move", v) + } + }) + } +} + +// A window held open seven days is a finding naming the node, cleared when it closes; dropping a node by hand is a hand +// act with its why (C4, C5), and the seat's verb `renames` names the nodes holding a window open. +func TestAWindowHeldSevenDaysIsAFindingAndAHandCloseIsAHandAct(t *testing.T) { + c, m, d, clock := aRenamingMesh(t) + m.declare(t, "player", newPlayer) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + m.want(t, "g14", "player", fp2, newPlayer) + d.haves["g14.player"] = fp2 + c.Windows("") + *clock = clock.Add(8 * 24 * time.Hour) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + fs := findingsOf(t, m, FindingWindowHeld) + if len(fs) != 1 || fs[0].Subject.Machine != "novox" || !strings.Contains(fs[0].Summary, "novox") { + t.Fatalf("findings %+v, want one naming novox", fs) + } + // The seat's verb names the node holding the window open. + var verb func(map[string]any) (any, error) + for _, tool := range Tools(c) { + if tool.Name == Seat+"."+RenamesVerb { + verb = tool.Run + } + } + got, err := verb(map[string]any{"module": "player"}) + if raw, _ := json.Marshal(got); err != nil || !strings.Contains(string(raw), `"open":["novox"]`) { + t.Errorf("renames answered %s, %v", raw, err) + } + if _, err := c.RenameClose("player", "level", "novox", ""); err == nil { + t.Error("a window was dropped with no why") + } + if _, err := c.RenameClose("player", "level", "g14", "g14 is closed already"); err == nil { + t.Error("a node holding no open window was dropped") + } + m.handActs = nil + if _, err := c.RenameClose("player", "level", "novox", "novox is retired for good"); err != nil { + t.Fatal(err) + } + if len(m.handActs) != 1 || !strings.Contains(m.handActs[0], "novox is retired for good") { + t.Errorf("hand acts %v", m.handActs) + } + if fs := findingsOf(t, m, FindingWindowHeld); len(fs) != 0 { + t.Errorf("the finding stayed after the window closed: %+v", fs) + } + if rows, _ := c.Renames("", true); len(rows) != 0 { + t.Errorf("open renames %+v", rows) + } +} + +// retiringMesh is a mesh where player no longer declares `volume`, a value of it is set on g14, and g14 runs the first +// build (which reads it). +func retiringMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *standIn, *fakeVault, *time.Time) { + t.Helper() + c, m, d, clock := aRenamingMesh(t) + set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"}) + m.declare(t, "player", donePlayer) + r := newStandIn() + v := &fakeVault{} + c.Asks, c.Vault = r, v + d.known = true + d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{}}} + return c, m, d, r, v, clock +} + +func retiringOf(t *testing.T, c *Config) []RetiringRow { + t.Helper() + got, err := c.ConfigRetiring("", "", "") + if err != nil { + t.Fatal(err) + } + return got.(map[string]any)["retiring"].([]RetiringRow) +} + +// **A value no manifest reads is retired only on the operator's Approve** (C24): while a node runs a build that reads it +// it is not marked and nobody is asked; once none does, `retiring` is said and the operator is asked once; the value stays +// until the Approve, then it is removed, `retired` is said for the node it was set on, and the warrant is recorded. +func TestAValueNoManifestReadsIsRetiredOnlyOnTheOperatorsApprove(t *testing.T) { + c, m, d, r, _, _ := retiringMesh(t) + // The indexer does not serve the tool yet: nothing retires on that. + d.known = false + if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "does not say") || len(retiringOf(t, c)) != 0 { + t.Errorf("an indexer without the tool: %v", said) + } + d.known = true + // g14 still runs a build that reads volume: draining, no ask, no event. + c.Retirements("player") + if rows := retiringOf(t, c); len(rows) != 1 || rows[0].State != RetireDraining || strings.Join(rows[0].Nodes, ",") != "g14,novox" { + t.Fatalf("retiring %+v, want draining on g14 and novox", rows) + } + if len(r.asks) != 0 || len(m.saidOf(EventRetiring)) != 0 { + t.Fatalf("asked %d times and said retiring %d times while two nodes read the value", len(r.asks), len(m.saidOf(EventRetiring))) + } + // Both nodes apply the build that reads it no longer. + for _, node := range []string{"g14", "novox"} { + m.want(t, node, "player", fp3, donePlayer) + d.haves[node+".player"] = fp3 + } + c.Windows("") + rows := retiringOf(t, c) + if len(rows) != 1 || rows[0].State != RetireAsked || len(r.asks) != 1 || len(m.saidOf(EventRetiring)) != 1 { + t.Fatalf("retiring %+v, %d asks, %d events", rows, len(r.asks), len(m.saidOf(EventRetiring))) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok { + t.Fatal("the value was removed before the operator answered") + } + // A second manifest-accepted asks nobody again. + c.Retirements("player") + if len(r.asks) != 1 { + t.Errorf("asked %d times", len(r.asks)) + } + r.choose(t, rows[0].Ask, OptionApprove, nil) + m.handActs = nil + if said, err := c.Decide(); err != nil || len(said) == 0 { + t.Fatalf("decide: %v, %v", said, err) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); ok { + t.Error("the approved value is still set") + } + if got := m.saidOf(EventRetired); len(got) != 1 || got[0].(RetiredEvent).Node != "g14" || got[0].(RetiredEvent).Name != "volume" { + t.Errorf("retired was said as %+v", got) + } + if len(retiringOf(t, c)) != 0 || len(m.handActs) == 0 { + t.Errorf("retiring %+v, hand acts %v", retiringOf(t, c), m.handActs) + } +} + +// A declined retirement is not asked again until the value is read again; a value read again is dropped; an ask unanswered +// is asked once more after seven days and then written as a finding, cleared by an answer or a cancel; a secret is +// retired through the seat mesh-vault with the ask that warranted it (C24, C7). +func TestARetirementDeclinedUnansweredCancelledOrASecret(t *testing.T) { + c, m, d, r, v, clock := retiringMesh(t) + for _, node := range []string{"g14", "novox"} { + m.want(t, node, "player", fp3, donePlayer) + d.haves[node+".player"] = fp3 + } + c.Retirements("player") + ask := retiringOf(t, c)[0].Ask + + // Unanswered: the ask expires, a week passes, it is asked once more, expires again: a finding. + r.choose(t, ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired }) + c.Decide() + if rows := retiringOf(t, c); rows[0].State != RetireUnanswered || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 { + t.Fatalf("after the first expiry: %+v", rows) + } + *clock = clock.Add(8 * 24 * time.Hour) + c.Decide() + rows := retiringOf(t, c) + if rows[0].State != RetireAsked || len(r.asks) != 2 || rows[0].Ask == ask { + t.Fatalf("after a week: %+v, %d asks", rows, len(r.asks)) + } + r.choose(t, rows[0].Ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired }) + c.Decide() + if fs := findingsOf(t, m, FindingRetireUnanswered); len(fs) != 1 { + t.Fatalf("findings %+v after two asks unanswered", fs) + } + // Asked again by hand, then declined: the finding goes and it is not asked again. + if _, err := c.ConfigRetiring("ask-again", "player", "volume"); err != nil { + t.Fatal(err) + } + third := retiringOf(t, c)[0].Ask + r.choose(t, third, OptionDecline, nil) + c.Decide() + if rows := retiringOf(t, c); rows[0].State != RetireDeclined || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 { + t.Fatalf("after the decline: %+v", rows) + } + asked := len(r.asks) + c.Retirements("player") + if len(r.asks) != asked { + t.Error("a declined retirement was asked again") + } + // Read again: dropped. + d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{"player"}}} + if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "read again") || len(retiringOf(t, c)) != 0 { + t.Errorf("read again: %v", said) + } + + // A secret: its nodes come from the indexer's answer; on the Approve the vault is asked for each, once. + d.rows = []ReaderRow{{Module: "player", Key: "token", Kind: KindSecret, Readers: []string{}, Nodes: []string{}}} + c.Retirements("player") + secret := retiringOf(t, c)[0] + if secret.Kind != KindSecret || secret.State != RetireAsked { + t.Fatalf("the secret: %+v", secret) + } + r.choose(t, secret.Ask, OptionApprove, nil) + c.Decide() + if len(v.asked) != 2 || v.asked[0].Node != "g14" || v.asked[0].Name != "token" || v.asked[0].Ask != secret.Ask { + t.Errorf("the vault was asked %+v", v.asked) + } + if len(m.saidOf(EventRetired)) != 2 || len(retiringOf(t, c)) != 0 { + t.Errorf("retired %d times, retiring %+v", len(m.saidOf(EventRetired)), retiringOf(t, c)) + } + // A secret whose readers' nodes the indexer cannot name is not marked. + d.rows = []ReaderRow{{Module: "player", Key: "other", Kind: KindSecret, Readers: []string{}}} + if c.Retirements("player"); len(retiringOf(t, c)) != 0 { + t.Errorf("a secret read on unknown nodes was marked: %+v", retiringOf(t, c)) + } + _ = fmt.Sprint +} diff --git a/modules/mesh-config/cmd/mesh-config/resolve.go b/modules/mesh-config/cmd/mesh-config/resolve.go index f911c225..72f41fd8 100644 --- a/modules/mesh-config/cmd/mesh-config/resolve.go +++ b/modules/mesh-config/cmd/mesh-config/resolve.go @@ -150,6 +150,8 @@ type Sources struct { // for them (pulls.go); neither is resolved. Pulls map[string]PullDecl Kept map[string]KeptValue + // Renames are the renames kept, by `.`, whose open windows answer the old name (renames.go). + Renames map[string]Rename } // Unresolved is a module placed on a node whose values are not resolved, and why. @@ -192,6 +194,7 @@ func Resolve(s Sources) map[string]Resolved { out[ResolvedKey(a.Node, a.Module, key)] = r } } + resolveOldNames(s, out) return out } diff --git a/modules/mesh-config/cmd/mesh-config/retiring.go b/modules/mesh-config/cmd/mesh-config/retiring.go new file mode 100644 index 00000000..e854737c --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/retiring.go @@ -0,0 +1,581 @@ +package main + +// A value no longer read (novox/hq ADR 0609, design 32 *A value no longer read*). +// +// A config value or secret that no manifest on any trunk reads any more is a **retiring value**. mesh-config learns it +// from the indexer's tool `indexer_config_readers`, asked on each of the indexer's `manifest-accepted` (Retirements); +// a tool the indexer does not serve yet answers "not known" and nothing retires. A value is marked retiring only once no +// node-engine reports applied a build that reads it (State draining until then, judged again on each `delivered`); +// mesh-config then says `retiring` and asks the operator once, through the operator channel, whether to remove it. It is +// removed only on the operator's Approve: a value by mesh-config, a secret through the seat `mesh-vault`'s verb +// `retire`. An ask unanswered after seven days is asked once more, and then written as a finding. + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// The state, and where a retirement stands. +const ( + RetiringState = "retiring" + // RetireVerb is the act an answer binds. + RetireVerb = Seat + ".retire" + + RetireDraining = "draining" // a node still runs a build that reads it + RetireAsked = "asked" // the operator is asked + RetireUnanswered = "unanswered" // the ask ended with no choice + RetireApproved = "approved" // the operator approved; being removed + RetireDeclined = "declined" // the operator declined, or a person cancelled: kept until the value is read again +) + +// Kinds of retiring value. +const ( + KindValue = "value" + KindSecret = "secret" +) + +// Retiring is one retiring value, as `retiring` keeps it, key `.`. +type Retiring struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Since time.Time `json:"since"` + State string `json:"state"` + // Nodes are the nodes that still read it (draining) or that it is removed on (asked, approved). + Nodes []string `json:"nodes,omitempty"` + // Ask is the open ask's id, Asked when the first ask was made, AskedAgain whether it was asked once more. + Ask string `json:"ask,omitempty"` + // AskedAt is when the open ask was made (it expires a day on), Asked when the first was. + AskedAt time.Time `json:"asked-at,omitempty"` + Asked time.Time `json:"asked,omitempty"` + AskedAgain bool `json:"asked-again,omitempty"` + Said bool `json:"said,omitempty"` + Outcome string `json:"outcome,omitempty"` + Answered string `json:"answered,omitempty"` + // Removed are the nodes it is removed on already. + Removed []string `json:"removed,omitempty"` +} + +func retiringKey(module, key string) string { return module + "." + key } + +// RetiringEvent is the body of `retiring`; RetiredEvent of `retired`. +type RetiringEvent struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Since time.Time `json:"since"` + Ask string `json:"ask"` +} + +// RetiredEvent is the body of `retired`: one node's value removed, with the ask that warranted it. +type RetiredEvent struct { + Module string `json:"module"` + Name string `json:"name"` + Node string `json:"node"` + Ask string `json:"ask"` +} + +// ReaderRow is one key the indexer's `indexer_config_readers` answers: the modules on each trunk that declare or refer +// to it. For a secret, Nodes are the nodes whose applied build reads it, when the indexer can say. +type ReaderRow struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Readers []string `json:"readers"` + Nodes []string `json:"nodes,omitempty"` +} + +// Delivery is what the delivery parts answer to this module: a node's have, a node-engine's statement of what it +// reads, and who reads a module's config keys. Each is a tool or a seat verb on the bus, never a bucket (ADR 0596). +type Delivery interface { + // Have is the fingerprint of the build node runs of module, as the status view's node_haves says it. + Have(node, module string) (fp string, found bool, err error) + // Reads asks the node-engine seat's manifest-reads for node: an error when it is not served or the node says nothing. + Reads(node string) error + // Readers asks the indexer who reads module's keys: known false when the indexer does not serve the tool yet. + Readers(module string) (rows []ReaderRow, known bool, err error) +} + +// RetireRequest is what the vault's verb `retire` takes: one module's own secret on a node, with the hand act that +// warrants it. +type RetireRequest struct { + Node, Module, Name, Ask string +} + +// Vault is the seat mesh-vault as this module reaches it. +type Vault interface { + Retire(r RetireRequest) error +} + +func (c *Config) readRetiring() (map[string]Retiring, error) { + out := map[string]Retiring{} + err := c.each(RetiringState, func(key string, raw []byte) error { + var r Retiring + if json.Unmarshal(raw, &r) != nil || r.Module == "" || retiringKey(r.Module, r.Key) != key { + return nil + } + out[key] = r + return nil + }) + return out, err +} + +// Retirements judges a module's keys on the indexer's word that its manifest was accepted. It answers a line for each +// decision. +func (c *Config) Retirements(module string) []string { + c.mu.Lock() + defer c.mu.Unlock() + if c.Delivery == nil || module == "" { + return nil + } + rows, known, err := c.Delivery.Readers(module) + if err != nil { + return []string{fmt.Sprintf("who reads %s's config is not asked yet: %v", module, err)} + } + if !known { + return []string{fmt.Sprintf("the indexer does not say who reads %s's config yet; nothing retires on that", module)} + } + held, err := c.readRetiring() + if err != nil { + return []string{"the retirements cannot be read: " + err.Error()} + } + s, err := c.sources() + if err != nil { + return []string{"the retirements cannot be judged: " + err.Error()} + } + var said []string + for _, row := range rows { + if row.Module != "" && row.Module != module { + continue + } + row.Module = module + if row.Kind == "" { + row.Kind = KindValue + } + key := retiringKey(module, row.Key) + e, has := held[key] + if len(row.Readers) > 0 { + if has { + said = append(said, c.dropRetiring(e, "it is read again")) + } + continue + } + if has { + continue // already marked; it is advanced by drainRetirements and Decide + } + if row.Kind == KindValue && !valueHeld(s, module, row.Key) { + continue // nothing set under that name: nothing to remove + } + if row.Kind == KindSecret && row.Nodes == nil && !row.nodesKnown() { + continue // which nodes read a secret is not known, and nothing retires on that + } + e = Retiring{Module: module, Key: row.Key, Kind: row.Kind, Since: c.now(), State: RetireDraining, Nodes: row.Nodes} + said = append(said, c.advanceRetiring(s, &e)...) + } + return said +} + +func (r ReaderRow) nodesKnown() bool { return r.Nodes != nil } + +// valueHeld is whether a module's own value is set under key on any node. +func valueHeld(s Sources, module, key string) bool { + for k := range s.Values { + if l, kk, ok := parseValueKey(k); ok && l.Kind == LevelModule && l.Module == module && kk == key { + return true + } + } + return false +} + +// nodesReading is the nodes whose applied build still reads a value (secrets: the nodes the indexer last named). +func (c *Config) nodesReading(s Sources, e Retiring) []string { + if e.Kind != KindValue || c.Delivery == nil { + return e.Nodes + } + var out []string + for _, node := range nodesPlaced(&s, e.Module) { + fp, found, err := c.Delivery.Have(node, e.Module) + if err != nil || !found || fp == "" { + if err != nil { + out = append(out, node) // not known is not "no longer read" + } + continue + } + d, err := c.builtDeclarations(e.Module, fp) + if err != nil { + out = append(out, node) + continue + } + if _, reads := d.Config[e.Key]; reads { + out = append(out, node) + } + } + sort.Strings(out) + return out +} + +// advanceRetiring moves a draining retirement on: while a node reads the value it stays draining (and is written, so a +// delivery judges it again); once none does it is marked retiring, said, and asked. The caller holds the lock. +func (c *Config) advanceRetiring(s Sources, e *Retiring) []string { + key := retiringKey(e.Module, e.Key) + now := c.now() + if e.State == RetireDraining { + e.Nodes = c.nodesReading(s, *e) + if len(e.Nodes) > 0 { + if err := c.Store.Put(RetiringState, key, *e); err != nil { + return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)} + } + if now.Sub(e.Since) >= HeldFinding { + c.setFinding(retirementHeld(*e, now)) + } + return []string{fmt.Sprintf("%s's %s is not read by any manifest, and %s still run a build that reads it", + e.Module, e.Key, strings.Join(e.Nodes, ", "))} + } + c.clearFinding(FindingRetireHeld, e.Module, e.Key) + e.Nodes = nodesPlaced(&s, e.Module) + if err := c.ask(e, false); err != nil { + return []string{fmt.Sprintf("the operator is not asked about retiring %s's %s: %v", e.Module, e.Key, err)} + } + } + if !e.Said { + if err := c.Store.Emit(EventRetiring, RetiringEvent{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since, + Ask: e.Ask}); err == nil { + e.Said = true + } + } + if err := c.Store.Put(RetiringState, key, *e); err != nil { + return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)} + } + return []string{fmt.Sprintf("%s's %s is retiring: the operator is asked whether to remove it", e.Module, e.Key)} +} + +// drainRetirements judges the draining retirements again, after a delivery or a returning node; the caller holds the lock. +func (c *Config) drainRetirements(string) []string { + held, err := c.readRetiring() + if err != nil || len(held) == 0 || c.Delivery == nil { + return nil + } + s, err := c.sources() + if err != nil { + return []string{"the retirements cannot be judged: " + err.Error()} + } + var said []string + for _, key := range sortedKeys(held) { + e := held[key] + if e.State == RetireDraining { + said = append(said, c.advanceRetiring(s, &e)...) + } + } + return said +} + +// dropRetiring ends a retirement: the value is read again, or a person cancelled it. +func (c *Config) dropRetiring(e Retiring, why string) string { + if err := c.Store.Delete(RetiringState, retiringKey(e.Module, e.Key)); err != nil { + return fmt.Sprintf("the retirement of %s's %s is not dropped: %v", e.Module, e.Key, err) + } + c.clearFinding(FindingRetireUnanswered, e.Module, e.Key) + c.clearFinding(FindingRetireHeld, e.Module, e.Key) + return fmt.Sprintf("%s's %s is no longer retiring: %s", e.Module, e.Key, why) +} + +// ---- the ask ------------------------------------------------------------------------------------------------ + +// retireAskID is an ask's id: the same retirement asked again is another id. +func retireAskID(e Retiring, again bool) string { + sum := sha256.Sum256([]byte(e.Module + "\x00" + e.Key + "\x00" + e.Since.UTC().Format(time.RFC3339Nano))) + id := "ret-" + hex.EncodeToString(sum[:6]) + if again { + id += "-2" + } + return id +} + +// act is what an answer binds: the retirement, the nodes it reaches, the ask and the answer. +func (e Retiring) act(ask, option string) asks.Act { + return asks.Act{"verb": RetireVerb, "option": option, "module": e.Module, "key": e.Key, "kind": e.Kind, + "nodes": strings.Join(e.Nodes, " "), "ask": ask} +} + +// retireAsk is the ask composed from the retirement alone. +func (e Retiring) retireAsk(id string, at time.Time) asks.Ask { + nodes := strings.Join(e.Nodes, ", ") + if nodes == "" { + nodes = "no node" + } + what := "config value" + if e.Kind == KindSecret { + what = "secret" + } + a := asks.Ask{ID: id, Headline: fmt.Sprintf("Remove %s's %s %s?", e.Module, what, e.Key), + Explanation: fmt.Sprintf("No manifest reads %s's %s %s any more, and no node runs a build that reads it. Approved, it is removed on %s.", e.Module, what, e.Key, nodes), + Who: asks.Operator, Expires: at.Add(ProposalLasts), OnExpiry: "nothing is removed; mesh-config asks once more in a week", + About: "config.retire." + e.Module + "." + e.Key, + Details: fmt.Sprintf("Retiring since %s UTC. Read it with mesh-config's config_retiring.", e.Since.UTC().Format("15:04 on 2 Jan"))} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "the " + what + " is removed on " + nodes}, + {OptionDecline, "Decline", "nothing is removed; mesh-config asks no more until it is read again"}, + } { + binds, _ := asks.ActDigest(e.act(id, o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// ask makes the ask for a retirement and publishes it; the caller keeps the entry. +func (c *Config) ask(e *Retiring, again bool) error { + if c.Asks == nil { + return fmt.Errorf("the operator cannot be asked from here") + } + now := c.now() + id := retireAskID(*e, again) + a := e.retireAsk(id, now) + if err := a.Check(now); err != nil { + return err + } + if err := c.Asks.Publish(asks.AskSubject(Module), a, a.ID); err != nil { + return err + } + e.Ask, e.State, e.AskedAt = id, RetireAsked, now + if e.Asked.IsZero() { + e.Asked = now + } + e.AskedAgain = e.AskedAgain || again + return nil +} + +// decideRetirements takes the router's word on each open retirement ask; the caller holds the lock. It also asks once +// more an ask unanswered for seven days, writes the finding of one asked twice, and finishes an approved removal. +func (c *Config) decideRetirements() []string { + held, err := c.readRetiring() + if err != nil || len(held) == 0 { + return nil + } + now := c.now() + var said []string + for _, key := range sortedKeys(held) { + e := held[key] + switch e.State { + case RetireAsked: + said = append(said, c.decideRetirement(&e, now)...) + case RetireUnanswered: + if !e.AskedAgain && now.Sub(e.Asked) >= HeldFinding { + if err := c.ask(&e, true); err != nil { + said = append(said, fmt.Sprintf("the operator is not asked again about %s's %s: %v", e.Module, e.Key, err)) + } else { + _ = c.Store.Put(RetiringState, key, e) + said = append(said, fmt.Sprintf("%s's %s: the operator is asked once more", e.Module, e.Key)) + } + } + case RetireApproved: + said = append(said, c.finishRetirement(&e)...) + } + } + return said +} + +func (c *Config) decideRetirement(e *Retiring, now time.Time) []string { + if c.Asks == nil { + return nil + } + key := retiringKey(e.Module, e.Key) + rec, err := c.Asks.Record(e.Ask) + if err != nil || rec == nil { + return nil + } + var r routerRecord + if json.Unmarshal(rec.Value, &r) != nil || r.State == "open" || r.Warrant == nil { + return nil + } + w := *r.Warrant + keep := func(state, outcome string) []string { + e.State, e.Outcome = state, outcome + if err := c.Store.Put(RetiringState, key, *e); err != nil { + return []string{fmt.Sprintf("%s's %s: %s, and that is not kept: %v", e.Module, e.Key, outcome, err)} + } + return []string{fmt.Sprintf("%s's %s: %s", e.Module, e.Key, outcome)} + } + if w.Asker != Module || w.Ask != e.Ask { + return keep(RetireUnanswered, "the router's record is for another ask; nothing is removed") + } + if w.Outcome != asks.OutcomeChosen { + out := keep(RetireUnanswered, "the ask "+string(w.Outcome)+"; nothing is removed") + if e.AskedAgain { + c.setFinding(retireUnanswered(*e, now)) + } + return out + } + option, err := w.For(Module, e.retireAsk(e.Ask, e.AskedAt)) + if err == nil { + err = option.Performs(e.act(e.Ask, option.ID)) + } + if err != nil { + c.logf("[mesh-config] REFUSED the answer to the retirement ask %s: %v", e.Ask, err) + return keep(RetireUnanswered, "the answer is refused and nothing is removed: "+err.Error()) + } + c.clearFinding(FindingRetireUnanswered, e.Module, e.Key) + e.Answered = w.Says() + switch option.ID { + case OptionDecline: + return keep(RetireDeclined, "declined by the operator ("+w.Says()+"); nothing is removed") + case OptionApprove: + out := keep(RetireApproved, "approved by the operator; being removed") + return append(out, c.finishRetirement(e)...) + } + return keep(RetireUnanswered, "the option "+option.ID+" is not one a retirement offers") +} + +// finishRetirement removes an approved retirement on each node it is not removed on yet: a value here, a secret through +// the vault. Each removal is said (`retired`); the entry goes when every node is done. The caller holds the lock. +func (c *Config) finishRetirement(e *Retiring) []string { + key := retiringKey(e.Module, e.Key) + var said, failed []string + for _, node := range e.Nodes { + if slices.Contains(e.Removed, node) { + continue + } + var err error + switch e.Kind { + case KindValue: + var removed bool + if removed, err = c.removeValue(*e, node); err == nil && !removed { + e.Removed = append(e.Removed, node) // nothing was set there + continue + } + default: + if c.Vault == nil { + err = fmt.Errorf("the seat mesh-vault is not reachable from here") + } else { + err = c.Vault.Retire(RetireRequest{Node: node, Module: e.Module, Name: e.Key, Ask: e.Ask}) + } + } + if err != nil { + failed = append(failed, node+": "+err.Error()) + continue + } + e.Removed = append(e.Removed, node) + if err := c.Store.Emit(EventRetired, RetiredEvent{Module: e.Module, Name: e.Key, Node: node, Ask: e.Ask}); err != nil { + c.logf("[mesh-config] retired for %s's %s on %s is not said: %v", e.Module, e.Key, node, err) + } + said = append(said, fmt.Sprintf("%s's %s is removed on %s", e.Module, e.Key, node)) + } + if len(failed) > 0 { + e.Outcome = "not removed everywhere yet: " + strings.Join(failed, "; ") + _ = c.Store.Put(RetiringState, key, *e) + return append(said, fmt.Sprintf("%s's %s: %s", e.Module, e.Key, e.Outcome)) + } + if err := c.Store.Delete(RetiringState, key); err != nil { + return append(said, fmt.Sprintf("the retirement %s is not dropped: %v", key, err)) + } + if _, err := c.reconcileLocked(); err != nil { + said = append(said, "resolving after the removal: "+err.Error()) + } + return said +} + +// removeValue deletes a module's own value on a node, recording it in the history, the hand-act log and config-changed. +func (c *Config) removeValue(e Retiring, node string) (bool, error) { + vk := moduleValueKey(node, e.Module, e.Key) + got, err := c.Store.Get(ValuesState, vk) + if err != nil { + return false, err + } + if got == nil { + return false, nil // nothing set there, or gone already + } + var was Value + _ = json.Unmarshal(got.Value, &was) + if err := c.Store.Delete(ValuesState, vk); err != nil { + return false, err + } + lvl := Level{Kind: LevelModule, Node: node, Module: e.Module} + ch := Change{Action: "retire", Level: &lvl, Key: e.Key, Was: was.Value, By: "the operator", Via: Module, + Why: "no manifest reads it any more (retiring), removed on the operator's Approve of ask " + e.Ask, At: c.now(), + Approved: e.Answered} + c.record(vk, ch) + if err := c.Store.HandAct(handActWhat(ch), ch.Why, "the operator's Approve of ask "+e.Ask); err != nil { + c.logf("[mesh-config] the hand-act log does not hold the retirement of %s: %v", vk, err) + } + return true, nil +} + +// ---- the tool ----------------------------------------------------------------------------------------------- + +// RetiringRow is one retirement as the tool says it. +type RetiringRow struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Since time.Time `json:"since"` + State string `json:"state"` + Nodes []string `json:"nodes,omitempty"` + Ask string `json:"ask,omitempty"` + Outcome string `json:"outcome,omitempty"` + Answered string `json:"answered,omitempty"` +} + +// ConfigRetiring lists what is retiring (module narrows), or with action ask-again asks the operator again about one, +// or with cancel drops one (kept apart so it is not asked again until the value is read again). +func (c *Config) ConfigRetiring(action, module, key string) (any, error) { + c.mu.Lock() + defer c.mu.Unlock() + held, err := c.readRetiring() + if err != nil { + return nil, err + } + switch action { + case "", "list": + rows := []RetiringRow{} + for _, k := range sortedKeys(held) { + e := held[k] + if module != "" && e.Module != module { + continue + } + rows = append(rows, RetiringRow{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since, State: e.State, + Nodes: e.Nodes, Ask: e.Ask, Outcome: e.Outcome, Answered: e.Answered}) + } + return map[string]any{"retiring": rows}, nil + case "ask-again", "cancel": + e, ok := held[retiringKey(module, key)] + if !ok { + return nil, fmt.Errorf("%s's %s is not retiring: config_retiring lists what is", module, key) + } + if action == "cancel" { + e.State, e.Outcome = RetireDeclined, "cancelled by hand" + if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil { + return nil, err + } + c.clearFinding(FindingRetireUnanswered, e.Module, e.Key) + if err := c.Store.HandAct("config_retiring cancel "+module+" "+key, "a retirement cancelled by hand", "config_retiring"); err != nil { + c.logf("[mesh-config] the hand-act log does not hold the cancel: %v", err) + } + return map[string]any{"said": module + "'s " + key + " is no longer asked about until it is read again"}, nil + } + if e.State == RetireDraining || e.State == RetireApproved { + return nil, fmt.Errorf("%s's %s is %s: there is nothing to ask again", module, key, e.State) + } + s, err := c.sources() + if err != nil { + return nil, err + } + if nodes := c.nodesReading(s, e); len(nodes) > 0 { + return nil, fmt.Errorf("%s still run a build that reads %s's %s", strings.Join(nodes, ", "), module, key) + } + if err := c.ask(&e, true); err != nil { + return nil, err + } + if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil { + return nil, err + } + return map[string]any{"said": "the operator is asked again", "ask": e.Ask}, nil + } + return nil, fmt.Errorf("config_retiring lists, or does ask-again or cancel; %q is none of them", action) +} diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index e8286bef..9a0620e0 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -17,7 +17,7 @@ import ( // from its own manifests as built (issue 1953, ADR 0565 §3); the controller's records are no longer read. func (c *Config) sources() (Sources, error) { s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{}, - Predefined: map[string]Published{}, Values: map[string]Value{}} + Predefined: map[string]Published{}, Values: map[string]Value{}, Renames: map[string]Rename{}} var problems []string if err := c.readPlacement(&s); err != nil { return s, fmt.Errorf("the seat mesh-placement's placement cannot be read: %w", err) @@ -51,6 +51,11 @@ func (c *Config) sources() (Sources, error) { if err := c.readPulls(&s); err != nil { return s, err } + if renames, err := c.readRenames(); err != nil { + return s, fmt.Errorf("the renames cannot be read: %w", err) + } else { + s.Renames = renames + } for _, p := range problems { c.logf("[mesh-config] a placed module's declarations are not read, so its values are not resolved (they "+ "stay as last resolved): %s", p) @@ -354,6 +359,8 @@ type Resolution struct { // did after the take-over. Declared []string `json:"declared,omitempty"` Candidates []string `json:"candidates,omitempty"` + // Renamed is what the renames did in this pass. + Renamed []string `json:"renamed,omitempty"` // SourceGone is said while the stored switch still names the controller, whose records are no longer read. SourceGone string `json:"source_gone,omitempty"` } @@ -364,6 +371,11 @@ type Resolution struct { func (c *Config) Reconcile() (Resolution, error) { c.mu.Lock() defer c.mu.Unlock() + return c.reconcileLocked() +} + +// reconcileLocked is Reconcile with the lock held. +func (c *Config) reconcileLocked() (Resolution, error) { var r Resolution published, err := c.publish() if err != nil { @@ -374,6 +386,8 @@ func (c *Config) Reconcile() (Resolution, error) { if err != nil { return r, err } + // The renames move their values first, so the same pass resolves them under the new names (renames.go). + r.Renamed = c.applyRenames(&s) moved, err := c.writeResolved(s) r.Moved = moved if len(moved) > 0 { diff --git a/modules/mesh-config/cmd/mesh-config/switch_tools.go b/modules/mesh-config/cmd/mesh-config/switch_tools.go index 51475b1f..f4de6e73 100644 --- a/modules/mesh-config/cmd/mesh-config/switch_tools.go +++ b/modules/mesh-config/cmd/mesh-config/switch_tools.go @@ -11,12 +11,18 @@ import ( // The switch's tools. const ( - DeclaredTool = "config_declared" - ResyncTool = "config_resync" + DeclaredTool = "config_declared" + ResyncTool = "config_resync" + RenamesTool = "config_renames" + RenameAgainTool = "config_rename_again" + RenameCloseTool = "config_rename_close" + RetiringTool = "config_retiring" + RenamesVerb = "renames" ) -// SeatVerbs are the seat's verbs, as the manifest claims them. -var SeatVerbs = []string{Verb} +// SeatVerbs are the seat's verbs, as the manifest claims them: `config`, and the optional `renames` the checker asks +// (design 32: the open renames and the nodes holding each window open). +var SeatVerbs = []string{Verb, RenamesVerb} func switchTools(c *Config) []stdio.Tool { return ownSwitchTools(c) } @@ -35,5 +41,48 @@ func ownSwitchTools(c *Config) []stdio.Tool { Description: "Read every source again and write what differs: the published list of predefined keys, the " + "resolved values, declared, and the indexer's config candidates.", Run: func(map[string]any) (any, error) { return c.Resync() }}, + {Name: RenamesTool, + Description: "Each config value renamed (renamed-from): from which name to which, when, the nodes its value " + + "moved on, the nodes still holding the old name open (the rename window) and how each closed one closed. " + + "Narrowed by module.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + rows, err := c.Renames(strings.TrimSpace(m), false) + return map[string]any{"renames": rows}, err + }}, + {Name: RenameAgainTool, + Description: "Move a renamed value by hand, the way a resolution does: after a refused move (its finding says " + + "why) once the cause is put right, or for a value set under the old name meanwhile.", + Input: map[string]any{"type": "object", "required": []string{"module", "to"}, "properties": map[string]any{ + "module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + to, _ := a["to"].(string) + return c.RenameAgain(strings.TrimSpace(m), strings.TrimSpace(to)) + }}, + {Name: RenameCloseTool, + Description: "Drop a node from a rename's window by hand (a node retired for good), recorded as a hand act: " + + "the old name is no longer answered for it.", + Input: map[string]any{"type": "object", "required": []string{"module", "to", "node", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"}, + "node": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}}, + Run: func(a map[string]any) (any, error) { + str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) } + return c.RenameClose(str("module"), str("to"), str("node"), str("why")) + }}, + {Name: RetiringTool, + Description: "The config values and secrets no manifest reads any more: since when, where each stands (draining " + + "while a node still runs a build that reads it, asked, unanswered, approved, declined) and its ask. With " + + "action ask-again, ask the operator once more about one; with cancel, drop one (it is asked about no more " + + "until it is read again).", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "action": map[string]any{"type": "string", "enum": []string{"list", "ask-again", "cancel"}}, + "module": map[string]any{"type": "string"}, "key": map[string]any{"type": "string"}}}, + Run: func(a map[string]any) (any, error) { + str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) } + return c.ConfigRetiring(str("action"), str("module"), str("key")) + }}, } } diff --git a/modules/mesh-config/cmd/mesh-config/whole_integration_test.go b/modules/mesh-config/cmd/mesh-config/whole_integration_test.go new file mode 100644 index 00000000..4aeb182b --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/whole_integration_test.go @@ -0,0 +1,790 @@ +//go:build integration + +package main + +// mesh-config whole against a real bus (design 32, C20 and C24): the built binary, played over its real stdin and stdout +// as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real nats-server's key-value buckets, +// watches handed over as the runtime hands them, mesh/event delivering the events it subscribed to, mesh/ask answered +// as the status view, the indexer, the node-engine seat and mesh-hand-acts answer it, mesh/seat.* as the router does). +// +// A renamed value moves with no input, keeps its value, and both names answer until the node-engine reports the new +// build on `delivered`; the module restarted inside the window keeps the window and says `renamed` once; a refused move +// writes its finding. A value no manifest reads is not asked about while a node reads it, is asked once when none does, +// stays until the operator's Approve and is then removed, `retired` said. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/meshconfig" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as mesh-config needs it, over a real bus. +type player struct { + t *testing.T + ctx context.Context + js jetstream.JetStream + router *standIn + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire + // haves is the status view's node_haves, by "."; readers the indexer's answer. + haves map[string]string + readers []ReaderRow + said []saidOnBus + subs []string + // owned is each delivery owner's state as the owner's tool answers it: its values and the revision of its last change. + owned map[string]*ownedState +} + +// ownedState is one state of an owner (the indexer's modules, the assigner's wants...), kept from its bucket. +type ownedState struct { + owner, state string + revision uint64 + values map[string]json.RawMessage +} + +var theCreated = time.Date(2026, 11, 4, 9, 0, 0, 0, time.UTC) + +// own follows the bucket of an owner's state, and says each change after the first listing on the owner's views family +// to the module (once it subscribed), as the owner does (ADR 0615 §4). +func (p *player) own(read string) { + p.t.Helper() + owner, state, _ := strings.Cut(read, ".") + kv, err := p.kv(read) + if err != nil { + p.t.Fatal(err) + } + w, err := kv.WatchAll(p.ctx) + if err != nil { + p.t.Fatal(err) + } + st := &ownedState{owner: owner, state: state, values: map[string]json.RawMessage{}} + p.owned[read] = st + listed := make(chan struct{}) + go func() { + current := true + for e := range w.Updates() { + if e == nil { + current = false + close(listed) + continue + } + p.mu.Lock() + deleted := e.Operation() != jetstream.KeyValuePut + if deleted { + delete(st.values, e.Key()) + } else { + st.values[e.Key()] = append(json.RawMessage(nil), e.Value()...) + } + if !current { + st.revision++ + ev := map[string]any{"v": 1, "owner": owner, "state": state, "key": e.Key(), "revision": st.revision, "created": theCreated} + if deleted { + ev["deleted"] = true + } else { + ev["value"] = json.RawMessage(e.Value()) + } + p.next++ + id := fmt.Sprintf("o-%d", p.next) + body, _ := json.Marshal(ev) + raw, _ := json.Marshal(map[string]any{"id": id, "jsonrpc": "2.0", "method": "mesh/event", "params": map[string]any{ + "envelope": map[string]any{"key": owner + ".views." + state, "node": "novox", "body": json.RawMessage(body)}}}) + _, _ = p.in.Write(append(raw, '\n')) + } + p.mu.Unlock() + } + }() + select { + case <-listed: + case <-time.After(10 * time.Second): + p.t.Fatalf("%s was not listed", read) + } +} + +// ownerAnswer is an owner's read tool asked with values: the state whole at its revision. +func (p *player) ownerAnswer(key string) (any, bool) { + for read, st := range p.owned { + if key == st.owner+"."+toolOf(st) { + _ = read + return map[string]any{"v": 1, "owner": st.owner, "state": st.state, "revision": st.revision, "created": theCreated, + "values": st.values, "at": time.Now()}, true + } + } + return nil, false +} + +func toolOf(st *ownedState) string { + prefix := "indexer_read_" + if st.owner == "mesh-delivery-assigner" { + prefix = "assigner_read_" + } + return prefix + strings.ReplaceAll(st.state, "-", "_") +} + +type saidOnBus struct { + Key string + Body json.RawMessage +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, read := strings.Cut(state, "."); read { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + Pattern string `json:"pattern"` + ID string `json:"id"` + Bucket string `json:"bucket"` + } + _ = json.Unmarshal(m.Params, &a) + kv, kvErr := p.kv(a.State) + switch m.Method { + case "mesh/state.get": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.create", "mesh/state.update", "mesh/state.put": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + var rev uint64 + var err error + switch m.Method { + case "mesh/state.create": + rev, err = kv.Create(p.ctx, a.Key, a.Value) + case "mesh/state.update": + rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision) + default: + rev, err = kv.Put(p.ctx, a.Key, a.Value) + } + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}}) + return + } + p.answer(m.ID, map[string]any{"revision": rev}, err) + case "mesh/state.keys": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + err := kvErr + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/state.watch": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + w, err := kv.WatchAll(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watchID string + _ = json.Unmarshal(m.ID, &watchID) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID, + "change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())), + "revision": e.Revision(), "current": current}}}) + } + case "mesh/log.append": + p.answer(m.ID, map[string]any{"sequence": 1}, nil) + case "mesh/log.read": + p.answer(m.ID, map[string]any{"entries": []any{}, "next": 0, "last": 0}, nil) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.mu.Lock() + p.said = append(p.said, saidOnBus{e.Key, e.Body}) + p.mu.Unlock() + p.answer(m.ID, map[string]any{}, nil) + case "mesh/subscribe": + p.mu.Lock() + p.subs = append(p.subs, a.Pattern) + p.mu.Unlock() + p.answer(m.ID, map[string]any{}, nil) + case "mesh/seat.publish": + var ask asks.Ask + if err := json.Unmarshal(a.Body, &ask); err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"sequence": 1}, p.router.Publish(a.Subject, ask, a.ID)) + case "mesh/seat.record": + e, err := p.router.Record(a.Key) + if e == nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, e, err) + case "mesh/seat.kinds": + p.answer(m.ID, p.router.kinds, nil) + case "mesh/ask": + var q struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &q) + p.answer(m.ID, p.asked(q.Key, q.Body), nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +// asked answers a tool or a seat verb mesh-config asks, as its holder does. +func (p *player) asked(key string, body json.RawMessage) any { + p.mu.Lock() + defer p.mu.Unlock() + if a, ok := p.ownerAnswer(key); ok { + return a + } + switch { + case key == HavesAsk: + var q struct{ Node, Module string } + _ = json.Unmarshal(body, &q) + hk, _ := delivery.HaveKey(q.Node, q.Module) + keys := map[string]any{} + if fp, ok := p.haves[q.Node+"."+q.Module]; ok { + keys[hk] = delivery.Have{V: delivery.Version, FP: fp, Phase: "applied", Health: "healthy"} + } + return map[string]any{"keys": keys} + case strings.HasPrefix(key, ManifestReadsAsk): + return map[string]any{} + case key == ConfigReadersAsk: + return map[string]any{"readers": p.readers} + case key == "seat:mesh-hand-acts.hand-act" || key == "seat:mesh-hand-acts.warranted": + return map[string]any{"said": "recorded"} + } + return map[string]any{"error": "no responders available for " + key} +} + +func nullIfEmpty(b []byte) []byte { + if len(b) == 0 { + return []byte("null") + } + return b +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// event hands the module one event it subscribed to, as the runtime does, and waits for its answer. +func (p *player) event(key string, body any) { + p.t.Helper() + raw, _ := json.Marshal(body) + p.mu.Lock() + p.next++ + id := fmt.Sprintf("e-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{"key": key, "node": "novox", + "body": json.RawMessage(raw)}}}) + select { + case m := <-ch: + if m.Error != nil { + p.t.Fatalf("%s was not taken: %s", key, m.Error.Message) + } + case <-time.After(30 * time.Second): + p.t.Fatalf("%s was not answered in 30s", key) + } +} + +// conforms decodes what the module published under key strictly into the seat definition's type (a field the type does +// not know is a fault here, though a reader drops it) and checks the subject the runtime publishes it on is the one the +// definition names: CAT-V4, the definition's fixtures matched by a real event. +func (p *player) conforms(key string, into any, subject func(holder string) string) { + p.t.Helper() + p.mu.Lock() + defer p.mu.Unlock() + found := false + for _, s := range p.said { + if s.Key != key { + continue + } + found = true + dec := json.NewDecoder(strings.NewReader(string(s.Body))) + dec.DisallowUnknownFields() + if err := dec.Decode(into); err != nil { + p.t.Errorf("the published %s %s does not match the seat's definition: %v", key, s.Body, err) + } + if got, want := "mesh.mod."+Module+".event."+s.Key, subject(Module); got != want { + p.t.Errorf("%s is published on %s, the definition names %s", key, got, want) + } + } + if !found { + p.t.Errorf("%s was never published", key) + } +} + +func (p *player) publishedCount(key string) int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, s := range p.said { + if s.Key == key { + n++ + } + } + return n +} + +func (p *player) put(state, key string, v any) { + p.t.Helper() + kv, err := p.kv(state) + if err != nil { + p.t.Fatal(err) + } + raw, _ := json.Marshal(v) + if _, err := kv.Put(p.ctx, key, raw); err != nil { + p.t.Fatal(err) + } +} + +func (p *player) get(state, key string) ([]byte, bool) { + kv, err := p.kv(state) + if err != nil { + return nil, false + } + e, err := kv.Get(p.ctx, key) + if err != nil { + return nil, false + } + return e.Value(), true +} + +// viewsReady waits until every first view of the module is answered (config_first_views). +func (p *player) viewsReady() { + p.t.Helper() + eventually(p.t, "every first view is answered", func() bool { + raw, err := p.call(FirstViewsTool, nil) + if err != nil { + return false + } + var a struct { + Views []struct { + Revision uint64 `json:"revision"` + Answered time.Time `json:"Answered"` + } `json:"first-views"` + } + if json.Unmarshal(raw, &a) != nil || len(a.Views) != len(followed) { + return false + } + for _, v := range a.Views { + if v.Answered.IsZero() { + return false + } + } + return true + }) +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +// manifestIn writes a module's manifest into the indexer's buckets as the indexer writes it, and names it as the trunk's. +func (p *player) manifestIn(module, raw string) string { + p.t.Helper() + digest := delivery.ManifestDigest([]byte(raw)) + value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw}) + if err != nil { + p.t.Fatal(err) + } + p.put("mesh-delivery-indexer.manifests", digest, json.RawMessage(value)) + p.put("mesh-delivery-indexer.modules", module, map[string]any{"v": 1, "manifest": digest, "repository": "novox/" + module, "commit": "c1"}) + return digest +} + +// buildIn writes a build of a module with a manifest, and the want that names it on a node. +func (p *player) buildIn(node, module, fp, raw string) { + p.t.Helper() + digest := delivery.ManifestDigest([]byte(raw)) + value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw}) + p.put("mesh-delivery-indexer.manifests", digest, json.RawMessage(value)) + bk, _ := delivery.BuilderBuildKey(module, fp) + p.put("mesh-delivery-builder.builds", bk, map[string]any{"v": 1, "state": "done", "manifest": digest}) + p.put("mesh-delivery-assigner.wants", "want."+node+"."+module, map[string]any{"v": 1, "fp": fp, "self": true}) +} + +func (p *player) placed(node string, modules ...string) { + p.t.Helper() + pl := delivery.Placement{V: delivery.Version, Node: node} + for _, m := range modules { + pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m}) + } + values, err := delivery.PlacementValues(pl) + if err != nil { + p.t.Fatal(err) + } + for k, raw := range values { + p.put("mesh-delivery-assigner.placement", k, json.RawMessage(raw)) + } +} + +func resolvedValue(p *player, node, module, key string) (string, bool) { + raw, ok := p.get(ResolvedState, ResolvedKey(node, module, key)) + if !ok { + return "", false + } + var r Resolved + _ = json.Unmarshal(raw, &r) + return string(r.Value), true +} + +// start runs the built binary on the bus and serves it as the runtime. +func start(t *testing.T, bin string, js jetstream.JetStream, router *standIn, haves map[string]string, readers []ReaderRow) *player { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + p := &player{t: t, ctx: ctx, js: js, router: router, in: stdin, pending: map[string]chan wire{}, haves: haves, readers: readers, + owned: map[string]*ownedState{}} + for _, f := range followed { + p.own(f.Read) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + go p.serve(stdout) + t.Cleanup(cancel) + return p +} + +// 1966/C9 (design 32, C20 and C24): the module whole on a real bus. +func TestConfigWholeRenamesAndRetiresAgainstTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-config does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + buckets := []string{"mesh-delivery-assigner_placement", "mesh-delivery-assigner_wants", "mesh-delivery-builder_builds", + "mesh-delivery-indexer_modules", "mesh-delivery-indexer_manifests", "mesh-delivery-indexer_predefined", + "mesh-delivery-indexer_config-candidates"} + for _, s := range []string{ValuesState, ResolvedState, PredefinedState, ProposalsState, PullsState, KeptState, DeclaredState, + RenamesState, RetiringState, FindingsState} { + buckets = append(buckets, Module+"_"+s) + } + for _, b := range buckets { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil { + t.Fatal(err) + } + } + router := newStandIn() + haves := map[string]string{"g14.player": fp1, "novox.player": fp1, "g14.radio": fp1} + p := start(t, bin, js, router, haves, nil) + p.viewsReady() + + // The mesh: player and radio on g14, player also on novox, all on their first build; dock declares both names. + const oldRadio = `{"module":"radio","config":{"band":{"type":"number","default":1}}}` + const newRadio = `{"module":"radio","config":{"gain":{"type":"number","default":1}}}` + const dock = `{"module":"dock","config":{"a":{"type":"number","default":1},"b":{"type":"number","default":1,"renamed-from":"a"}}}` + p.manifestIn("player", oldPlayer) + p.manifestIn("radio", oldRadio) + p.buildIn("g14", "player", fp1, oldPlayer) + p.buildIn("novox", "player", fp1, oldPlayer) + p.buildIn("g14", "radio", fp1, oldRadio) + p.placed("g14", "player", "radio") + p.placed("novox", "player") + if _, err := p.call(ResyncTool, nil); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) // the watches start after the first call + set := func(module, key string, value any) { + t.Helper() + if _, err := p.call(Seat+"."+Verb, map[string]any{"action": "set", "module": module, "node": "g14", "key": key, + "value": value, "why": "the operator asked for it"}); err != nil { + t.Fatal(err) + } + } + set("player", "volume", 8) + set("radio", "band", 3) + + // C20: the new manifest renames volume to level; the value moves, both names answer. + p.manifestIn("player", newPlayer) + eventually(t, "the value moves to the new name", func() bool { + raw, ok := p.get(ValuesState, "module.g14.player.level") + return ok && strings.Contains(string(raw), `"value":8`) + }) + if _, ok := p.get(ValuesState, "module.g14.player.volume"); ok { + t.Error("the old name still holds a value") + } + eventually(t, "both names answer on g14", func() bool { + a, okA := resolvedValue(p, "g14", "player", "level") + b, okB := resolvedValue(p, "g14", "player", "volume") + return okA && okB && a == "8" && b == "8" + }) + eventually(t, "renamed is said", func() bool { return p.publishedCount(EventRenamed) == 1 }) + + p.conforms(EventRenamed, &meshconfig.RenamedHolderEvent{}, meshconfig.HolderSubjectRenamed) + + // A refused move writes its finding: dock declares both names. + p.manifestIn("dock", dock) + p.buildIn("g14", "dock", fp1, dock) + p.placed("g14", "player", "radio", "dock") + eventually(t, "the refused rename's finding", func() bool { + keys, _ := (func() ([]string, error) { kv, _ := p.kv(FindingsState); return kv.Keys(p.ctx) })() + for _, k := range keys { + raw, _ := p.get(FindingsState, k) + if f, err := health.DecodeFinding(raw); err == nil && f.Kind == FindingRenameRefused && strings.Contains(f.Summary, "dock") { + return true + } + } + return false + }) + + // The module restarted inside the window keeps it, and says renamed no more. + said := p.publishedCount(EventRenamed) + cancelOld := p.ctx + _ = cancelOld + p2 := start(t, bin, js, router, haves, nil) + p2.viewsReady() + if _, err := p2.call(ResyncTool, nil); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if v, ok := resolvedValue(p2, "g14", "player", "volume"); !ok || v != "8" { + t.Errorf("after the restart the old name answers %q (%v), want 8", v, ok) + } + if n := p2.publishedCount(EventRenamed); n != 0 || said != 1 { + t.Errorf("renamed was said %d times before the restart and %d after", said, n) + } + raw, _ := p2.get(RenamesState, "player.level") + var rn Rename + if json.Unmarshal(raw, &rn) != nil || strings.Join(rn.Open(), ",") != "g14,novox" { + t.Fatalf("the window after the restart: %s", raw) + } + + // The seat's verb answers the open window in the definition's shape (CAT-C10). + raw, err = p2.call(Seat+"."+RenamesVerb, map[string]any{"module": "player"}) + if err != nil { + t.Fatal(err) + } + var open meshconfig.RenamesAnswer + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(&open); err != nil || len(open.Renames) != 1 || open.Renames[0].To != "level" || + strings.Join(open.Renames[0].Open, ",") != "g14,novox" { + t.Fatalf("renames answered %s (%v), want player's level open on g14 and novox", raw, err) + } + + // g14 applies the build that reads the new name; the delivery says so; its window closes, novox's stays. + p2.buildIn("g14", "player", fp2, newPlayer) + p2.mu.Lock() + p2.haves["g14.player"] = fp2 + p2.mu.Unlock() + p2.event("mesh-delivery-view.delivered", map[string]any{"revision": "r1"}) + eventually(t, "g14's window closes and the old name goes", func() bool { + _, old := resolvedValue(p2, "g14", "player", "volume") + v, still := resolvedValue(p2, "novox", "player", "volume") + return !old && still && v == "5" + }) + + // C24: radio's manifest drops band; g14 still runs the build that reads it: no ask. Then it runs the next: one ask. + p2.manifestIn("radio", newRadio) + p2.mu.Lock() + p2.readers = []ReaderRow{{Module: "radio", Key: "band", Kind: KindValue, Readers: []string{}}} + p2.mu.Unlock() + p2.event("mesh-delivery-indexer.manifest-accepted", map[string]any{"module": "radio"}) + eventually(t, "the retirement waits on g14", func() bool { + raw, ok := p2.get(RetiringState, "radio.band") + return ok && strings.Contains(string(raw), RetireDraining) + }) + if len(router.asks) != 0 || p2.publishedCount(EventRetiring) != 0 { + t.Fatalf("asked %d times, retiring said %d times while g14 reads the value", len(router.asks), p2.publishedCount(EventRetiring)) + } + p2.buildIn("g14", "radio", fp2, newRadio) + p2.mu.Lock() + p2.haves["g14.radio"] = fp2 + p2.mu.Unlock() + p2.event("mesh-delivery-view.delivered", map[string]any{"revision": "r2"}) + eventually(t, "the operator is asked once", func() bool { + router.mu.Lock() + defer router.mu.Unlock() + return len(router.asks) == 1 && p2.publishedCount(EventRetiring) == 1 + }) + if _, ok := p2.get(ValuesState, "module.g14.radio.band"); !ok { + t.Fatal("the value was removed before the operator answered") + } + var id string + router.mu.Lock() + for k := range router.asks { + id = k + } + router.mu.Unlock() + w := router.choose(t, id, OptionApprove, nil) + p2.event(asks.Seat+".decided."+Module, w) + eventually(t, "the approved value is removed and retired is said", func() bool { + _, still := p2.get(ValuesState, "module.g14.radio.band") + return !still && p2.publishedCount(EventRetired) == 1 + }) + p2.conforms(EventRetiring, &meshconfig.RetiringHolderEvent{}, meshconfig.HolderSubjectRetiring) + p2.conforms(EventRetired, &meshconfig.RetiredHolderEvent{}, meshconfig.HolderSubjectRetired) + eventually(t, "the retirement is dropped", func() bool { _, ok := p2.get(RetiringState, "radio.band"); return !ok }) +} diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index 95e35ffb..d781033f 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,13 +2,23 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 +require ( + git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 + github.com/nats-io/nats.go v1.54.0 +) require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.0 // indirect - github.com/nats-io/nats.go v1.54.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index 02d6daae..43cc4ec7 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,7 +1,17 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= @@ -10,5 +20,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index d62def76..f52841dc 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -7,12 +7,16 @@ "name": "mesh-config", "scope": "mesh", "serves": [ - "config" + "config", + "renames" ] } ], "emits": [ - "config-changed" + "config-changed", + "renamed", + "retiring", + "retired" ], "uses": [ "operator-channel" @@ -20,25 +24,44 @@ "consumes": [ "gitea.pull.merged", "gitea.pull.closed", - "gitea.forge.pull-request.closed" + "gitea.forge.pull-request.closed", + "mesh-delivery-view.delivered", + "node-engine.manifest-reads-changed", + "mesh-delivery-indexer.manifest-accepted", + "mesh-delivery-indexer.views.modules", + "mesh-delivery-indexer.views.manifests", + "mesh-delivery-indexer.views.predefined", + "mesh-delivery-indexer.views.config-candidates", + "mesh-delivery-assigner.views.placement", + "mesh-delivery-assigner.views.wants" ], "invokes": [ "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.warranted", - "gitea.gitea_get_pull_request" + "gitea.gitea_get_pull_request", + "mesh-delivery-view.node_haves", + "seat:node-engine.manifest-reads", + "mesh-delivery-indexer.indexer_config_readers", + "seat:mesh-vault.retire", + "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_read_manifests", + "mesh-delivery-indexer.indexer_read_predefined", + "mesh-delivery-indexer.indexer_read_config_candidates", + "mesh-delivery-assigner.assigner_read_placement", + "mesh-delivery-assigner.assigner_read_wants" ], "reads": [ - "mesh-delivery-assigner.placement", - "mesh-delivery-assigner.wants", - "mesh-delivery-builder.builds", - "mesh-delivery-indexer.modules", - "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.predefined", - "mesh-delivery-indexer.config-candidates" + "mesh-delivery-builder.builds" ], "tools": [ "config_declared", - "config_resync" + "config_resync", + "config_renames", + "config_rename_again", + "config_rename_close", + "config_retiring", + "config_first_views", + "config_first_view_again" ], "state": [ "values", @@ -47,7 +70,10 @@ "proposals", "pulls", "kept", - "declared" + "declared", + "renames", + "retiring", + "findings" ], "logs": [ "history" @@ -87,5 +113,16 @@ ] } ] - } + }, + "streams": [ + { + "name": "config-events", + "events": [ + "renamed", + "retiring", + "retired" + ], + "max-age-days": 7 + } + ] } diff --git a/modules/mesh-vault/README.md b/modules/mesh-vault/README.md index 75b089c7..dad246e7 100644 --- a/modules/mesh-vault/README.md +++ b/modules/mesh-vault/README.md @@ -229,6 +229,15 @@ restart; it is on until switched off, and `own_drift` shows it. - `provisioner.retirement` (ADR 0230 §35) — `waiting`, `settled`, `approved`, `rejected`, `retired`, `reenabled`, `deleted`, naming holders, never a value. +## Retiring an own secret: `retire` (design 32) + +`retire` removes one module's own secret on a node because no manifest reads it any more, and says `own-retired` +(node, module, name, ask, caller). Two proofs: the caller is on the vault's trusted list `retire-callers` (the holder of +the seat `mesh-config` alone, a file the mesh writes from the config value, read at start; a list that cannot be read +allows nobody), and `warrant` is the id of a hand act that caller recorded through `mesh-hand-acts` for this verb, +naming exactly the node, module, name and the ask of the operator's Approve, within a day. A bus account and the tool +runner's `ledgers` are never retired this way. A waiter on `rotated` or `own-replaced` does not wake on it. + ## The tools `secret_export`, `secret_holders`, `secret_verify` answer as the TypeScript vault's did, and its custody diff --git a/modules/mesh-vault/cmd/mesh-vault/enrolment_tools.go b/modules/mesh-vault/cmd/mesh-vault/enrolment_tools.go index 7b5c244d..a976eb3b 100644 --- a/modules/mesh-vault/cmd/mesh-vault/enrolment_tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/enrolment_tools.go @@ -128,6 +128,23 @@ func enrolmentTools(env *Env) []toolDef { } return v.AbandonRotation(caller, arg(a, "node")) }}, + {Name: Seat + "." + RetireVerb, + Description: "Retire one module's own secret on a node because no manifest reads it any more (design 32): removed " + + "whole, and own-retired said (node, module, name, the ask, the caller; never a value). Answered to the holder of " + + "the seat mesh-config alone, as the vault's list retire-callers names it, and only for a hand act it recorded " + + "through mesh-hand-acts for this verb naming exactly this node, module and name and the ask of the operator's Approve " + + "(`warrant`, the act's id), read back here.", + Input: map[string]any{"node": node, "module": str("the module whose secret it is"), "name": str("the secret's name"), + "warrant": str("the operator's warrant for this node, module and name: the id of the hand act recorded through mesh-hand-acts"), + "ask": str("the id of the operator's ask the warrant answers")}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.RetireOwn(caller, RetireArgs{Node: arg(a, "node"), Module: arg(a, "module"), Name: arg(a, "name"), + Warrant: arg(a, "warrant"), Ask: arg(a, "ask")}) + }}, {Name: "node_logins", Description: "Every node's key of the vault's state node-logins: the login its node-engine dials as and, while a rotation " + "runs, the other login and its stage; with the rotations queued and the enrolment tokens held. Never a value or a hash.", diff --git a/modules/mesh-vault/cmd/mesh-vault/intake.go b/modules/mesh-vault/cmd/mesh-vault/intake.go index b8a03715..9b5648ad 100644 --- a/modules/mesh-vault/cmd/mesh-vault/intake.go +++ b/modules/mesh-vault/cmd/mesh-vault/intake.go @@ -728,6 +728,10 @@ func (v *Vault) RotateFor(caller, secret string, one Narrow, by, why string) (Ch return Changed{}, fmt.Errorf("not rotated: %s is a bus account the controller mints and hands here (ADR 0374 "+ "clause 4): only a password whose hash the bus's user list holds can log in, so the vault makes no new "+ "value for it; `module issue` again hands a new one and replaces this in place", secret) + case isLedgersSecret(secret): + return Changed{}, fmt.Errorf("not rotated: %s is the tool runner's own secret of the ledger accounts of its machine, "+ + "which the vault rewrites on each ledger account minted or retired (ADR 0597 §9); a person's rotation would "+ + "replace every module's credential with a value no ledger account holds", secret) case rec.Origin == OriginIssued: return Changed{}, fmt.Errorf("not rotated: %s is issued by its provider's software, which the mesh cannot "+ "make or rotate (ADR 0305 §9): change it in that software, and `secret issued` takes the value it holds now", diff --git a/modules/mesh-vault/cmd/mesh-vault/ledgers.go b/modules/mesh-vault/cmd/mesh-vault/ledgers.go index e329ed5b..65067bff 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ledgers.go +++ b/modules/mesh-vault/cmd/mesh-vault/ledgers.go @@ -26,6 +26,12 @@ func ledgersSecret(node string) (string, error) { return OwnSecret(node, toolRunnerModule, buscompose.LedgersSecret) } +// isLedgersSecret says secret is a machine's `ledgers`: `own//node-tools/ledgers`. +func isLedgersSecret(secret string) bool { + parts := strings.Split(secret, "/") + return len(parts) == 4 && parts[0] == "own" && parts[2] == toolRunnerModule && parts[3] == buscompose.LedgersSecret +} + // toolRunnerNodes are the machines whose tool runner is wanted, sorted. func (v *Vault) toolRunnerNodes() []string { v.minting.mu.Lock() diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 3333b3c0..1a134558 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -19,6 +19,8 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + + "git.novox.be/novox/mesh-delivery/sdk/views" ) func say(format string, args ...any) { @@ -61,7 +63,7 @@ func build() *Env { Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, WantRead: readWant, HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{}, AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), EnrolCallers: askCallers(os.Getenv("MESH_VAULT_ENROLMENT_CALLERS")), - Logins: busNodeLogins{}, ReadAct: readHandAct, TokenBase: TokenBaseFromFile(os.Getenv("MESH_VAULT_STATE")), + RetireCallers: retireCallers(os.Getenv("MESH_VAULT_RETIRE_CALLERS")), Logins: busNodeLogins{}, ReadAct: readHandAct, TokenBase: TokenBaseFromFile(os.Getenv("MESH_VAULT_STATE")), Bound: Bound{ Get: func(key string) ([]byte, error) { e, err := stdio.State(BoundState).Get(key) @@ -108,21 +110,24 @@ func readHandAct(id string) (*HandActSeen, error) { return nil, nil } -// watchLeavers deletes a node's key of node-logins when its node key of the placement is deleted (noderotation.go): a -// watch, never a poll. It answers whether the watch began; the minute pass (LoginsPass) catches what it missed. +// watchLeavers begins the picture of the placement and deletes a node's key of node-logins when its node key of the +// placement is deleted (noderotation.go): the owner's event, never a poll. It answers whether it began; the minute pass +// (LoginsPass) catches what it missed. func watchLeavers(env *Env) bool { if env.Vault == nil { return true } - err := stdio.State(PlacementRead).Watch("*.node", func(c stdio.StateChange) error { - if c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0 { - env.Vault.NodeLeft(strings.TrimSuffix(c.Key, ".node")) - } - return nil - }) - if err != nil { - say("the placement's nodes are not watched for one that leaves; the minute pass finds it: %v", err) - return false + if placementView == nil { + placementView = newPlacementView(stdio.Ask, func(key string) { + if node, ok := leaverKey(key); ok { + env.Vault.NodeLeft(node) + } + }, func(key string, v json.RawMessage) error { + _, err := stdio.State(FindingsState).Put(key, v) + return err + }, + func(key string) error { return stdio.State(FindingsState).Delete(key) }) + beginPlacementView(placementView) } return true } @@ -232,17 +237,9 @@ func watchCredentialAsks(env *Env, kick chan<- struct{}) bool { return true } -// placement reads the seat mesh-placement's placement through the runtime under the vault's own account (its manifest reads it). -var placement = PlacementReader{ - Keys: func() ([]string, error) { return stdio.State(PlacementRead).Keys() }, - Read: func(key string) ([]byte, error) { - e, err := stdio.State(PlacementRead).Get(key) - if err != nil || e == nil { - return nil, err - } - return e.Value, nil - }, -} +// placement reads the seat mesh-placement's placement through its owner's tool and events (placementview.go), never the +// owner's bucket. +var placement = viewPlacement(func() *views.Follower { return placementView }) // readWant is one wanted account of the grants part, read afresh (issue-account). func readWant(user string) ([]byte, bool, error) { @@ -368,6 +365,26 @@ func main() { } } +// retireCallers reads the vault's list `retire-callers`, the file the mesh writes from its trusted config value +// (stdio.ParseCallers): a list that cannot be read allows nobody, and says why. +func retireCallers(path string) stdio.Callers { + if path == "" { + say("the vault was started without its retire-callers list (MESH_VAULT_RETIRE_CALLERS): retire answers nobody") + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + say("the retire-callers list cannot be read, so retire answers nobody: %v", err) + return nil + } + callers, err := stdio.ParseCallers(raw) + if err != nil { + say("%v: retire answers nobody", err) + return nil + } + return callers +} + // askCallers are the callers the vault's manifest names for secret-ask besides the controller (novox/hq ADR 0563 §8): // MESH_VAULT_SECRET_ASK_CALLERS, comma-separated, each a module (on any machine), . or person.. func askCallers(list string) stdio.Callers { diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go index e631d5d0..1ec93240 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go @@ -31,6 +31,7 @@ import ( "golang.org/x/crypto/bcrypt" "golang.org/x/crypto/nacl/box" + "git.novox.be/novox/mesh-contracts/go/seats/meshvault" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/seal" "git.novox.be/novox/mesh-sdk/go/testbus" @@ -62,6 +63,70 @@ type player struct { said []string // ask answers a mesh/ask (a call of another module's tool); nil answers none, as a call with no responder. ask func(key string, body json.RawMessage) (any, error) + + // The assigner's placement as its owner's tool and views family say it (ADR 0615): the values, and the revision of the + // last change, kept from the bucket. + placeMu sync.Mutex + placeValues map[string]json.RawMessage + placeRevision uint64 +} + +var placeCreated = time.Date(2026, 11, 4, 9, 0, 0, 0, time.UTC) + +// followPlacement keeps the bucket of the placement as the assigner's owner does: listed once, then each change said on +// its views family to the vault. Call it before the player serves. +func (p *player) followPlacement() { + p.t.Helper() + kv, err := p.js.KeyValue(p.ctx, "mesh-delivery-assigner_placement") + if err != nil { + p.t.Fatal(err) + } + w, err := kv.WatchAll(p.ctx) + if err != nil { + p.t.Fatal(err) + } + p.placeValues = map[string]json.RawMessage{} + listed := make(chan struct{}) + go func() { + current := true + for e := range w.Updates() { + if e == nil { + current = false + close(listed) + continue + } + p.placeMu.Lock() + deleted := e.Operation() != jetstream.KeyValuePut + if deleted { + delete(p.placeValues, e.Key()) + } else { + p.placeValues[e.Key()] = append(json.RawMessage(nil), e.Value()...) + } + if !current { + p.placeRevision++ + ev := map[string]any{"v": 1, "owner": "mesh-delivery-assigner", "state": "placement", "key": e.Key(), + "revision": p.placeRevision, "created": placeCreated} + if deleted { + ev["deleted"] = true + } else { + ev["value"] = json.RawMessage(e.Value()) + } + body, _ := json.Marshal(ev) + p.mu.Lock() + p.next++ + id := fmt.Sprintf("o-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{ + "key": "mesh-delivery-assigner.views.placement", "node": "novox", "body": json.RawMessage(body)}}}) + } + p.placeMu.Unlock() + } + }() + select { + case <-listed: + case <-time.After(10 * time.Second): + p.t.Fatal("the placement was not listed") + } } // bucketOf is the bucket a state names: the vault's own by its local name, another's as `.`. @@ -223,6 +288,15 @@ func (p *player) handle(m wire) { Body json.RawMessage `json:"body"` } _ = json.Unmarshal(m.Params, &ask) + if ask.Key == PlacementTool && p.placeValues != nil { + p.placeMu.Lock() + answer := map[string]any{"v": 1, "owner": "mesh-delivery-assigner", "state": "placement", "revision": p.placeRevision, + "created": placeCreated, "values": p.placeValues, "at": time.Now()} + raw, _ := json.Marshal(answer) + p.placeMu.Unlock() + p.answer(m.ID, json.RawMessage(raw), nil) + return + } if p.ask != nil { result, err := p.ask(ask.Key, ask.Body) p.answer(m.ID, result, err) @@ -300,6 +374,16 @@ func (p *player) hashOf(t *testing.T, user string) *AccountHash { return &a } +// strictly decodes raw into v, a field the type does not know being a fault (the definition's type is the contract). +func strictly(t *testing.T, raw []byte, v any) { + t.Helper() + dec := json.NewDecoder(strings.NewReader(string(raw))) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + t.Errorf("%s does not match the seat's definition: %v", raw, err) + } +} + func eventually(t *testing.T, what string, ok func() bool) { t.Helper() for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { @@ -347,10 +431,30 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) { if err := os.WriteFile(filepath.Join(root, "export.json"), export, 0o600); err != nil { t.Fatal(err) } + // An own secret of dunst on g14 is held before the vault starts, for the retire case below (CAT-V1). + seedStore, err := OpenStore(filepath.Join(dir, "store")) + if err != nil { + t.Fatal(err) + } + seedKey, err := OwnKey(filepath.Join(dir, "state"), seedStore) + if err != nil { + t.Fatal(err) + } + seed := &Vault{Store: seedStore, Own: seedKey, Now: time.Now, Log: t.Logf, HashCost: bcrypt.MinCost} + if _, err := seed.SetOperatorKey(asPEM(t, base64.StdEncoding.EncodeToString(opPub[:])), "the test"); err != nil { + t.Fatal(err) + } + if _, err := seed.keep("own/g14/dunst/token", "a-token-value", "the test", "g14.dunst", OriginMade); err != nil { + t.Fatal(err) + } + callers := filepath.Join(dir, "retire-callers.json") + if err := os.WriteFile(callers, []byte(`{"callers":["mesh-config"]}`), 0o600); err != nil { + t.Fatal(err) + } cmd := exec.CommandContext(ctx, bin) cmd.Env = append(os.Environ(), "MESH_VAULT_STORE="+filepath.Join(dir, "store"), "MESH_VAULT_STATE="+filepath.Join(dir, "state"), "MESH_VAULT_ROOT="+root, "MESH_VAULT_LEDGER="+filepath.Join(dir, "ledger"), "MESH_RECEIVES="+filepath.Join(dir, "mesh.json"), - "MESH_NODE=novox", "MESH_MODULE=mesh-vault") + "MESH_VAULT_RETIRE_CALLERS="+callers, "MESH_NODE=novox", "MESH_MODULE=mesh-vault") stdin, _ := cmd.StdinPipe() stdout, _ := cmd.StdoutPipe() cmd.Stderr = os.Stderr @@ -359,6 +463,16 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) { } t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) p := &player{t: t, ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + // mesh-hand-acts holds the acts mesh-config recorded: one warranting the token's retirement, one for another name. + p.ask = func(key string, _ json.RawMessage) (any, error) { + if key != "mesh-hand-acts.hand-acts" { + return nil, errors.New("nats: no responders available for request") + } + return map[string]any{"acts": []HandActSeen{ + {ID: "act-ok", By: "novox.mesh-config", At: time.Now(), Verb: "mesh-vault.retire", Args: []string{"g14 dunst token ret-1"}}, + {ID: "act-other", By: "novox.mesh-config", At: time.Now(), Verb: "mesh-vault.retire", Args: []string{"g14 dunst other ret-1"}}}}, nil + } + p.followPlacement() go p.serve(stdout) status := func() MintState { @@ -418,4 +532,45 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) { if !strings.Contains(said, AccountIssuedEvent+` {"`) || !strings.Contains(said, `"user":"ace.mod-x"`) || strings.Contains(said, held.Password) { t.Errorf("the vault said: %s", said) } + + // CAT-V1 (design 32): retire on the real bus. A waiter on rotated and own-replaced hears nothing of it. + waiter, err := nc.SubscribeSync("mesh.mod.mesh-vault.event.>") + if err != nil { + t.Fatal(err) + } + for _, c := range []struct{ caller, act string }{{"g14.claude-code", "act-ok"}, {"novox.mesh-config", "act-other"}, {"novox.mesh-config", ""}} { + if _, err := p.call(Seat+".retire", c.caller, map[string]any{"node": "g14", "module": "dunst", "name": "token", "warrant": c.act}); err == nil { + t.Errorf("%s retired the token with act %q", c.caller, c.act) + } + } + if m, err := waiter.NextMsg(300 * time.Millisecond); err == nil { + t.Fatalf("a refused retire said %s", m.Subject) + } + // The call is the definition's input, the answer its answer, and the event its body on its subject (CAT-V4). + input, _ := json.Marshal(meshvault.RetireInput{Node: "g14", Module: "dunst", Name: "token", Warrant: "act-ok", Ask: "ret-1"}) + var args map[string]any + _ = json.Unmarshal(input, &args) + raw, err = p.call(Seat+"."+meshvault.VerbRetire, "novox.mesh-config", args) + if err != nil { + t.Fatalf("retire with its warrant: %v", err) + } + var answered meshvault.RetireAnswer + strictly(t, raw, &answered) + if !answered.Retired { + t.Errorf("retire answered %s", raw) + } + m, err := waiter.NextMsg(10 * time.Second) + if err != nil || m.Subject != meshvault.HolderSubjectOwnRetired("mesh-vault") { + t.Fatalf("retire said %v, %v", m, err) + } + var event meshvault.OwnRetiredHolderEvent + strictly(t, m.Data, &event) + if event.Ask != "ret-1" || event.Caller != "novox.mesh-config" || event.Node != "g14" || event.Module != "dunst" || event.Name != "token" { + t.Errorf("own-retired said %s", m.Data) + } + for _, other := range []string{"rotated", OwnReplacedEvent} { + if extra, err := waiter.NextMsg(300 * time.Millisecond); err == nil { + t.Errorf("a waiter on %s heard %s", other, extra.Subject) + } + } } diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_test.go index e380c69d..697ab901 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint_test.go @@ -196,6 +196,21 @@ func TestTheVaultMintsAWantedAccountOnceAndNothingItHolds(t *testing.T) { t.Errorf("wanted again, m is not back in ledgers (%d) with its hash", len(got)) } }}, + // 1966/CAT-V2 (ADR 0597 §9): a person's rotate of a machine's `ledgers` is refused as a bus account's is: it + // would replace every module's credential with a value no ledger account holds. + {"a person's rotate of ledgers is refused", func(t *testing.T, v testVault, accounts *fakeAccounts, priv string) { + w := nodeToolsWant("ace") + v.WantHeard(w.User, wantRaw(t, w), false) + v.LedgersPass() + before, _ := ledgersOf(t, v, priv) + if _, err := v.Rotate("person.jochen", "own/ace/node-tools/ledgers", "jochen", "a test"); err == nil || + !strings.Contains(err.Error(), "ledger accounts of its machine") { + t.Errorf("a person rotated ledgers: %v", err) + } + if after, _ := ledgersOf(t, v, priv); len(after) != len(before) { + t.Errorf("a refused rotation changed ledgers: %d, was %d", len(after), len(before)) + } + }}, } { t.Run(c.name, func(t *testing.T) { v, accounts, priv := mintingVault(t) diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode.go b/modules/mesh-vault/cmd/mesh-vault/ownnode.go index aefc537e..f13db35f 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ownnode.go +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode.go @@ -455,6 +455,11 @@ func (v *Vault) HandedOf(node string) ([]HandedJudged, error) { } plan := v.plansOf(node) placed, placedErr := plan.placedModules() + if placedErr != nil { + // Never dropped: an answer that leaves out the modules placed there, with no word of it, would read as "the others + // are not placed" (issue 1966 review note). + return nil, fmt.Errorf("the placement of %s could not be read, so the modules placed there are not all judged: %w", node, placedErr) + } for _, m := range placed { modules[m] = true } @@ -462,9 +467,6 @@ func (v *Vault) HandedOf(node string) ([]HandedJudged, error) { for _, module := range slices.Sorted(maps.Keys(modules)) { out = append(out, plan.judged(module)) } - if placedErr != nil && len(out) == 0 { - return nil, fmt.Errorf("the placement of %s could not be read and the vault keeps no own secret there: %w", node, placedErr) - } return out, nil } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go index 5362dd7e..08a1a6b7 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode_test.go @@ -270,6 +270,7 @@ func TestOwnLeavesOutOnlyAModuleWhosePlanHandsIt(t *testing.T) { } v := newVault(t, nil) v.Askers = Declared{Read: plans.read, Store: v.Store} + v.Placement = placementsOf(t, map[string][]string{"anchor": {"grafana", "web", "db"}}) pub, _ := aSealingKey(t) for _, m := range []MoveOf{ {Node: "anchor", Module: "grafana", Name: "admin", Origin: "made", Generation: "g-1", SealingKey: pub}, @@ -314,4 +315,14 @@ func TestOwnLeavesOutOnlyAModuleWhosePlanHandsIt(t *testing.T) { if len(judged) != 4 { t.Errorf("own_handed names %v, want builder, db, grafana and web", judged) } + // Review note of issue 1966: a placement that cannot be read is an error the tool answers, never a list that leaves out + // the modules placed there while the vault keeps an own secret of others. + v.Placement = PlacementReader{Keys: func() ([]string, error) { return nil, errors.New("the bus did not answer") }, + Read: func(string) ([]byte, error) { return nil, errors.New("the bus did not answer") }} + if got, err := v.HandedOf("anchor"); err == nil || !strings.Contains(err.Error(), "placement of anchor could not be read") || got != nil { + t.Errorf("an unreadable placement answered %v, %v", got, err) + } + if _, err := toolNamed(t, definitions(&Env{Vault: v.Vault}), "own_handed").Run(map[string]any{"node": "anchor"}); err == nil { + t.Error("own_handed answered with the placement unread") + } } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownretire.go b/modules/mesh-vault/cmd/mesh-vault/ownretire.go new file mode 100644 index 00000000..b85d1936 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ownretire.go @@ -0,0 +1,136 @@ +package main + +// The seat's verb `retire` (design 32 *A value no longer read*, novox/hq ADR 0609): one module's own secret on one node +// removed because no manifest reads it any more, asked by the holder of the seat mesh-config alone after the operator's +// Approve. Two proofs, both required: +// +// - the caller is on the vault's list `retire-callers`, a trusted config value of the vault (ADR 0363's form, as +// mesh-mail-reader's `callers`), which names the holder of mesh-config alone, checked against the caller the tool +// runner proved (stdio.Callers); and +// - the warrant (the verb's argument, as the seat's definition names it): the id of a hand act the caller recorded +// through mesh-hand-acts for this verb, read back here, recent, naming exactly this node, module and secret and the +// ask whose Approve it answers (ADR 0259 §6). An act for another name warrants nothing. +// +// The vault then removes the record whole and says `own-retired` (node, module, name, the ask's id, the caller it proved) +// beside `rotated` and `own-replaced`, under its own subject: a waiter on either does not wake on it. Never a value. + +import ( + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// OwnRetiredEvent is what the vault says when it retired an own secret. +const OwnRetiredEvent = "own-retired" + +// RetireVerb is the seat's verb, and the verb of the hand act that warrants it (`mesh-vault.retire`). +const RetireVerb = "retire" + +// RetireArgs are `retire`'s arguments, as the seat's definition names them: the secret by node, module and name, the +// warrant (the id of the hand act that warrants it) and the ask of the operator's Approve it answers. +type RetireArgs struct { + Node, Module, Name, Warrant, Ask string +} + +// OwnRetiredBody is the body of own-retired, exactly the seat definition's (node, module, name, ask, caller). +type OwnRetiredBody struct { + Node string `json:"node"` + Module string `json:"module"` + Name string `json:"name"` + Ask string `json:"ask"` + Caller string `json:"caller"` +} + +// OwnRetired is what one `retire` answers, the seat definition's answer. +type OwnRetired struct { + Retired bool `json:"retired"` +} + +// fromRetireCallers refuses every caller the vault's list `retire-callers` does not name. +func (v *Vault) fromRetireCallers(caller string) error { + if !v.RetireCallers.Allows(caller) { + if strings.TrimSpace(caller) == "" { + caller = stdio.Unproven + } + return fmt.Errorf("retire is asked by the holder of the seat mesh-config alone, as the vault's list retire-callers "+ + "names it; %s is not on it (design 32)", caller) + } + return nil +} + +// warrantsRetire reads the hand act back and refuses what it does not warrant for exactly this secret; it answers the +// ask the act names. +func (v *Vault) warrantsRetire(caller string, a RetireArgs) (ask string, err error) { + if strings.TrimSpace(a.Ask) == "" { + return "", fmt.Errorf("retire names the ask of the operator's Approve it answers (`ask`)") + } + if strings.TrimSpace(a.Warrant) == "" { + return "", fmt.Errorf("retire carries the hand act that warrants it (`warrant`): record it through mesh-hand-acts first (design 32)") + } + if v.ReadAct == nil { + return "", fmt.Errorf("the vault reads no hand acts, so it retires nothing") + } + act, err := v.ReadAct(a.Warrant) + if err != nil { + return "", fmt.Errorf("the hand act %s could not be read back from mesh-hand-acts: %w", a.Warrant, err) + } + switch { + case act == nil: + return "", fmt.Errorf("mesh-hand-acts holds no hand act %s: record one first", a.Warrant) + case act.By != caller: + return "", fmt.Errorf("the hand act %s was recorded by %s, not by %s who asks", a.Warrant, act.By, caller) + case v.Now().Sub(act.At) > HandActWithin: + return "", fmt.Errorf("the hand act %s is older than %s: record a new one", a.Warrant, HandActWithin) + case act.Verb != Seat+"."+RetireVerb: + return "", fmt.Errorf("the hand act %s is %q, not %s.%s: it warrants no retirement", a.Warrant, act.Verb, Seat, RetireVerb) + } + var named []string + for _, arg := range act.Args { + named = append(named, strings.Fields(arg)...) + } + // The warrant is bound to the node, module and name it was asked for, and names the ask: those four words, no others. + if len(named) != 4 || named[0] != a.Node || named[1] != a.Module || named[2] != a.Name || named[3] != a.Ask { + return "", fmt.Errorf("the hand act %s names %q, not %s %s %s and ask %s: a warrant retires nothing else", + a.Warrant, strings.Join(named, " "), a.Node, a.Module, a.Name, a.Ask) + } + return named[3], nil +} + +// RetireOwn removes one module's own secret on a node on the warrant of a hand act, and says `own-retired`. +func (v *Vault) RetireOwn(caller string, a RetireArgs) (OwnRetired, error) { + if err := v.fromRetireCallers(caller); err != nil { + return OwnRetired{}, err + } + secret, err := OwnSecret(a.Node, a.Module, a.Name) + if err != nil { + return OwnRetired{}, err + } + ask, err := v.warrantsRetire(caller, a) + if err != nil { + return OwnRetired{}, err + } + switch { + case Covered(secret) || isLedgersSecret(secret): + return OwnRetired{}, fmt.Errorf("%s is a bus account or the tool runner's ledgers, which the vault retires and rewrites with the wants (ADR 0230, ADR 0597 §9), never a retirement of a value no manifest reads", secret) + } + _, held, err := v.Store.Read(secret) + if err != nil { + return OwnRetired{}, err + } + if !held { + return OwnRetired{}, fmt.Errorf("the vault holds no secret named %s, so there is nothing to retire", secret) + } + if err := v.Store.Delete(secret); err != nil { + return OwnRetired{}, fmt.Errorf("%s is not retired: %w", secret, err) + } + out := OwnRetired{Retired: true} + v.log("retired %s on the warrant of the hand act %s, asked by %s (ask %s)", secret, a.Warrant, caller, ask) + if v.Emit != nil { + body := map[string]any{"node": a.Node, "module": a.Module, "name": a.Name, "ask": ask, "caller": caller} + if err := v.Emit(OwnRetiredEvent, body); err != nil { + return out, fmt.Errorf("%s is retired, but own-retired was not said: %w", secret, err) + } + } + return out, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/placementview.go b/modules/mesh-vault/cmd/mesh-vault/placementview.go new file mode 100644 index 00000000..e14e766a --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placementview.go @@ -0,0 +1,198 @@ +package main + +// The vault's reading of the seat mesh-placement's placement, through its owner (novox/hq ADR 0615 §1, §4 to §6, ADR +// 0596): the assigner's tool `assigner_read_placement` asked for the whole state at start, the owner's `views.placement` +// events after it, and the tool once more only on a gap. Started while the assigner is down it keeps asking and says so; +// it never reads the assigner's bucket instead. (The vault's other reads, the node-engines' haves, the grants part's +// accounts and the provisioner's plans and provisions, are not the indexer's or the assigner's state and stay as they are.) + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-delivery/sdk/views" +) + +// PlacementTool is the assigner's ordinary read tool of its placement; PlacementViewsEvent the event of its views family. +const ( + PlacementTool = delivery.AssignerModule + ".assigner_read_placement" + PlacementViewsEvent = delivery.AssignerModule + ".views.placement" + FirstViewsTool = "vault_first_views" + FirstViewAgainTool = "vault_first_view_again" +) + +// FindingsState is the vault's state of findings, which the condition keeper reads (ADR 0564): a finding names no secret. +const FindingsState = "findings" + +// placementView is the picture; nil until main begins it. +var placementView *views.Follower + +// newPlacementView makes the follower: ask calls the tool, gone is told of a deleted key. +func newPlacementView(ask func(key string, body any) (json.RawMessage, error), gone func(key string), + put func(key string, value json.RawMessage) error, drop func(key string) error) *views.Follower { + return &views.Follower{Owner: delivery.AssignerModule, Tool: PlacementTool, State: "placement", Logf: say, + Ask: func(_ context.Context, after string) (views.Answer, error) { + body := map[string]any{"values": true} + if after != "" { + body["after"] = after + } + raw, err := ask(PlacementTool, body) + if err != nil { + return views.Answer{}, err + } + return views.DecodeAnswer(unwrapToolAnswer(raw)) + }, + // Past its bound the vault's finding names the owner and the tool, and the condition keeper raises it (the + // answer deletes it); the vault keeps asking meanwhile. + Raise: func(s views.Status) { + say("the vault has no first view of the assigner's placement: %s has not answered since %s (%s); it keeps asking", s.Tool, + s.Asked.Format(time.RFC3339), s.Error) + if k, v, err := views.UnansweredFinding(Seat, s, time.Now()); err == nil { + if err := put(k, v); err != nil { + say("the finding %s is not written: %v", k, err) + } + } + }, + Clear: func(s views.Status) { + if k, _, err := views.UnansweredFinding(Seat, s, time.Now()); err == nil { + _ = drop(k) + } + }, + Changed: func(key string, _ json.RawMessage, deleted bool, _ time.Time) { + if deleted { + gone(key) + } + }} +} + +// unwrapToolAnswer takes a tool's answer out of the protocol's content list when it comes in one. +func unwrapToolAnswer(raw json.RawMessage) []byte { + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } + if json.Unmarshal(raw, &r) == nil && len(r.Content) == 1 { + return []byte(r.Content[0].Text) + } + return raw +} + +// beginPlacementView asks the first view until answered and hears the owner's events. +func beginPlacementView(f *views.Follower) { + go func() { + if err := f.Start(context.Background()); err != nil { + say("the first view of the assigner's placement ended unanswered: %v", err) + } + }() + go func() { + <-stdio.Served() + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(PlacementViewsEvent, func(e stdio.Envelope) error { + ev, err := views.DecodeEvent(e.Body) + if err != nil { + say("%s is not a views event: %v", PlacementViewsEvent, err) + return nil + } + if gap, why := f.Apply(ev); gap { + say("a gap in the assigner's placement (%s): its first view is asked once more", why) + go func() { _ = f.Start(context.Background()) }() + } + return nil + }) + if err == nil { + return + } + say("does not hear %s yet (%v); asking again in %s", PlacementViewsEvent, err, wait) + time.Sleep(wait) + } + }() +} + +// viewPlacement is the placement reader over the picture: a key asked before the first view is answered is an error, so +// the caller waits (the "not yet" of the vault's other reads), never a read of the owner's bucket. +func viewPlacement(f func() *views.Follower) PlacementReader { + ready := func() (*views.Follower, error) { + fol := f() + if fol == nil { + return nil, fmt.Errorf("the vault reads the assigner's placement only when it runs as the module") + } + select { + case <-fol.Ready(): + return fol, nil + default: + s := fol.Status() + return nil, fmt.Errorf("the first view of the assigner's placement is not answered yet (%s asked %d times)", s.Tool, s.Asks) + } + } + return PlacementReader{ + Keys: func() ([]string, error) { + fol, err := ready() + if err != nil { + return nil, err + } + var keys []string + for k := range fol.Values() { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, nil + }, + Read: func(key string) ([]byte, error) { + fol, err := ready() + if err != nil { + return nil, err + } + v, ok := fol.Get(key) + if !ok { + return nil, nil + } + return v, nil + }, + } +} + +// leaverKey is whether a deleted key of the placement is a node's own key, `.node`. +func leaverKey(key string) (string, bool) { + node, ok := strings.CutSuffix(key, ".node") + return node, ok && node != "" +} + +// viewTools are the read tool of the first view and its re-read, as every reader has them (design 25 § 12.2). +func viewTools(env *Env) []toolDef { + return []toolDef{ + {Name: FirstViewsTool, + Description: "The vault's first view of the assigner's placement: the tool it asks, the revision its picture holds, when it " + + "asked and was answered, how many asks (and by hand), and whether it is waiting.", + run: func(string, map[string]any) (any, error) { + if placementView == nil { + return nil, fmt.Errorf("the vault reads the assigner's placement only when it runs as the module") + } + return map[string]any{"first-views": []views.Status{placementView.Status()}}, nil + }}, + {Name: FirstViewAgainTool, + Description: "Ask the first view of the assigner's placement once more by hand, with why, recorded in the vault's log.", + Input: map[string]any{"why": str("why the first view is asked again")}, + run: func(caller string, a map[string]any) (any, error) { + if placementView == nil { + return nil, fmt.Errorf("the vault reads the assigner's placement only when it runs as the module") + } + why := strings.TrimSpace(arg(a, "why")) + if why == "" { + return nil, fmt.Errorf("say why: a first view asked by hand keeps its why") + } + if err := fromAGrantedCaller(caller, FirstViewAgainTool); err != nil { + return nil, err + } + say("the first view of the assigner's placement is asked again by %s: %s", caller, why) + return map[string]any{"asked-again": true}, placementView.Again(context.Background(), why) + }}, + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/placementview_test.go b/modules/mesh-vault/cmd/mesh-vault/placementview_test.go new file mode 100644 index 00000000..2195d623 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placementview_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/sdk/views" +) + +// ADR 0615 §6: the vault started while the assigner is down keeps asking, never reads its bucket, and past the bound +// writes a finding naming the owner and the tool for the condition keeper; the answer deletes it. Before the answer a read +// of the placement is an error. +func TestAnUnansweredFirstViewOfThePlacementIsAFindingTheAnswerClears(t *testing.T) { + var mu sync.Mutex + up := false + held := map[string]json.RawMessage{} + ask := func(string, any) (json.RawMessage, error) { + mu.Lock() + defer mu.Unlock() + if !up { + return nil, errors.New("nats: no responders available for request") + } + raw, _ := json.Marshal(views.Answer{V: 1, Owner: "mesh-delivery-assigner", State: "placement", Revision: 1, Created: time.Now(), + Values: map[string]json.RawMessage{"g14.node": json.RawMessage(`{}`)}}) + return raw, nil + } + put := func(k string, v json.RawMessage) error { mu.Lock(); held[k] = v; mu.Unlock(); return nil } + drop := func(k string) error { mu.Lock(); delete(held, k); mu.Unlock(); return nil } + f := newPlacementView(ask, func(string) {}, put, drop) + f.Bound, f.Wait = 30*time.Millisecond, 10*time.Millisecond + reader := viewPlacement(func() *views.Follower { return f }) + if _, err := reader.Keys(); err == nil { + t.Fatal("the placement was read before its first view") + } + done := make(chan error, 1) + go func() { done <- f.Start(context.Background()) }() + count := func() int { mu.Lock(); defer mu.Unlock(); return len(held) } + for end := time.Now().Add(5 * time.Second); count() == 0 && time.Now().Before(end); time.Sleep(10 * time.Millisecond) { + } + if count() != 1 { + t.Fatalf("past its bound the vault wrote %d findings, want one", count()) + } + mu.Lock() + up = true + mu.Unlock() + if err := <-done; err != nil { + t.Fatal(err) + } + if count() != 0 { + t.Errorf("the answer left %d findings", count()) + } + if keys, err := reader.Keys(); err != nil || len(keys) != 1 || keys[0] != "g14.node" { + t.Errorf("keys %v, %v", keys, err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retire_test.go b/modules/mesh-vault/cmd/mesh-vault/retire_test.go index e4e4c55e..61376480 100644 --- a/modules/mesh-vault/cmd/mesh-vault/retire_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/retire_test.go @@ -4,6 +4,8 @@ import ( "strings" "testing" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" ) // 774/AC9 (novox/hq ADR 0561 §2, ADR 0230 §15, §18): a wanted account whose key is deleted is retired (its hash taken @@ -80,3 +82,90 @@ func TestAWantGoneRetiresItsAccountOnlyWithinTheBounds(t *testing.T) { t.Errorf("after a person's retirement %d keys remain", len(accounts.keys)) } } + +// 1966/CAT-V1 (design 32): `retire` removes one module's own secret on a node, and says own-retired, only for a caller on +// the vault's list retire-callers who carries a hand act it recorded for this verb naming exactly this node, module and +// secret. Without the act, with another name's, with another caller's or an older one, from a caller not on the list, or +// for a bus account, nothing is removed and nothing is said; a waiter on rotated or own-replaced is not woken. +func TestRetireRemovesAnOwnSecretOnlyOnTheWarrantOfTheListedCaller(t *testing.T) { + v := newVault(t, nil) + const caller = "novox.mesh-config" + v.RetireCallers = stdio.Callers{"mesh-config"} + acts := map[string]*HandActSeen{} + v.ReadAct = func(id string) (*HandActSeen, error) { return acts[id], nil } + act := func(id, by, verb, what string, age time.Duration) { + acts[id] = &HandActSeen{ID: id, By: by, At: v.Now().Add(-age), Verb: verb, Args: []string{what}} + } + act("ok", caller, "mesh-vault.retire", "g14 dunst token ret-1", time.Minute) + act("other-name", caller, "mesh-vault.retire", "g14 dunst other ret-1", time.Minute) + act("other-node", caller, "mesh-vault.retire", "ace dunst token ret-1", time.Minute) + act("other-caller", "g14.claude-code", "mesh-vault.retire", "g14 dunst token ret-1", time.Minute) + act("other-verb", caller, "mesh-vault.rotate", "g14 dunst token ret-1", time.Minute) + act("ledgers", caller, "mesh-vault.retire", "g14 node-tools ledgers ret-1", time.Minute) + act("no-ask", caller, "mesh-vault.retire", "g14 dunst token", time.Minute) + act("old", caller, "mesh-vault.retire", "g14 dunst token ret-1", 25*time.Hour) + const secret = "own/g14/dunst/token" + for _, name := range []string{secret, "own/g14/dunst/other", "own/g14/node-tools/ledgers"} { + if _, err := v.keep(name, "a-value-of-"+name, "the test", "g14.dunst", OriginMade); err != nil { + t.Fatal(err) + } + } + for _, tc := range []struct { + name, caller string + args RetireArgs + want string + }{ + {"no act", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Ask: "ret-1"}, "carries the hand act"}, + {"an act that does not exist", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "nope", Ask: "ret-1"}, "holds no hand act"}, + {"an act for another name", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-name", Ask: "ret-1"}, "a warrant retires nothing else"}, + {"an act for another node", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-node", Ask: "ret-1"}, "a warrant retires nothing else"}, + {"an act of another caller", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-caller", Ask: "ret-1"}, "was recorded by g14.claude-code"}, + {"an act for another verb", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-verb", Ask: "ret-1"}, "warrants no retirement"}, + {"an ask other than the act's", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-2"}, "a warrant retires nothing else"}, + {"no ask", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok"}, "names the ask"}, + {"an act that names no ask", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "no-ask", Ask: "ret-1"}, "a warrant retires nothing else"}, + {"an act older than a day", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "old", Ask: "ret-1"}, "older than"}, + {"a caller not on the list", "g14.claude-code", RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-caller", Ask: "ret-1"}, "retire-callers"}, + {"an unproven caller", stdio.Unproven, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-1"}, "retire-callers"}, + {"the tool runner's ledgers", caller, RetireArgs{Node: "g14", Module: "node-tools", Name: "ledgers", Warrant: "ledgers", Ask: "ret-1"}, "bus account or the tool runner's ledgers"}, + } { + t.Run(tc.name, func(t *testing.T) { + before := len(v.bus.events) + if _, err := v.RetireOwn(tc.caller, tc.args); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("answered %v, want a refusal saying %q", err, tc.want) + } + if _, held, _ := v.Store.Read(secret); !held { + t.Error("a refused retire removed the secret") + } + if n := len(v.bus.events) - before; n != 0 { + t.Errorf("a refused retire said %d events", n) + } + }) + } + before := len(v.bus.events) + got, err := v.RetireOwn(caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-1"}) + if err != nil || !got.Retired { + t.Fatalf("retire answered %+v, %v", got, err) + } + if _, held, _ := v.Store.Read(secret); held { + t.Error("the secret is still held") + } + if _, held, _ := v.Store.Read("own/g14/dunst/other"); !held { + t.Error("another secret of the module was removed") + } + if n := eventsNamed(v, before, OwnRetiredEvent); n != 1 || len(v.bus.events)-before != 1 { + t.Fatalf("said %d own-retired among %d events, want one and no other", n, len(v.bus.events)-before) + } + if body, _ := v.bus.events[before]["body"].(map[string]any); len(body) != 5 || body["ask"] != "ret-1" || body["caller"] != caller || body["node"] != "g14" || + body["module"] != "dunst" || body["name"] != "token" { + t.Errorf("own-retired said %+v", body) + } + if OwnRetiredEvent == OwnReplacedEvent || OwnRetiredEvent == "rotated" { + t.Error("own-retired shares a name with an event a waiter hears") + } + // Asked again for what is gone: nothing to retire. + if _, err := v.RetireOwn(caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-1"}); err == nil || + !strings.Contains(err.Error(), "nothing to retire") { + t.Errorf("a second retire answered %v", err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go index 2db1d205..fcb44b18 100644 --- a/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go @@ -88,6 +88,7 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) { } return map[string]any{"acts": []HandActSeen{{ID: "act-1", By: "person.jochen", At: time.Now(), Verb: "rotate-node", Args: []string{"g14"}}}}, nil } + p.followPlacement() go p.serve(stdout) // The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index f15e428b..cc5d4b19 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -79,7 +79,7 @@ func str(description string) map[string]any { } func definitions(env *Env) []toolDef { - return append(enrolmentTools(env), []toolDef{ + return append(append(enrolmentTools(env), viewTools(env)...), []toolDef{ {Name: Seat + ".get", Description: "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its " + "generation; answered only to an asker the vault allows, as the runtime proved it (ADR 0349 §11). " + diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index 7cd0b706..d9081224 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -77,6 +77,9 @@ type Vault struct { // EnrolCallers are the callers of the enrolment verbs `enrolment-token` and `node-account` and of `sign` for an // enrolment: the seat mesh-enrolment (enrolment.go). Nil: that module alone. EnrolCallers stdio.Callers + // RetireCallers are the callers of the seat's verb `retire`: the vault's trusted config value `retire-callers`, the + // holder of the seat mesh-config alone (ownretire.go). Nil: nobody. + RetireCallers stdio.Callers // TokenBase says where the bus is and its certificate's fingerprint, for the token shown (enrolment.go): nil shows none. TokenBase func() (TokenBase, error) // Logins is the vault's state node-logins (noderotation.go): nil writes none. diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go index fd113fba..d5b8eca9 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -146,14 +146,14 @@ func TestNoBucketHoldsASecret(t *testing.T) { if err := json.Unmarshal(raw, &manifest); err != nil { t.Fatal(err) } - if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound","node-logins"]` { + if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound","node-logins","findings"]` { t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the "+ "buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+ "credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6) and each node's login in use (ADR 0583)", s) } - if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` { + if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-provisioner.plans","mesh-provisioner.provisions"]` { t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+ - "grants part's wanted accounts and the placement alone, never the controller's lease (issue 1126)", r) + "grants part's wanted accounts alone (the placement is the assigner's, through its tool and events, ADR 0615), never the controller's lease (issue 1126)", r) } sources, _ := filepath.Glob("*.go") bucket := regexp.MustCompile(`stdio\.State\(|nats-io|jetstream|KeyValue`) @@ -195,11 +195,9 @@ func TestNoBucketHoldsASecret(t *testing.T) { code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Keys(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Watch(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Get(", "", 1)) - // And the seat's placement, where modules stand (placed.go): its keys and one key, read. - code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1)) - code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1)) - // And one watch of its nodes' keys, for a node that leaves: its node-logins key goes with it (noderotation.go). - code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Watch(", "", 1)) + // And its findings (ADR 0615 §6): the finding of a first view unanswered, written and deleted, never a secret. + code = []byte(strings.Replace(string(code), "stdio.State(FindingsState).Put(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(FindingsState).Delete(", "", 1)) // And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and // one read of the seat's provisions, which hold none. for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} { diff --git a/modules/mesh-vault/files/retire-callers.json b/modules/mesh-vault/files/retire-callers.json new file mode 100644 index 00000000..6d0792ab --- /dev/null +++ b/modules/mesh-vault/files/retire-callers.json @@ -0,0 +1,3 @@ +{ + "callers": ["mesh-config"] +} diff --git a/modules/mesh-vault/go.mod b/modules/mesh-vault/go.mod index 55f6b0da..85849672 100644 --- a/modules/mesh-vault/go.mod +++ b/modules/mesh-vault/go.mod @@ -3,6 +3,8 @@ module mesh-vault go 1.26.0 require ( + git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index b872c37c..1312f04c 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -24,15 +24,16 @@ "signed", "rotation-begun", "node-account-ready", - "rotation-done" + "rotation-done", + "own-retired" ], "invokes": [ - "seat:mesh-hand-acts.hand-acts" + "seat:mesh-hand-acts.hand-acts", + "mesh-delivery-assigner.assigner_read_placement" ], "reads": [ "node-engine.have", "mesh-grants.accounts", - "mesh-delivery-assigner.placement", "mesh-provisioner.plans", "mesh-provisioner.provisions" ], @@ -43,7 +44,8 @@ }, "signers", "bound", - "node-logins" + "node-logins", + "findings" ], "receives": { "secret": "${dir:grants}/mesh.json" @@ -142,6 +144,19 @@ "mode": "0700", "owner": "mesh-vault", "unknown": "program" + }, + { + "id": "retire-callers", + "trusted": true, + "type": "file", + "path": "${dir:state}/retire-callers.json", + "mode": "0600", + "owner": "mesh-vault", + "merge": "json", + "template": "files/retire-callers.json", + "takes-config": [ + "retire-callers" + ] } ], "build": { @@ -163,8 +178,14 @@ "MESH_VAULT_STORE": "${dir:store}", "MESH_VAULT_STATE": "${dir:state}", "MESH_VAULT_SECRET_ASK_CALLERS": "node-tools", - "MESH_VAULT_ENROLMENT_CALLERS": "mesh-enrolment" + "MESH_VAULT_ENROLMENT_CALLERS": "mesh-enrolment", + "MESH_VAULT_RETIRE_CALLERS": "${dir:state}/retire-callers.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] }, @@ -191,7 +212,9 @@ "vault_signers", "vault_signed", "signers_reassert", - "node_logins" + "node_logins", + "vault_first_views", + "vault_first_view_again" ], "claims": [ { @@ -211,8 +234,22 @@ "node-account", "rotate-node", "node-account-confirm", - "abandon" + "abandon", + "retire" ] } + ], + "config": { + "retire-callers": { + "type": "list", + "description": "Who may call the seat's verb retire, which removes one module's own secret on a node because no manifest reads it any more: a module's name (on every machine), ., or person.. The holder of the seat mesh-config alone by default (design 32); the verb also needs a hand act the caller recorded for exactly that node, module and secret.", + "trusted": true, + "default": [ + "mesh-config" + ] + } + }, + "consumes": [ + "mesh-delivery-assigner.views.placement" ] }