diff --git a/modules/mesh-conditions/cmd/mesh-conditions/homes.go b/modules/mesh-conditions/cmd/mesh-conditions/homes.go index 86072e9f2..e98549ae4 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/homes.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/homes.go @@ -93,6 +93,12 @@ var Homes = map[string]Home{ "asks-undelivered": {Source: "asks", Controller: true, Observer: messenger}, // The seat mesh-artifact-sweep (L7): an image a want names is missing from the store, artifact-store-tools' finding. "image-missing": {Source: "S25", Controller: true, Observer: artifactStore}, + // mesh-config's own (design 32): a rename it refused, a rename window or a retirement held seven days by a node, a + // retirement asked twice and not answered. + "rename-refused": {Source: "config", Observer: configModule}, + "rename-window-held": {Source: "config", Observer: configModule}, + "retirement-held": {Source: "config", Observer: configModule}, + "retiring-unanswered": {Source: "config", Observer: configModule}, // The kinds that end with the controller and are carried until its lease write (clauses 4, 9). "controller-deaf": {Source: "S4", Controller: true, Carried: true}, "lease-lost": {Source: "S12", Controller: true, Carried: true}, @@ -155,6 +161,7 @@ const ( provisioner = "mesh-provisioner" messenger = "messenger" artifactStore = "artifact-store-tools" + configModule = "mesh-config" ) // homeOf is the home of a condition's kind, and whether the table has one. diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index e72cd4269..34c4c93c0 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -40,7 +40,7 @@ const HistoryKeptFor = 90 * 24 * time.Hour // table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder, // which observes S25, image-missing). var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule, - "mesh-provisioner", "messenger", artifactStore} + "mesh-provisioner", "messenger", artifactStore, configModule} // The other states it reads, as `.`. const ( diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index c2d6b3bb2..dc33bab1b 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -294,7 +294,7 @@ func TestEveryControllerKindHasItsHome(t *testing.T) { t.Errorf("the ended probe's kind %s is %+v", kind, h) } } - if len(Homes) != 84 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled + if len(Homes) != 88 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled, and mesh-config's four (design 32) t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried) } } diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index ce6ecbae8..e3ce5c361 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -21,6 +21,7 @@ "messenger.findings", "mesh-provisioner.findings", "artifact-store-tools.findings", + "mesh-config.findings", "mesh-healers.attempts", "mesh-delivery-assigner.wants", "node-engine.have", diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index 1db9388f7..a745180e4 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -93,6 +93,10 @@ type Config struct { Asks AskBus // Forge is the forge as pulls are asked of it (pulls.go); nil, a merge or a close is heard from its events alone. Forge PullForge + // Delivery is what the delivery parts answer (renames.go, retiring.go); nil, no window closes and nothing retires. + Delivery Delivery + // Vault is the seat mesh-vault, for the retiring of a secret; nil, a secret does not retire. + Vault Vault // RouterWithin and RouterEvery bound how long a proposal waits for the router's word on its ask, and how often // it looks; zero is RouterAnswersWithin and RouterAnswersEvery. RouterWithin, RouterEvery time.Duration diff --git a/modules/mesh-config/cmd/mesh-config/config_test.go b/modules/mesh-config/cmd/mesh-config/config_test.go index 7201384ac..5b4334d04 100644 --- a/modules/mesh-config/cmd/mesh-config/config_test.go +++ b/modules/mesh-config/cmd/mesh-config/config_test.go @@ -20,6 +20,7 @@ type memStore struct { states map[string]map[string][]byte logs map[string][]LogEntry events []Change + said []saidEvent handActs []string // failHandAct makes the hand-act log refuse. failHandAct bool @@ -89,15 +90,38 @@ func (m *memStore) Read(log string) ([]LogEntry, error) { } func (m *memStore) Emit(event string, body any) error { - if event != ConfigChanged { - return fmt.Errorf("mesh-config emits no %s", event) - } m.mu.Lock() defer m.mu.Unlock() - m.events = append(m.events, body.(Change)) + switch event { + case ConfigChanged: + m.events = append(m.events, body.(Change)) + case EventRenamed, EventRetiring, EventRetired: + m.said = append(m.said, saidEvent{event, body}) + default: + return fmt.Errorf("mesh-config emits no %s", event) + } return nil } +// saidEvent is an event of the family config.events as it was emitted. +type saidEvent struct { + Event string + Body any +} + +// saidOf is the events of one name emitted so far. +func (m *memStore) saidOf(event string) []any { + m.mu.Lock() + defer m.mu.Unlock() + var out []any + for _, e := range m.said { + if e.Event == event { + out = append(out, e.Body) + } + } + return out +} + func (m *memStore) HandAct(what, why, cause string) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/modules/mesh-config/cmd/mesh-config/declared.go b/modules/mesh-config/cmd/mesh-config/declared.go index 205c215d3..b4c0d340c 100644 --- a/modules/mesh-config/cmd/mesh-config/declared.go +++ b/modules/mesh-config/cmd/mesh-config/declared.go @@ -41,6 +41,9 @@ type Declaration struct { Predefined string `json:"predefined,omitempty"` Required bool `json:"required,omitempty"` Runtime bool `json:"runtime,omitempty"` + // RenamedFrom is the key's old name when it was renamed (novox/hq ADR 0609 §4, design 32): on the merge the value + // set under the old name moves to this key (renames.go). + RenamedFrom string `json:"renamed-from,omitempty"` } // HasDefault is whether the declaration gives a default: present and not null. diff --git a/modules/mesh-config/cmd/mesh-config/delivery.go b/modules/mesh-config/cmd/mesh-config/delivery.go new file mode 100644 index 000000000..7e4171d47 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/delivery.go @@ -0,0 +1,133 @@ +package main + +// What mesh-config asks of the delivery parts and the vault, through their tools and seat verbs on the bus: never a +// bucket of theirs (novox/hq ADR 0596). Each key is the address a module's `invokes` names (mesh-sdk calladdress). + +import ( + "encoding/json" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// The addresses asked. +const ( + // HavesAsk is the status view's read of the node-engines' haves, narrowed to one node and module. + HavesAsk = "mesh-delivery-view.node_haves" + // ManifestReadsAsk is the node-engine seat's verb saying what manifest forms a node-engine reads (ADR 0618 §3), asked + // of one node by the address's `@`. + ManifestReadsAsk = "seat:node-engine.manifest-reads" + // ConfigReadersAsk is the indexer's own tool saying who reads a module's config keys (design 32). + ConfigReadersAsk = "mesh-delivery-indexer.indexer_config_readers" + // VaultRetireAsk is the seat mesh-vault's verb retiring one module's own secret on a node (design 32). + VaultRetireAsk = "seat:mesh-vault.retire" +) + +// toolDelivery answers Delivery through the bus. +type toolDelivery struct { + ask func(key string, body any) (json.RawMessage, error) +} + +// Have asks node_haves for one node and module and reads the fingerprint the node runs. +func (d toolDelivery) Have(node, module string) (string, bool, error) { + raw, err := d.ask(HavesAsk, map[string]any{"node": node, "module": module}) + if err != nil { + return "", false, err + } + answer, output, ok := unwrapAnswer(raw) + if !ok { + return "", false, fmt.Errorf("node_haves refused: %s", output) + } + var kept struct { + Keys map[string]json.RawMessage `json:"keys"` + } + if err := json.Unmarshal(answer, &kept); err != nil { + return "", false, fmt.Errorf("node_haves answered what is not its listing: %s", output) + } + key, err := delivery.HaveKey(node, module) + if err != nil { + return "", false, err + } + have, ok := kept.Keys[key] + if !ok { + return "", false, nil + } + var h delivery.Have + if err := json.Unmarshal(have, &h); err != nil { + return "", false, fmt.Errorf("the have of %s on %s is not readable: %w", module, node, err) + } + return h.FP, true, nil +} + +// Reads asks a node's node-engine what it reads; an answer means it states it. +func (d toolDelivery) Reads(node string) error { + _, err := d.ask(ManifestReadsAsk+"@"+node, map[string]any{}) + return err +} + +// Readers asks the indexer who reads a module's config keys; an indexer that does not serve the tool yet is "not known". +func (d toolDelivery) Readers(module string) ([]ReaderRow, bool, error) { + raw, err := d.ask(ConfigReadersAsk, map[string]any{"module": module}) + if err != nil { + if notServed(err) { + return nil, false, nil + } + return nil, false, err + } + answer, output, ok := unwrapAnswer(raw) + if !ok { + if notServed(fmt.Errorf("%s", output)) { + return nil, false, nil + } + return nil, false, fmt.Errorf("indexer_config_readers refused: %s", output) + } + var a struct { + Readers []ReaderRow `json:"readers"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return nil, false, fmt.Errorf("indexer_config_readers answered what is not its listing: %s", output) + } + return a.Readers, true, nil +} + +// notServed is whether an error says the address has no holder serving it. +func notServed(err error) bool { + m := strings.ToLower(err.Error()) + for _, w := range []string{"no responders", "not served", "unknown tool", "no such tool", "serves no"} { + if strings.Contains(m, w) { + return true + } + } + return false +} + +// toolVault asks the seat mesh-vault to retire a secret: the hand act that warrants it is recorded first, by this +// module, naming the node, module and secret; the vault reads it back by its id. +type toolVault struct { + ask func(key string, body any) (json.RawMessage, error) + recorder handacts.Recorder +} + +// Retire records the warrant and calls the vault's verb. +func (v toolVault) Retire(r RetireRequest) error { + taken, err := v.recorder.Record(handacts.Request{What: r.Node + " " + r.Module + " " + r.Name, + Why: "the operator approved retiring " + r.Module + "'s secret " + r.Name + " on " + r.Node + " (ask " + r.Ask + ")", + Cause: "the operator's Approve of ask " + r.Ask, Verb: "mesh-vault.retire"}) + if err != nil { + return err + } + answer, _, _ := unwrapAnswer(taken.Answer) + var rec struct { + Act struct { + ID string `json:"id"` + } `json:"act"` + } + if json.Unmarshal(answer, &rec) != nil || rec.Act.ID == "" { + return fmt.Errorf("mesh-hand-acts recorded the warrant and named no act") + } + _, err = v.ask(VaultRetireAsk, map[string]any{"node": r.Node, "module": r.Module, "name": r.Name, "act": rec.Act.ID, + "ask": r.Ask}) + return err +} diff --git a/modules/mesh-config/cmd/mesh-config/findings.go b/modules/mesh-config/cmd/mesh-config/findings.go new file mode 100644 index 000000000..fcccb2145 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/findings.go @@ -0,0 +1,138 @@ +package main + +// mesh-config's findings (novox/hq ADR 0564 clause 2): what it sees wrong about a rename or a retirement, kept in its +// state `findings` as the condition keeper reads them, written when what they say changes and deleted when the fault +// is gone. The keeper raises and clears the condition of the same key (mesh-conditions' Observers and Homes). + +import ( + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// FindingsState is the state, health.FindingsState. +const FindingsState = health.FindingsState + +// The kinds of finding mesh-config writes. +const ( + // FindingRenameRefused is a move mesh-config refused; cleared once the value moved or `renamed-from` left the + // trunk's manifest. + FindingRenameRefused = "rename-refused" + // FindingWindowHeld is a rename window held open seven days by a node; cleared when the window closes. + FindingWindowHeld = "rename-window-held" + // FindingRetireUnanswered is a retirement's ask unanswered after it was asked once more; cleared by an answer, a + // cancel, or the value read again. + FindingRetireUnanswered = "retiring-unanswered" + // FindingRetireHeld is a retirement waiting seven days for a node still running a build that reads the value. + FindingRetireHeld = "retirement-held" +) + +// findingKey is the key of a finding of kind about a module on a node: the condition's key. +func findingKey(kind, module, node, ref string) string { + id := module + if node != "" { + id += "." + node + } + return health.Key(health.ScopeModule, id, kind+"-"+ref) +} + +func finding(kind, module, node, ref, summary, headline, explanation, needs, resolved, said string, at time.Time) health.Finding { + id := module + if node != "" { + id += "." + node + } + return health.Finding{V: health.FindingVersion, Key: findingKey(kind, module, node, ref), Kind: kind, Ref: kind + "-" + ref, + Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node}, + Severity: health.Warning, Resolver: health.ResolverOperator, Summary: summary, Headline: headline, + Explanation: explanation, Needs: needs, Resolved: resolved, + Evidence: []health.Evidence{{At: at.UTC(), Said: said}}, Source: "config", Observer: Module, At: at.UTC()} +} + +// renameRefused is the finding of a move mesh-config refused. +func renameRefused(module, to, from, why string, at time.Time) health.Finding { + return finding(FindingRenameRefused, module, "", to, + fmt.Sprintf("the rename of %s's config value %s to %s is refused: %s", module, from, to, why), + fmt.Sprintf("%s's value %s is not moved to %s", module, from, to), + fmt.Sprintf("The new version of %s names %s as the new name of its config value %s, and mesh-config refused to move the value: %s. Nothing was moved.", module, to, from, why), + fmt.Sprintf("put the cause right, then move it with config_rename_again (module %s, to %s); the details name it.", module, to), + fmt.Sprintf("Resolved: %s's value %s is under %s", module, from, to), why, at) +} + +// windowHeld is the finding of a rename window held open past HeldFinding by a node. +func windowHeld(r Rename, node string, at time.Time) health.Finding { + return finding(FindingWindowHeld, r.Module, node, r.To, + fmt.Sprintf("%s on %s has held the old config name %s open for %d days", r.Module, node, r.From, int(HeldFinding.Hours()/24)), + fmt.Sprintf("%s on %s still runs the old config name", r.Module, node), + fmt.Sprintf("The rename of %s's config value %s to %s is not complete on %s: its node-engine has not reported a build that reads %s (it may be away).", r.Module, r.From, r.To, node, r.To), + fmt.Sprintf("bring %s back, or drop it from the window with config_rename_close.", node), + fmt.Sprintf("Resolved: %s reads %s on %s", r.Module, r.To, node), + fmt.Sprintf("the window opened %s and %s has not reported a build reading %s", r.Nodes[node].Since.UTC().Format(time.RFC3339), node, r.To), at) +} + +// setFinding writes a finding when it differs from the one kept. +func (c *Config) setFinding(f health.Finding) { + if _, err := health.EncodeFinding(f); err != nil { + c.logf("[mesh-config] a finding is not valid: %v", err) + return + } + if e, err := c.Store.Get(FindingsState, f.Key); err == nil && e != nil { + if old, derr := health.DecodeFinding(e.Value); derr == nil && old.Summary == f.Summary { + return // what it says has not changed + } + } + if err := c.Store.Put(FindingsState, f.Key, f); err != nil { + c.logf("[mesh-config] the finding %s is not written: %v", f.Key, err) + } +} + +// clearFinding deletes the findings of a kind and ref about a module, on whatever node but those in keep. +func (c *Config) clearFinding(kind, module, ref string, keep ...string) { + keys, err := c.Store.Keys(FindingsState) + if err != nil { + return + } + want := kind + "-" + ref + for _, k := range keys { + e, err := c.Store.Get(FindingsState, k) + if err != nil || e == nil { + continue + } + f, err := health.DecodeFinding(e.Value) + if err != nil || f.Ref != want || !(f.Subject.ID == module || strings.HasPrefix(f.Subject.ID, module+".")) || + (f.Subject.Machine != "" && slices.Contains(keep, f.Subject.Machine)) { + continue + } + if err := c.Store.Delete(FindingsState, k); err != nil { + c.logf("[mesh-config] the finding %s is not cleared: %v", k, err) + } + } +} + +// retirementHeld is the finding of a retirement that waits seven days for nodes that still run a build reading the value. +func retirementHeld(e Retiring, at time.Time) health.Finding { + node := "" + if len(e.Nodes) > 0 { + node = e.Nodes[0] + } + return finding(FindingRetireHeld, e.Module, node, e.Key, + fmt.Sprintf("%s's %s has waited %d days to retire: %s still run a build that reads it", e.Module, e.Key, int(HeldFinding.Hours()/24), strings.Join(e.Nodes, ", ")), + fmt.Sprintf("%s's %s cannot be retired yet", e.Module, e.Key), + fmt.Sprintf("No manifest reads %s's %s any more, but %s still run a build that does, so the operator is not asked to remove it.", e.Module, e.Key, strings.Join(e.Nodes, ", ")), + "bring those nodes up to date, or cancel the retirement with config_retiring.", + fmt.Sprintf("Resolved: no node reads %s's %s", e.Module, e.Key), + "retiring since "+e.Since.UTC().Format(time.RFC3339)+", still read on "+strings.Join(e.Nodes, ", "), at) +} + +// retireUnanswered is the finding of a retirement asked twice and not answered. +func retireUnanswered(e Retiring, at time.Time) health.Finding { + return finding(FindingRetireUnanswered, e.Module, "", e.Key, + fmt.Sprintf("the operator did not answer the ask to remove %s's %s, asked twice", e.Module, e.Key), + fmt.Sprintf("%s's %s waits for an answer", e.Module, e.Key), + fmt.Sprintf("No manifest reads %s's %s any more. The operator was asked twice whether to remove it and did not answer, so it stays.", e.Module, e.Key), + "answer the ask again with config_retiring ask-again, or cancel it.", + fmt.Sprintf("Resolved: %s's %s is answered or read again", e.Module, e.Key), + "asked "+e.Asked.UTC().Format(time.RFC3339)+" and once more", at) +} diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 93b271a80..40ad38d2c 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -100,9 +100,11 @@ func (runtimeAsks) Warranted(ask string) error { } func main() { - c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}} + c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}, + Delivery: toolDelivery{ask: stdio.Ask}, Vault: toolVault{ask: stdio.Ask, recorder: recorder}} go resolving(c) go hearing(c) + go hearingDelivery(c) if err := stdio.Serve("", Tools(c)); err != nil { logf("%v", err) os.Exit(1) @@ -221,9 +223,69 @@ func hearing(c *Config) { } } +// hearingDelivery takes the delivery parts' words that close a rename window or retire a value, each asked again until +// the runtime takes the subscription: the status view's `delivered`, the node-engine seat's `manifest-reads-changed` +// (a returning node-engine; design 32) and the indexer's `manifest-accepted`. At its start it judges the windows once, +// since an event said while this module was down is not said again. +func hearingDelivery(c *Config) { + <-stdio.Served() + for _, h := range []struct { + pattern string + handle func(stdio.Envelope) []string + }{ + {"mesh-delivery-view.delivered", func(stdio.Envelope) []string { return c.Windows("") }}, + {"node-engine.manifest-reads-changed", func(e stdio.Envelope) []string { + var b struct { + Node string `json:"node"` + } + _ = json.Unmarshal(e.Body, &b) + return c.Windows(b.Node) + }}, + {"mesh-delivery-indexer.manifest-accepted", func(e stdio.Envelope) []string { + var b struct { + Module string `json:"module"` + } + if json.Unmarshal(e.Body, &b) != nil { + return nil + } + return c.Retirements(b.Module) + }}, + } { + go func(pattern string, handle func(stdio.Envelope) []string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(pattern, func(e stdio.Envelope) error { + for _, s := range handle(e) { + logf("[mesh-config] %s", s) + } + return nil + }) + if err == nil { + return + } + logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait) + time.Sleep(wait) + } + }(h.pattern, h.handle) + } + for _, s := range c.Windows("") { + logf("[mesh-config] %s", s) + } +} + // Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them. func Tools(c *Config) []stdio.Tool { - return append(switchTools(c), stdio.Tool{ + renames := stdio.Tool{Name: Seat + "." + RenamesVerb, + Description: "The open renames of config values: for each, the module, the old and the new name, and the nodes " + + "still holding the old name open (the rename window). The checker asks it before it lets a manifest add or " + + "drop renamed-from (novox/hq design 32). Narrowed by module.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + rows, err := c.Renames(strings.TrimSpace(m), true) + return map[string]any{"renames": rows}, err + }} + return append(switchTools(c), renames, stdio.Tool{ Name: Seat + "." + Verb, Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " + "one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " + diff --git a/modules/mesh-config/cmd/mesh-config/manifest_test.go b/modules/mesh-config/cmd/mesh-config/manifest_test.go index 7c3b73a96..b93fe4573 100644 --- a/modules/mesh-config/cmd/mesh-config/manifest_test.go +++ b/modules/mesh-config/cmd/mesh-config/manifest_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself @@ -35,6 +37,9 @@ func TestEveryStateAndActHasItsTool(t *testing.T) { PlacementRead: "config list", WantsRead: "config list", BuildsRead: "config list", ModulesRead: DeclaredTool, ManifestsRead: DeclaredTool, IndexerPredefinedRead: "config list", CandidatesRead: ResyncTool, "a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, + // design 32: each new state with the tool that reads it, each act with the tool that starts it. + RenamesState: RenamesTool, RetiringState: RetiringTool, FindingsState: "config_renames and config_retiring", + "a move refused": RenameAgainTool, "a window dropped": RenameCloseTool, "an ask again": RetiringTool, } served := map[string]bool{} for _, tool := range Tools(&Config{}) { @@ -43,6 +48,21 @@ func TestEveryStateAndActHasItsTool(t *testing.T) { if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") { t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs) } + // The stream and its events are declared as the module emits them, and the manifest decodes whole. + if dm, err := catalogue.Decode(raw); err != nil { + t.Errorf("the manifest does not decode: %v", err) + } else { + emits := strings.Join(dm.Emits, ",") + for _, e := range []string{ConfigChanged, EventRenamed, EventRetiring, EventRetired} { + if !strings.Contains(emits, e) { + t.Errorf("the manifest does not declare the event %s", e) + } + } + if len(dm.Streams) != 1 || dm.Streams[0].Name != StreamName || dm.Streams[0].Days() != 7 || !dm.Streams[0].Carries(EventRenamed) || + !dm.Streams[0].Carries(EventRetiring) || !dm.Streams[0].Carries(EventRetired) || dm.Streams[0].Carries(ConfigChanged) { + t.Errorf("streams %+v: want %s, seven days, carrying renamed, retiring and retired but not config-changed", dm.Streams, StreamName) + } + } for _, verb := range SeatVerbs { if !served[Seat+"."+verb] { t.Errorf("the seat's verb %s is not served", verb) diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index d6cb304df..d92247985 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -759,6 +759,8 @@ func (c *Config) Decide() ([]string, error) { said = append(said, c.decide(r.p, r.w)) } c.recordWarrants(now) + // The retirements' asks are taken from the router's record the same way (retiring.go). + said = append(said, c.decideRetirements()...) return said, nil } diff --git a/modules/mesh-config/cmd/mesh-config/renames.go b/modules/mesh-config/cmd/mesh-config/renames.go new file mode 100644 index 000000000..b2e29f790 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/renames.go @@ -0,0 +1,494 @@ +package main + +// A config value renamed (novox/hq ADR 0609 §4, design 32 *A config value renamed*). +// +// A declaration naming `renamed-from` moves the value set under the old name to the new name, with the value's trust +// (its Value, whole), and opens a **rename window** per node: until that node's node-engine has applied a build of the +// module that reads the new name, the old name still answers with the new value (Resolve). A window closes on the +// status view's `delivered` event and, for a node that was away, on the node-engine seat's `manifest-reads-changed` +// (Windows). Its state is `renames`, key `.`; what it says is `events.renamed` on the stream `config-events`. + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" +) + +// The state, the stream's events and the bounds this part keeps. +const ( + RenamesState = "renames" + // EventRenamed is `renamed`, EventRetiring `retiring` and EventRetired `retired`, the family `config.events` + // (design 26): each carried by the stream StreamName and kept seven days. + EventRenamed = "events.renamed" + EventRetiring = "events.retiring" + EventRetired = "events.retired" + StreamName = "config-events" + // HeldFinding is how long a window or a retirement may stay open before it is a finding naming the node. + HeldFinding = 7 * 24 * time.Hour +) + +// Rename is one rename, as `renames` keeps it, key `.`. +type Rename struct { + Module string `json:"module"` + From string `json:"from"` + To string `json:"to"` + At time.Time `json:"at"` + // Moved are the nodes a value was moved on, by the last move of each. + Moved []string `json:"moved,omitempty"` + // Nodes are the windows, by node. + Nodes map[string]Window `json:"nodes"` + // Said says `renamed` was emitted; false until the bus took it. + Said bool `json:"said"` +} + +// Window is one node's rename window: open until Closed is set. +type Window struct { + Since time.Time `json:"since"` + Closed *time.Time `json:"closed,omitempty"` + // How is what closed it: "applied" (the node runs a build reading the new name), "unplaced" (the module left the + // node), or "by hand" (config_rename_close). + How string `json:"how,omitempty"` +} + +// Open is the nodes whose window is open, sorted. +func (r Rename) Open() []string { + var out []string + for node, w := range r.Nodes { + if w.Closed == nil { + out = append(out, node) + } + } + sort.Strings(out) + return out +} + +// renameKey is a rename's key in `renames`. +func renameKey(module, to string) string { return module + "." + to } + +// RenamedEvent is the body of `renamed`. +type RenamedEvent struct { + Module string `json:"module"` + From string `json:"from"` + To string `json:"to"` + At time.Time `json:"at"` +} + +// readRenames is every rename kept, by key. +func (c *Config) readRenames() (map[string]Rename, error) { + out := map[string]Rename{} + err := c.each(RenamesState, func(key string, raw []byte) error { + var r Rename + if json.Unmarshal(raw, &r) != nil || r.Module == "" || renameKey(r.Module, r.To) != key { + return nil // not one this module wrote + } + if r.Nodes == nil { + r.Nodes = map[string]Window{} + } + out[key] = r + return nil + }) + return out, err +} + +// moduleValueKey is the key of a module's own value on a node. +func moduleValueKey(node, module, key string) string { + return Level{Kind: LevelModule, Node: node, Module: module}.ValueKey(key) +} + +// applyRenames moves the values of every rename the trunk's manifests declare, opens the windows of the nodes the +// module is placed on, and writes `renamed` once: run on each resolution, under the lock, before the resolved values +// are written. s.Values and s.Renames are brought to what it did, so the same resolution sees it. A rename it refuses +// is a finding (findings.go), cleared once it moved or `renamed-from` left the manifest. It answers a line for what it +// did or could not do. +func (c *Config) applyRenames(s *Sources) []string { + held, err := c.readRenames() + if err != nil { + return []string{"the renames cannot be read: " + err.Error()} + } + now := c.now() + var said []string + seen := map[string]bool{} + for _, module := range modulesOf(s.Assignments) { + trunk, err := c.trunkDeclarations(module) + if err != nil { + continue // unresolved, said by the resolution + } + for _, to := range sortedKeys(trunk.Config) { + decl := trunk.Config[to] + if decl.RenamedFrom == "" { + continue + } + key := renameKey(module, to) + seen[key] = true + said = append(said, c.rename(s, held, trunk, to, decl, now)...) + } + } + // A rename whose `renamed-from` left the trunk's manifest goes once every window is closed. + for key, r := range held { + if seen[key] { + continue + } + if len(r.Open()) > 0 { + s.Renames[key] = r + continue + } + if err := c.Store.Delete(RenamesState, key); err != nil { + said = append(said, fmt.Sprintf("the rename %s is not removed: %v", key, err)) + continue + } + delete(s.Renames, key) + c.clearFinding(FindingRenameRefused, r.Module, r.To) + c.clearFinding(FindingWindowHeld, r.Module, r.To) + } + c.sweepRenameFindings(s, now) + return said +} + +// rename judges and applies one declared rename. +func (c *Config) rename(s *Sources, held map[string]Rename, trunk ModuleDeclarations, to string, decl Declaration, + now time.Time) []string { + module, from := trunk.Module, decl.RenamedFrom + key := renameKey(module, to) + nodes := nodesPlaced(s, module) + var said []string + refuse := func(why string) []string { + c.setFinding(renameRefused(module, to, from, why, now)) + if r, ok := held[key]; ok { + s.Renames[key] = r + } + return []string{fmt.Sprintf("the rename of %s's %s to %s is refused: %s", module, from, to, why)} + } + if _, both := trunk.Config[from]; both { + return refuse("the manifest declares both names") + } + if from == to { + return refuse("the new name is the old one") + } + trusted := decl.Trusted + if p, ok := s.Predefined[decl.Predefined]; ok && p.Trusted { + trusted = true + } + type move struct { + node string + old, new Value + hadNew bool + oldKey string + newKey string + } + var moves []move + for _, node := range nodes { + oldKey, newKey := moduleValueKey(node, module, from), moduleValueKey(node, module, to) + ov, hadOld := s.Values[oldKey] + if !hadOld { + continue + } + nv, hadNew := s.Values[newKey] + switch { + case hadNew && string(compact(nv.Value)) != string(compact(ov.Value)): + return refuse(fmt.Sprintf("on %s both names hold a value, and they differ (%s under the old name, %s under the new)", + node, ov.Value, nv.Value)) + case trusted && ov.Approved == "": + return refuse(fmt.Sprintf("on %s the value under the old name was set without the operator's Approve, and "+ + "the new name is trusted (a rename keeps a value's trust, novox/hq ADR 0348 §8)", node)) + } + moves = append(moves, move{node: node, old: ov, new: nv, hadNew: hadNew, oldKey: oldKey, newKey: newKey}) + } + r, had := held[key] + if !had { + r = Rename{Module: module, From: from, To: to, At: now, Nodes: map[string]Window{}} + } + changed := !had + for _, m := range moves { + if !m.hadNew { + if err := c.Store.Put(ValuesState, m.newKey, m.old); err != nil { + return refuse("the value on " + m.node + " is not written under the new name: " + err.Error()) + } + } + if err := c.Store.Delete(ValuesState, m.oldKey); err != nil { + return refuse("the old name's value on " + m.node + " is not removed: " + err.Error()) + } + s.Values[m.newKey] = m.old + delete(s.Values, m.oldKey) + lvl := Level{Kind: LevelModule, Node: m.node, Module: module} + _ = c.Store.Append(HistoryLog, m.newKey, Change{Action: "rename", Level: &lvl, Key: to, Value: m.old.Value, + Was: m.old.Value, By: Module, Via: Module, Why: "renamed from " + from + " (renamed-from)", At: now}) + if !slices.Contains(r.Moved, m.node) { + r.Moved = append(r.Moved, m.node) + sort.Strings(r.Moved) + } + changed = true + said = append(said, fmt.Sprintf("%s's %s on %s moved to %s", module, from, m.node, to)) + } + for _, node := range nodes { + if _, ok := r.Nodes[node]; !ok { + r.Nodes[node] = Window{Since: now} + changed = true + } + } + c.clearFinding(FindingRenameRefused, module, to) + if !r.Said { + if err := c.Store.Emit(EventRenamed, RenamedEvent{Module: module, From: from, To: to, At: r.At}); err != nil { + said = append(said, fmt.Sprintf("renamed for %s's %s is not said yet: %v", module, to, err)) + } else { + r.Said, changed = true, true + } + } + if changed { + if err := c.Store.Put(RenamesState, key, r); err != nil { + said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err)) + } + } + s.Renames[key] = r + return said +} + +// nodesPlaced is the nodes a module is placed on, sorted. +func nodesPlaced(s *Sources, module string) []string { + var out []string + for _, a := range s.Assignments { + if a.Module == module { + out = append(out, a.Node) + } + } + sort.Strings(out) + return slices.Compact(out) +} + +// resolveOldNames adds, for each rename with a window open on a node, the old name answering the new name's value on +// that node (design 32: until then mesh-config answers both names). +func resolveOldNames(s Sources, out map[string]Resolved) { + for _, key := range sortedKeys(s.Renames) { + r := s.Renames[key] + for _, node := range r.Open() { + nv, ok := out[ResolvedKey(node, r.Module, r.To)] + if !ok { + continue + } + old := nv + old.Key = r.From + out[ResolvedKey(node, r.Module, r.From)] = old + } + } +} + +// ---- the windows -------------------------------------------------------------------------------------------- + +// Windows closes the rename windows (and the retirements waiting for nodes) that the delivery says are done, at start +// and on each `delivered`; for node, the one a returning node-engine's `manifest-reads-changed` names, it first asks +// the node-engine seat's `manifest-reads`: a node-engine that does not serve it states nothing and its window stays +// open (design 32). It answers a line for each window closed. +func (c *Config) Windows(node string) []string { + c.mu.Lock() + defer c.mu.Unlock() + said := c.closeWindows(node) + said = append(said, c.drainRetirements(node)...) + if len(said) > 0 { + if _, err := c.reconcileLocked(); err != nil { + said = append(said, "resolving after the windows: "+err.Error()) + } + } + return said +} + +// closeWindows is Windows' rename half; the caller holds the lock. +func (c *Config) closeWindows(only string) []string { + renames, err := c.readRenames() + if err != nil { + return []string{"the renames cannot be read: " + err.Error()} + } + if c.Delivery == nil { + return nil + } + s, err := c.sources() + if err != nil { + return []string{"the windows cannot be judged: " + err.Error()} + } + now := c.now() + var said []string + for _, key := range sortedKeys(renames) { + r := renames[key] + changed := false + for _, node := range r.Open() { + if only != "" && only != node { + continue + } + how, why := c.windowShut(s, r, node, only != "") + if why != "" { + said = append(said, fmt.Sprintf("%s's window for %s stays open on %s: %s", r.Module, r.To, node, why)) + } + if how == "" { + continue + } + w := r.Nodes[node] + w.Closed, w.How = &now, how + r.Nodes[node] = w + changed = true + said = append(said, fmt.Sprintf("%s's window for %s is closed on %s (%s)", r.Module, r.To, node, how)) + } + if !changed { + continue + } + if err := c.Store.Put(RenamesState, key, r); err != nil { + said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err)) + } + if len(r.Open()) == 0 { + c.clearFinding(FindingWindowHeld, r.Module, r.To) + } + } + return said +} + +// windowShut says how a node's window may close now ("" when it may not) and, when it may not, why. +func (c *Config) windowShut(s Sources, r Rename, node string, returning bool) (how, why string) { + if !slices.Contains(nodesPlaced(&s, r.Module), node) { + return "unplaced", "" + } + if returning { + if err := c.Delivery.Reads(node); err != nil { + return "", "the node-engine states nothing yet (" + err.Error() + ")" + } + } + fp, found, err := c.Delivery.Have(node, r.Module) + switch { + case err != nil: + return "", "its have cannot be read: " + err.Error() + case !found || fp == "": + return "", "it reports no build of the module" + } + d, err := c.builtDeclarations(r.Module, fp) + if err != nil { + return "", "the build it runs cannot be read: " + err.Error() + } + if _, reads := d.Config[r.To]; !reads { + return "", "it still runs a build that reads " + r.From + } + return "applied", "" +} + +// ---- the tools --------------------------------------------------------------------------------------------- + +// RenameRow is one rename as the tools say it. +type RenameRow struct { + Module string `json:"module"` + From string `json:"from"` + To string `json:"to"` + At time.Time `json:"at"` + Moved []string `json:"moved,omitempty"` + // Open are the nodes holding the old name open; Closed the others, each with how. + Open []string `json:"open"` + Closed map[string]string `json:"closed,omitempty"` +} + +// Renames lists the renames, the open ones only for the seat's verb, narrowed by module. +func (c *Config) Renames(module string, openOnly bool) ([]RenameRow, error) { + renames, err := c.readRenames() + if err != nil { + return nil, err + } + rows := []RenameRow{} + for _, key := range sortedKeys(renames) { + r := renames[key] + if module != "" && r.Module != module { + continue + } + row := RenameRow{Module: r.Module, From: r.From, To: r.To, At: r.At, Moved: r.Moved, Open: r.Open()} + if row.Open == nil { + row.Open = []string{} + } + if openOnly && len(row.Open) == 0 { + continue + } + for node, w := range r.Nodes { + if w.Closed != nil { + if row.Closed == nil { + row.Closed = map[string]string{} + } + row.Closed[node] = w.How + } + } + rows = append(rows, row) + } + return rows, nil +} + +// RenameAgain moves a rename's values now, the way a resolution does, and answers what it did: the recovery for a +// refused move once its cause is put right. +func (c *Config) RenameAgain(module, to string) (any, error) { + c.mu.Lock() + defer c.mu.Unlock() + if module == "" || to == "" { + return nil, fmt.Errorf("name the module and the new name of the rename") + } + r, err := c.reconcileLocked() + if err != nil { + return r, err + } + renames, err := c.readRenames() + if err != nil { + return nil, err + } + if _, ok := renames[renameKey(module, to)]; !ok { + return nil, fmt.Errorf("no rename of %s to %s is kept: its trunk manifest declares no renamed-from for %s, or the rename is refused (config_renames and the findings say which)", module, to, to) + } + return map[string]any{"said": r.Renamed, "moved": r.Moved}, nil +} + +// RenameClose drops a node from a rename's window by hand, recorded as a hand act. +func (c *Config) RenameClose(module, to, node, why string) (any, error) { + if strings.TrimSpace(why) == "" { + return nil, fmt.Errorf("say why the window is dropped: a hand act keeps its why") + } + c.mu.Lock() + defer c.mu.Unlock() + key := renameKey(module, to) + renames, err := c.readRenames() + if err != nil { + return nil, err + } + r, ok := renames[key] + if !ok { + return nil, fmt.Errorf("no rename of %s to %s is kept", module, to) + } + w, ok := r.Nodes[node] + if !ok || w.Closed != nil { + return nil, fmt.Errorf("%s holds no open window for %s's %s; it holds: %s", node, module, to, strings.Join(r.Open(), ", ")) + } + now := c.now() + w.Closed, w.How = &now, "by hand" + r.Nodes[node] = w + if err := c.Store.HandAct(fmt.Sprintf("config_rename_close %s %s on %s", module, to, node), why, + "the operator dropped a node from the rename window of "+key); err != nil { + return nil, fmt.Errorf("the hand-act log does not hold it, so the window is not dropped: %w", err) + } + if err := c.Store.Put(RenamesState, key, r); err != nil { + return nil, fmt.Errorf("the window is not dropped: %w", err) + } + if len(r.Open()) == 0 { + c.clearFinding(FindingWindowHeld, r.Module, r.To) + } + if _, err := c.reconcileLocked(); err != nil { + return nil, err + } + return map[string]any{"closed": node, "open": r.Open()}, nil +} + +// sweepRenameFindings writes a finding for each window held open past HeldFinding, naming the node, and clears one +// whose window closed. +func (c *Config) sweepRenameFindings(s *Sources, now time.Time) { + for _, key := range sortedKeys(s.Renames) { + r := s.Renames[key] + var held []string + for _, node := range r.Open() { + if now.Sub(r.Nodes[node].Since) >= HeldFinding { + held = append(held, node) + } + } + c.clearFinding(FindingWindowHeld, r.Module, r.To, held...) + for _, node := range held { + c.setFinding(windowHeld(r, node, now)) + } + } +} diff --git a/modules/mesh-config/cmd/mesh-config/renames_test.go b/modules/mesh-config/cmd/mesh-config/renames_test.go new file mode 100644 index 000000000..d3e705b55 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/renames_test.go @@ -0,0 +1,436 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// fakeDelivery is the delivery parts as mesh-config asks them: each node's have, whether a node-engine states what it +// reads, and who reads a module's keys (nil rows and known false: the indexer does not serve the tool yet). +type fakeDelivery struct { + mu sync.Mutex + haves map[string]string // "." → fingerprint + notSays map[string]bool // node → its node-engine states nothing + rows []ReaderRow + known bool +} + +func (d *fakeDelivery) Have(node, module string) (string, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + fp, ok := d.haves[node+"."+module] + return fp, ok, nil +} + +func (d *fakeDelivery) Reads(node string) error { + d.mu.Lock() + defer d.mu.Unlock() + if d.notSays[node] { + return errors.New("the verb manifest-reads is not served") + } + return nil +} + +func (d *fakeDelivery) Readers(string) ([]ReaderRow, bool, error) { + d.mu.Lock() + defer d.mu.Unlock() + return d.rows, d.known, nil +} + +// fakeVault records the retirements asked of the seat mesh-vault. +type fakeVault struct{ asked []RetireRequest } + +func (v *fakeVault) Retire(r RetireRequest) error { v.asked = append(v.asked, r); return nil } + +// The fingerprints of the first, second and third build of player. +var ( + fp1 = "src2:" + strings.Repeat("1", 64) + fp2 = "src2:" + strings.Repeat("2", 64) + fp3 = "src2:" + strings.Repeat("3", 64) +) + +const ( + oldPlayer = `{"module":"player","config":{"volume":{"type":"number","default":5}}}` + newPlayer = `{"module":"player","config":{"level":{"type":"number","default":5,"renamed-from":"volume"}}}` + // donePlayer is the manifest once `renamed-from` left it. + donePlayer = `{"module":"player","config":{"level":{"type":"number","default":5}}}` +) + +// aRenamingMesh is a mesh with player on g14 and novox, both running its first build (reading `volume`), a clock the +// test moves, and the delivery parts faked. +func aRenamingMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *time.Time) { + t.Helper() + m := newStore() + m.declare(t, "player", oldPlayer) + indexerList(t, m, "aaaaaaaaaaaa", theList) + for _, node := range []string{"g14", "novox"} { + m.assign(t, node, "player") + m.want(t, node, "player", fp1, oldPlayer) + } + clock := time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC) + d := &fakeDelivery{haves: map[string]string{"g14.player": fp1, "novox.player": fp1}, notSays: map[string]bool{}} + c := &Config{Store: m, Now: func() time.Time { return clock }, Logf: t.Logf, Delivery: d} + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + return c, m, d, &clock +} + +func findingsOf(t *testing.T, m *memStore, kind string) []health.Finding { + t.Helper() + var out []health.Finding + for _, k := range m.keysOf(FindingsState) { + e, _ := m.Get(FindingsState, k) + f, err := health.DecodeFinding(e.Value) + if err != nil { + t.Fatalf("finding %s: %v", k, err) + } + if f.Kind == kind { + out = append(out, f) + } + } + return out +} + +// **A renamed value moves with its trust, both names answer until the node-engines run the new build, and the window +// closes per node** (design 32, C20): the value set under the old name is under the new one with no operator input, +// `renamed` is said once, the old name answers the new value on a node until that node's have runs a build that reads the +// new name (on `delivered`, or for a returning node on `manifest-reads-changed` once its node-engine states what it +// reads), and the rename goes once every window is closed and `renamed-from` left the manifest. +func TestARenamedValueMovesAndTheOldNameAnswersUntilEachNodeRunsTheNewBuild(t *testing.T) { + c, m, d, _ := aRenamingMesh(t) + set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"}) + m.declare(t, "player", newPlayer) + r, err := c.Reconcile() + if err != nil || len(r.Renamed) != 1 { + t.Fatalf("reconcile: %+v, %v", r, err) + } + if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" || v.By == "" { + t.Errorf("the value is %+v under the new name (set: %v)", v, ok) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); ok { + t.Error("the old name still holds a value") + } + for _, c := range []struct{ node, key, want string }{{"g14", "level", "8"}, {"g14", "volume", "8"}, {"novox", "level", "5"}, {"novox", "volume", "5"}} { + if r := m.resolved(t, c.node, "player", c.key); string(r.Value) != c.want { + t.Errorf("%s answers %s on %s as %s, want %s", c.key, "player", c.node, r.Value, c.want) + } + } + if got := m.saidOf(EventRenamed); len(got) != 1 || got[0].(RenamedEvent).From != "volume" || got[0].(RenamedEvent).To != "level" { + t.Errorf("renamed was said as %+v", got) + } + // The history keeps the move; a second pass says and moves nothing more. + if h, _ := c.History(Call{Module: "player", Node: "g14"}); len(h) == 0 || h[len(h)-1].Action != "resolve" && h[len(h)-1].Action != "rename" { + t.Errorf("history %+v", h) + } + if _, err := c.Reconcile(); err != nil || len(m.saidOf(EventRenamed)) != 1 { + t.Errorf("a second pass said renamed %d times: %v", len(m.saidOf(EventRenamed)), err) + } + rows, _ := c.Renames("", true) + if len(rows) != 1 || strings.Join(rows[0].Open, ",") != "g14,novox" || strings.Join(rows[0].Moved, ",") != "g14" { + t.Fatalf("renames %+v", rows) + } + + // A delivery while both nodes still run the first build closes nothing. + if said := c.Windows(""); len(said) != 2 || !strings.Contains(said[0], "still runs a build that reads volume") { + t.Errorf("windows said %v", said) + } + // g14 applies the new build. + m.want(t, "g14", "player", fp2, newPlayer) + d.haves["g14.player"] = fp2 + if said := c.Windows(""); len(said) == 0 { + t.Error("g14 applied the new build and its window did not close") + } + if _, err := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); err != nil { + t.Fatal(err) + } else if e, _ := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); e != nil { + t.Error("g14 still answers the old name after its window closed") + } + m.resolved(t, "novox", "player", "volume") // novox still answers it + // novox applies it, but its node-engine, returning, states nothing yet: the window stays open. + m.want(t, "novox", "player", fp2, newPlayer) + d.haves["novox.player"] = fp2 + d.notSays["novox"] = true + c.Windows("novox") + if rows, _ := c.Renames("", true); len(rows) != 1 || strings.Join(rows[0].Open, ",") != "novox" { + t.Errorf("a node-engine that states nothing closed its window: %+v", rows) + } + d.notSays["novox"] = false + c.Windows("novox") + if rows, _ := c.Renames("", true); len(rows) != 0 { + t.Errorf("windows still open: %+v", rows) + } + // The rename stays while `renamed-from` is in the manifest, and goes when it left. + if all, _ := c.Renames("", false); len(all) != 1 || all[0].Closed["g14"] != "applied" { + t.Errorf("renames %+v", all) + } + m.declare(t, "player", donePlayer) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + if all, _ := c.Renames("", false); len(all) != 0 { + t.Errorf("the rename stayed after renamed-from left the manifest: %+v", all) + } +} + +// A refused move is a finding, cleared once it moved (C4): both names declared, both names holding different values, a +// trusted new name over a value set without the operator's Approve. Nothing is moved, nothing is said. +func TestARefusedRenameIsAFindingAndIsClearedOnceItMoves(t *testing.T) { + both := `{"module":"player","config":{"volume":{"type":"number","default":5},"level":{"type":"number","default":5,"renamed-from":"volume"}}}` + trusted := `{"module":"player","config":{"level":{"type":"number","default":5,"trusted":true,"renamed-from":"volume"}}}` + for _, tc := range []struct { + name, manifest string + both bool + why string + fixed string + }{ + {"both names declared", both, false, "declares both names", newPlayer}, + {"both names hold a value", newPlayer, true, "both names hold a value", newPlayer}, + {"a trusted name over an unapproved value", trusted, false, "without the operator's Approve", newPlayer}, + } { + t.Run(tc.name, func(t *testing.T) { + c, m, _, _ := aRenamingMesh(t) + set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"}) + if tc.both { + // The new name is set meanwhile, to another value, in the state a faulty hand wrote it. + if err := m.Put(ValuesState, "module.g14.player.level", Value{Value: json.RawMessage("9"), By: "x", At: time.Now()}); err != nil { + t.Fatal(err) + } + } + m.declare(t, "player", tc.manifest) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + fs := findingsOf(t, m, FindingRenameRefused) + if len(fs) != 1 || !strings.Contains(fs[0].Summary, tc.why) || fs[0].Observer != Module { + t.Fatalf("findings %+v, want one saying %q", fs, tc.why) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok || len(m.saidOf(EventRenamed)) != 0 { + t.Errorf("a refused rename moved the value or said renamed") + } + // Put right, it moves by hand and the finding goes. + m.declare(t, "player", tc.fixed) + if tc.both { + _ = m.Delete(ValuesState, "module.g14.player.level") + } + if _, err := c.RenameAgain("player", "level"); err != nil { + t.Fatal(err) + } + if fs := findingsOf(t, m, FindingRenameRefused); len(fs) != 0 { + t.Errorf("the finding stayed after the move: %+v", fs) + } + if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" { + t.Errorf("the value is %+v after the move", v) + } + }) + } +} + +// A window held open seven days is a finding naming the node, cleared when it closes; dropping a node by hand is a hand +// act with its why (C4, C5), and the seat's verb `renames` names the nodes holding a window open. +func TestAWindowHeldSevenDaysIsAFindingAndAHandCloseIsAHandAct(t *testing.T) { + c, m, d, clock := aRenamingMesh(t) + m.declare(t, "player", newPlayer) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + m.want(t, "g14", "player", fp2, newPlayer) + d.haves["g14.player"] = fp2 + c.Windows("") + *clock = clock.Add(8 * 24 * time.Hour) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + fs := findingsOf(t, m, FindingWindowHeld) + if len(fs) != 1 || fs[0].Subject.Machine != "novox" || !strings.Contains(fs[0].Summary, "novox") { + t.Fatalf("findings %+v, want one naming novox", fs) + } + // The seat's verb names the node holding the window open. + var verb func(map[string]any) (any, error) + for _, tool := range Tools(c) { + if tool.Name == Seat+"."+RenamesVerb { + verb = tool.Run + } + } + got, err := verb(map[string]any{"module": "player"}) + if raw, _ := json.Marshal(got); err != nil || !strings.Contains(string(raw), `"open":["novox"]`) { + t.Errorf("renames answered %s, %v", raw, err) + } + if _, err := c.RenameClose("player", "level", "novox", ""); err == nil { + t.Error("a window was dropped with no why") + } + if _, err := c.RenameClose("player", "level", "g14", "g14 is closed already"); err == nil { + t.Error("a node holding no open window was dropped") + } + m.handActs = nil + if _, err := c.RenameClose("player", "level", "novox", "novox is retired for good"); err != nil { + t.Fatal(err) + } + if len(m.handActs) != 1 || !strings.Contains(m.handActs[0], "novox is retired for good") { + t.Errorf("hand acts %v", m.handActs) + } + if fs := findingsOf(t, m, FindingWindowHeld); len(fs) != 0 { + t.Errorf("the finding stayed after the window closed: %+v", fs) + } + if rows, _ := c.Renames("", true); len(rows) != 0 { + t.Errorf("open renames %+v", rows) + } +} + +// retiringMesh is a mesh where player no longer declares `volume`, a value of it is set on g14, and g14 runs the first +// build (which reads it). +func retiringMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *standIn, *fakeVault, *time.Time) { + t.Helper() + c, m, d, clock := aRenamingMesh(t) + set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"}) + m.declare(t, "player", donePlayer) + r := newStandIn() + v := &fakeVault{} + c.Asks, c.Vault = r, v + d.known = true + d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{}}} + return c, m, d, r, v, clock +} + +func retiringOf(t *testing.T, c *Config) []RetiringRow { + t.Helper() + got, err := c.ConfigRetiring("", "", "") + if err != nil { + t.Fatal(err) + } + return got.(map[string]any)["retiring"].([]RetiringRow) +} + +// **A value no manifest reads is retired only on the operator's Approve** (C24): while a node runs a build that reads it +// it is not marked and nobody is asked; once none does, `retiring` is said and the operator is asked once; the value stays +// until the Approve, then it is removed, `retired` is said for the node it was set on, and the warrant is recorded. +func TestAValueNoManifestReadsIsRetiredOnlyOnTheOperatorsApprove(t *testing.T) { + c, m, d, r, _, _ := retiringMesh(t) + // The indexer does not serve the tool yet: nothing retires on that. + d.known = false + if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "does not say") || len(retiringOf(t, c)) != 0 { + t.Errorf("an indexer without the tool: %v", said) + } + d.known = true + // g14 still runs a build that reads volume: draining, no ask, no event. + c.Retirements("player") + if rows := retiringOf(t, c); len(rows) != 1 || rows[0].State != RetireDraining || strings.Join(rows[0].Nodes, ",") != "g14,novox" { + t.Fatalf("retiring %+v, want draining on g14 and novox", rows) + } + if len(r.asks) != 0 || len(m.saidOf(EventRetiring)) != 0 { + t.Fatalf("asked %d times and said retiring %d times while two nodes read the value", len(r.asks), len(m.saidOf(EventRetiring))) + } + // Both nodes apply the build that reads it no longer. + for _, node := range []string{"g14", "novox"} { + m.want(t, node, "player", fp3, donePlayer) + d.haves[node+".player"] = fp3 + } + c.Windows("") + rows := retiringOf(t, c) + if len(rows) != 1 || rows[0].State != RetireAsked || len(r.asks) != 1 || len(m.saidOf(EventRetiring)) != 1 { + t.Fatalf("retiring %+v, %d asks, %d events", rows, len(r.asks), len(m.saidOf(EventRetiring))) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok { + t.Fatal("the value was removed before the operator answered") + } + // A second manifest-accepted asks nobody again. + c.Retirements("player") + if len(r.asks) != 1 { + t.Errorf("asked %d times", len(r.asks)) + } + r.choose(t, rows[0].Ask, OptionApprove, nil) + m.handActs = nil + if said, err := c.Decide(); err != nil || len(said) == 0 { + t.Fatalf("decide: %v, %v", said, err) + } + if _, ok := storedValue(t, m, "module.g14.player.volume"); ok { + t.Error("the approved value is still set") + } + if got := m.saidOf(EventRetired); len(got) != 1 || got[0].(RetiredEvent).Node != "g14" || got[0].(RetiredEvent).Name != "volume" { + t.Errorf("retired was said as %+v", got) + } + if len(retiringOf(t, c)) != 0 || len(m.handActs) == 0 { + t.Errorf("retiring %+v, hand acts %v", retiringOf(t, c), m.handActs) + } +} + +// A declined retirement is not asked again until the value is read again; a value read again is dropped; an ask unanswered +// is asked once more after seven days and then written as a finding, cleared by an answer or a cancel; a secret is +// retired through the seat mesh-vault with the ask that warranted it (C24, C7). +func TestARetirementDeclinedUnansweredCancelledOrASecret(t *testing.T) { + c, m, d, r, v, clock := retiringMesh(t) + for _, node := range []string{"g14", "novox"} { + m.want(t, node, "player", fp3, donePlayer) + d.haves[node+".player"] = fp3 + } + c.Retirements("player") + ask := retiringOf(t, c)[0].Ask + + // Unanswered: the ask expires, a week passes, it is asked once more, expires again: a finding. + r.choose(t, ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired }) + c.Decide() + if rows := retiringOf(t, c); rows[0].State != RetireUnanswered || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 { + t.Fatalf("after the first expiry: %+v", rows) + } + *clock = clock.Add(8 * 24 * time.Hour) + c.Decide() + rows := retiringOf(t, c) + if rows[0].State != RetireAsked || len(r.asks) != 2 || rows[0].Ask == ask { + t.Fatalf("after a week: %+v, %d asks", rows, len(r.asks)) + } + r.choose(t, rows[0].Ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired }) + c.Decide() + if fs := findingsOf(t, m, FindingRetireUnanswered); len(fs) != 1 { + t.Fatalf("findings %+v after two asks unanswered", fs) + } + // Asked again by hand, then declined: the finding goes and it is not asked again. + if _, err := c.ConfigRetiring("ask-again", "player", "volume"); err != nil { + t.Fatal(err) + } + third := retiringOf(t, c)[0].Ask + r.choose(t, third, OptionDecline, nil) + c.Decide() + if rows := retiringOf(t, c); rows[0].State != RetireDeclined || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 { + t.Fatalf("after the decline: %+v", rows) + } + asked := len(r.asks) + c.Retirements("player") + if len(r.asks) != asked { + t.Error("a declined retirement was asked again") + } + // Read again: dropped. + d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{"player"}}} + if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "read again") || len(retiringOf(t, c)) != 0 { + t.Errorf("read again: %v", said) + } + + // A secret: its nodes come from the indexer's answer; on the Approve the vault is asked for each, once. + d.rows = []ReaderRow{{Module: "player", Key: "token", Kind: KindSecret, Readers: []string{}, Nodes: []string{}}} + c.Retirements("player") + secret := retiringOf(t, c)[0] + if secret.Kind != KindSecret || secret.State != RetireAsked { + t.Fatalf("the secret: %+v", secret) + } + r.choose(t, secret.Ask, OptionApprove, nil) + c.Decide() + if len(v.asked) != 2 || v.asked[0].Node != "g14" || v.asked[0].Name != "token" || v.asked[0].Ask != secret.Ask { + t.Errorf("the vault was asked %+v", v.asked) + } + if len(m.saidOf(EventRetired)) != 2 || len(retiringOf(t, c)) != 0 { + t.Errorf("retired %d times, retiring %+v", len(m.saidOf(EventRetired)), retiringOf(t, c)) + } + // A secret whose readers' nodes the indexer cannot name is not marked. + d.rows = []ReaderRow{{Module: "player", Key: "other", Kind: KindSecret, Readers: []string{}}} + if c.Retirements("player"); len(retiringOf(t, c)) != 0 { + t.Errorf("a secret read on unknown nodes was marked: %+v", retiringOf(t, c)) + } + _ = fmt.Sprint +} diff --git a/modules/mesh-config/cmd/mesh-config/resolve.go b/modules/mesh-config/cmd/mesh-config/resolve.go index f911c225c..72f41fd81 100644 --- a/modules/mesh-config/cmd/mesh-config/resolve.go +++ b/modules/mesh-config/cmd/mesh-config/resolve.go @@ -150,6 +150,8 @@ type Sources struct { // for them (pulls.go); neither is resolved. Pulls map[string]PullDecl Kept map[string]KeptValue + // Renames are the renames kept, by `.`, whose open windows answer the old name (renames.go). + Renames map[string]Rename } // Unresolved is a module placed on a node whose values are not resolved, and why. @@ -192,6 +194,7 @@ func Resolve(s Sources) map[string]Resolved { out[ResolvedKey(a.Node, a.Module, key)] = r } } + resolveOldNames(s, out) return out } diff --git a/modules/mesh-config/cmd/mesh-config/retiring.go b/modules/mesh-config/cmd/mesh-config/retiring.go new file mode 100644 index 000000000..e854737c1 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/retiring.go @@ -0,0 +1,581 @@ +package main + +// A value no longer read (novox/hq ADR 0609, design 32 *A value no longer read*). +// +// A config value or secret that no manifest on any trunk reads any more is a **retiring value**. mesh-config learns it +// from the indexer's tool `indexer_config_readers`, asked on each of the indexer's `manifest-accepted` (Retirements); +// a tool the indexer does not serve yet answers "not known" and nothing retires. A value is marked retiring only once no +// node-engine reports applied a build that reads it (State draining until then, judged again on each `delivered`); +// mesh-config then says `retiring` and asks the operator once, through the operator channel, whether to remove it. It is +// removed only on the operator's Approve: a value by mesh-config, a secret through the seat `mesh-vault`'s verb +// `retire`. An ask unanswered after seven days is asked once more, and then written as a finding. + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// The state, and where a retirement stands. +const ( + RetiringState = "retiring" + // RetireVerb is the act an answer binds. + RetireVerb = Seat + ".retire" + + RetireDraining = "draining" // a node still runs a build that reads it + RetireAsked = "asked" // the operator is asked + RetireUnanswered = "unanswered" // the ask ended with no choice + RetireApproved = "approved" // the operator approved; being removed + RetireDeclined = "declined" // the operator declined, or a person cancelled: kept until the value is read again +) + +// Kinds of retiring value. +const ( + KindValue = "value" + KindSecret = "secret" +) + +// Retiring is one retiring value, as `retiring` keeps it, key `.`. +type Retiring struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Since time.Time `json:"since"` + State string `json:"state"` + // Nodes are the nodes that still read it (draining) or that it is removed on (asked, approved). + Nodes []string `json:"nodes,omitempty"` + // Ask is the open ask's id, Asked when the first ask was made, AskedAgain whether it was asked once more. + Ask string `json:"ask,omitempty"` + // AskedAt is when the open ask was made (it expires a day on), Asked when the first was. + AskedAt time.Time `json:"asked-at,omitempty"` + Asked time.Time `json:"asked,omitempty"` + AskedAgain bool `json:"asked-again,omitempty"` + Said bool `json:"said,omitempty"` + Outcome string `json:"outcome,omitempty"` + Answered string `json:"answered,omitempty"` + // Removed are the nodes it is removed on already. + Removed []string `json:"removed,omitempty"` +} + +func retiringKey(module, key string) string { return module + "." + key } + +// RetiringEvent is the body of `retiring`; RetiredEvent of `retired`. +type RetiringEvent struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Since time.Time `json:"since"` + Ask string `json:"ask"` +} + +// RetiredEvent is the body of `retired`: one node's value removed, with the ask that warranted it. +type RetiredEvent struct { + Module string `json:"module"` + Name string `json:"name"` + Node string `json:"node"` + Ask string `json:"ask"` +} + +// ReaderRow is one key the indexer's `indexer_config_readers` answers: the modules on each trunk that declare or refer +// to it. For a secret, Nodes are the nodes whose applied build reads it, when the indexer can say. +type ReaderRow struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Readers []string `json:"readers"` + Nodes []string `json:"nodes,omitempty"` +} + +// Delivery is what the delivery parts answer to this module: a node's have, a node-engine's statement of what it +// reads, and who reads a module's config keys. Each is a tool or a seat verb on the bus, never a bucket (ADR 0596). +type Delivery interface { + // Have is the fingerprint of the build node runs of module, as the status view's node_haves says it. + Have(node, module string) (fp string, found bool, err error) + // Reads asks the node-engine seat's manifest-reads for node: an error when it is not served or the node says nothing. + Reads(node string) error + // Readers asks the indexer who reads module's keys: known false when the indexer does not serve the tool yet. + Readers(module string) (rows []ReaderRow, known bool, err error) +} + +// RetireRequest is what the vault's verb `retire` takes: one module's own secret on a node, with the hand act that +// warrants it. +type RetireRequest struct { + Node, Module, Name, Ask string +} + +// Vault is the seat mesh-vault as this module reaches it. +type Vault interface { + Retire(r RetireRequest) error +} + +func (c *Config) readRetiring() (map[string]Retiring, error) { + out := map[string]Retiring{} + err := c.each(RetiringState, func(key string, raw []byte) error { + var r Retiring + if json.Unmarshal(raw, &r) != nil || r.Module == "" || retiringKey(r.Module, r.Key) != key { + return nil + } + out[key] = r + return nil + }) + return out, err +} + +// Retirements judges a module's keys on the indexer's word that its manifest was accepted. It answers a line for each +// decision. +func (c *Config) Retirements(module string) []string { + c.mu.Lock() + defer c.mu.Unlock() + if c.Delivery == nil || module == "" { + return nil + } + rows, known, err := c.Delivery.Readers(module) + if err != nil { + return []string{fmt.Sprintf("who reads %s's config is not asked yet: %v", module, err)} + } + if !known { + return []string{fmt.Sprintf("the indexer does not say who reads %s's config yet; nothing retires on that", module)} + } + held, err := c.readRetiring() + if err != nil { + return []string{"the retirements cannot be read: " + err.Error()} + } + s, err := c.sources() + if err != nil { + return []string{"the retirements cannot be judged: " + err.Error()} + } + var said []string + for _, row := range rows { + if row.Module != "" && row.Module != module { + continue + } + row.Module = module + if row.Kind == "" { + row.Kind = KindValue + } + key := retiringKey(module, row.Key) + e, has := held[key] + if len(row.Readers) > 0 { + if has { + said = append(said, c.dropRetiring(e, "it is read again")) + } + continue + } + if has { + continue // already marked; it is advanced by drainRetirements and Decide + } + if row.Kind == KindValue && !valueHeld(s, module, row.Key) { + continue // nothing set under that name: nothing to remove + } + if row.Kind == KindSecret && row.Nodes == nil && !row.nodesKnown() { + continue // which nodes read a secret is not known, and nothing retires on that + } + e = Retiring{Module: module, Key: row.Key, Kind: row.Kind, Since: c.now(), State: RetireDraining, Nodes: row.Nodes} + said = append(said, c.advanceRetiring(s, &e)...) + } + return said +} + +func (r ReaderRow) nodesKnown() bool { return r.Nodes != nil } + +// valueHeld is whether a module's own value is set under key on any node. +func valueHeld(s Sources, module, key string) bool { + for k := range s.Values { + if l, kk, ok := parseValueKey(k); ok && l.Kind == LevelModule && l.Module == module && kk == key { + return true + } + } + return false +} + +// nodesReading is the nodes whose applied build still reads a value (secrets: the nodes the indexer last named). +func (c *Config) nodesReading(s Sources, e Retiring) []string { + if e.Kind != KindValue || c.Delivery == nil { + return e.Nodes + } + var out []string + for _, node := range nodesPlaced(&s, e.Module) { + fp, found, err := c.Delivery.Have(node, e.Module) + if err != nil || !found || fp == "" { + if err != nil { + out = append(out, node) // not known is not "no longer read" + } + continue + } + d, err := c.builtDeclarations(e.Module, fp) + if err != nil { + out = append(out, node) + continue + } + if _, reads := d.Config[e.Key]; reads { + out = append(out, node) + } + } + sort.Strings(out) + return out +} + +// advanceRetiring moves a draining retirement on: while a node reads the value it stays draining (and is written, so a +// delivery judges it again); once none does it is marked retiring, said, and asked. The caller holds the lock. +func (c *Config) advanceRetiring(s Sources, e *Retiring) []string { + key := retiringKey(e.Module, e.Key) + now := c.now() + if e.State == RetireDraining { + e.Nodes = c.nodesReading(s, *e) + if len(e.Nodes) > 0 { + if err := c.Store.Put(RetiringState, key, *e); err != nil { + return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)} + } + if now.Sub(e.Since) >= HeldFinding { + c.setFinding(retirementHeld(*e, now)) + } + return []string{fmt.Sprintf("%s's %s is not read by any manifest, and %s still run a build that reads it", + e.Module, e.Key, strings.Join(e.Nodes, ", "))} + } + c.clearFinding(FindingRetireHeld, e.Module, e.Key) + e.Nodes = nodesPlaced(&s, e.Module) + if err := c.ask(e, false); err != nil { + return []string{fmt.Sprintf("the operator is not asked about retiring %s's %s: %v", e.Module, e.Key, err)} + } + } + if !e.Said { + if err := c.Store.Emit(EventRetiring, RetiringEvent{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since, + Ask: e.Ask}); err == nil { + e.Said = true + } + } + if err := c.Store.Put(RetiringState, key, *e); err != nil { + return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)} + } + return []string{fmt.Sprintf("%s's %s is retiring: the operator is asked whether to remove it", e.Module, e.Key)} +} + +// drainRetirements judges the draining retirements again, after a delivery or a returning node; the caller holds the lock. +func (c *Config) drainRetirements(string) []string { + held, err := c.readRetiring() + if err != nil || len(held) == 0 || c.Delivery == nil { + return nil + } + s, err := c.sources() + if err != nil { + return []string{"the retirements cannot be judged: " + err.Error()} + } + var said []string + for _, key := range sortedKeys(held) { + e := held[key] + if e.State == RetireDraining { + said = append(said, c.advanceRetiring(s, &e)...) + } + } + return said +} + +// dropRetiring ends a retirement: the value is read again, or a person cancelled it. +func (c *Config) dropRetiring(e Retiring, why string) string { + if err := c.Store.Delete(RetiringState, retiringKey(e.Module, e.Key)); err != nil { + return fmt.Sprintf("the retirement of %s's %s is not dropped: %v", e.Module, e.Key, err) + } + c.clearFinding(FindingRetireUnanswered, e.Module, e.Key) + c.clearFinding(FindingRetireHeld, e.Module, e.Key) + return fmt.Sprintf("%s's %s is no longer retiring: %s", e.Module, e.Key, why) +} + +// ---- the ask ------------------------------------------------------------------------------------------------ + +// retireAskID is an ask's id: the same retirement asked again is another id. +func retireAskID(e Retiring, again bool) string { + sum := sha256.Sum256([]byte(e.Module + "\x00" + e.Key + "\x00" + e.Since.UTC().Format(time.RFC3339Nano))) + id := "ret-" + hex.EncodeToString(sum[:6]) + if again { + id += "-2" + } + return id +} + +// act is what an answer binds: the retirement, the nodes it reaches, the ask and the answer. +func (e Retiring) act(ask, option string) asks.Act { + return asks.Act{"verb": RetireVerb, "option": option, "module": e.Module, "key": e.Key, "kind": e.Kind, + "nodes": strings.Join(e.Nodes, " "), "ask": ask} +} + +// retireAsk is the ask composed from the retirement alone. +func (e Retiring) retireAsk(id string, at time.Time) asks.Ask { + nodes := strings.Join(e.Nodes, ", ") + if nodes == "" { + nodes = "no node" + } + what := "config value" + if e.Kind == KindSecret { + what = "secret" + } + a := asks.Ask{ID: id, Headline: fmt.Sprintf("Remove %s's %s %s?", e.Module, what, e.Key), + Explanation: fmt.Sprintf("No manifest reads %s's %s %s any more, and no node runs a build that reads it. Approved, it is removed on %s.", e.Module, what, e.Key, nodes), + Who: asks.Operator, Expires: at.Add(ProposalLasts), OnExpiry: "nothing is removed; mesh-config asks once more in a week", + About: "config.retire." + e.Module + "." + e.Key, + Details: fmt.Sprintf("Retiring since %s UTC. Read it with mesh-config's config_retiring.", e.Since.UTC().Format("15:04 on 2 Jan"))} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "the " + what + " is removed on " + nodes}, + {OptionDecline, "Decline", "nothing is removed; mesh-config asks no more until it is read again"}, + } { + binds, _ := asks.ActDigest(e.act(id, o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// ask makes the ask for a retirement and publishes it; the caller keeps the entry. +func (c *Config) ask(e *Retiring, again bool) error { + if c.Asks == nil { + return fmt.Errorf("the operator cannot be asked from here") + } + now := c.now() + id := retireAskID(*e, again) + a := e.retireAsk(id, now) + if err := a.Check(now); err != nil { + return err + } + if err := c.Asks.Publish(asks.AskSubject(Module), a, a.ID); err != nil { + return err + } + e.Ask, e.State, e.AskedAt = id, RetireAsked, now + if e.Asked.IsZero() { + e.Asked = now + } + e.AskedAgain = e.AskedAgain || again + return nil +} + +// decideRetirements takes the router's word on each open retirement ask; the caller holds the lock. It also asks once +// more an ask unanswered for seven days, writes the finding of one asked twice, and finishes an approved removal. +func (c *Config) decideRetirements() []string { + held, err := c.readRetiring() + if err != nil || len(held) == 0 { + return nil + } + now := c.now() + var said []string + for _, key := range sortedKeys(held) { + e := held[key] + switch e.State { + case RetireAsked: + said = append(said, c.decideRetirement(&e, now)...) + case RetireUnanswered: + if !e.AskedAgain && now.Sub(e.Asked) >= HeldFinding { + if err := c.ask(&e, true); err != nil { + said = append(said, fmt.Sprintf("the operator is not asked again about %s's %s: %v", e.Module, e.Key, err)) + } else { + _ = c.Store.Put(RetiringState, key, e) + said = append(said, fmt.Sprintf("%s's %s: the operator is asked once more", e.Module, e.Key)) + } + } + case RetireApproved: + said = append(said, c.finishRetirement(&e)...) + } + } + return said +} + +func (c *Config) decideRetirement(e *Retiring, now time.Time) []string { + if c.Asks == nil { + return nil + } + key := retiringKey(e.Module, e.Key) + rec, err := c.Asks.Record(e.Ask) + if err != nil || rec == nil { + return nil + } + var r routerRecord + if json.Unmarshal(rec.Value, &r) != nil || r.State == "open" || r.Warrant == nil { + return nil + } + w := *r.Warrant + keep := func(state, outcome string) []string { + e.State, e.Outcome = state, outcome + if err := c.Store.Put(RetiringState, key, *e); err != nil { + return []string{fmt.Sprintf("%s's %s: %s, and that is not kept: %v", e.Module, e.Key, outcome, err)} + } + return []string{fmt.Sprintf("%s's %s: %s", e.Module, e.Key, outcome)} + } + if w.Asker != Module || w.Ask != e.Ask { + return keep(RetireUnanswered, "the router's record is for another ask; nothing is removed") + } + if w.Outcome != asks.OutcomeChosen { + out := keep(RetireUnanswered, "the ask "+string(w.Outcome)+"; nothing is removed") + if e.AskedAgain { + c.setFinding(retireUnanswered(*e, now)) + } + return out + } + option, err := w.For(Module, e.retireAsk(e.Ask, e.AskedAt)) + if err == nil { + err = option.Performs(e.act(e.Ask, option.ID)) + } + if err != nil { + c.logf("[mesh-config] REFUSED the answer to the retirement ask %s: %v", e.Ask, err) + return keep(RetireUnanswered, "the answer is refused and nothing is removed: "+err.Error()) + } + c.clearFinding(FindingRetireUnanswered, e.Module, e.Key) + e.Answered = w.Says() + switch option.ID { + case OptionDecline: + return keep(RetireDeclined, "declined by the operator ("+w.Says()+"); nothing is removed") + case OptionApprove: + out := keep(RetireApproved, "approved by the operator; being removed") + return append(out, c.finishRetirement(e)...) + } + return keep(RetireUnanswered, "the option "+option.ID+" is not one a retirement offers") +} + +// finishRetirement removes an approved retirement on each node it is not removed on yet: a value here, a secret through +// the vault. Each removal is said (`retired`); the entry goes when every node is done. The caller holds the lock. +func (c *Config) finishRetirement(e *Retiring) []string { + key := retiringKey(e.Module, e.Key) + var said, failed []string + for _, node := range e.Nodes { + if slices.Contains(e.Removed, node) { + continue + } + var err error + switch e.Kind { + case KindValue: + var removed bool + if removed, err = c.removeValue(*e, node); err == nil && !removed { + e.Removed = append(e.Removed, node) // nothing was set there + continue + } + default: + if c.Vault == nil { + err = fmt.Errorf("the seat mesh-vault is not reachable from here") + } else { + err = c.Vault.Retire(RetireRequest{Node: node, Module: e.Module, Name: e.Key, Ask: e.Ask}) + } + } + if err != nil { + failed = append(failed, node+": "+err.Error()) + continue + } + e.Removed = append(e.Removed, node) + if err := c.Store.Emit(EventRetired, RetiredEvent{Module: e.Module, Name: e.Key, Node: node, Ask: e.Ask}); err != nil { + c.logf("[mesh-config] retired for %s's %s on %s is not said: %v", e.Module, e.Key, node, err) + } + said = append(said, fmt.Sprintf("%s's %s is removed on %s", e.Module, e.Key, node)) + } + if len(failed) > 0 { + e.Outcome = "not removed everywhere yet: " + strings.Join(failed, "; ") + _ = c.Store.Put(RetiringState, key, *e) + return append(said, fmt.Sprintf("%s's %s: %s", e.Module, e.Key, e.Outcome)) + } + if err := c.Store.Delete(RetiringState, key); err != nil { + return append(said, fmt.Sprintf("the retirement %s is not dropped: %v", key, err)) + } + if _, err := c.reconcileLocked(); err != nil { + said = append(said, "resolving after the removal: "+err.Error()) + } + return said +} + +// removeValue deletes a module's own value on a node, recording it in the history, the hand-act log and config-changed. +func (c *Config) removeValue(e Retiring, node string) (bool, error) { + vk := moduleValueKey(node, e.Module, e.Key) + got, err := c.Store.Get(ValuesState, vk) + if err != nil { + return false, err + } + if got == nil { + return false, nil // nothing set there, or gone already + } + var was Value + _ = json.Unmarshal(got.Value, &was) + if err := c.Store.Delete(ValuesState, vk); err != nil { + return false, err + } + lvl := Level{Kind: LevelModule, Node: node, Module: e.Module} + ch := Change{Action: "retire", Level: &lvl, Key: e.Key, Was: was.Value, By: "the operator", Via: Module, + Why: "no manifest reads it any more (retiring), removed on the operator's Approve of ask " + e.Ask, At: c.now(), + Approved: e.Answered} + c.record(vk, ch) + if err := c.Store.HandAct(handActWhat(ch), ch.Why, "the operator's Approve of ask "+e.Ask); err != nil { + c.logf("[mesh-config] the hand-act log does not hold the retirement of %s: %v", vk, err) + } + return true, nil +} + +// ---- the tool ----------------------------------------------------------------------------------------------- + +// RetiringRow is one retirement as the tool says it. +type RetiringRow struct { + Module string `json:"module"` + Key string `json:"key"` + Kind string `json:"kind"` + Since time.Time `json:"since"` + State string `json:"state"` + Nodes []string `json:"nodes,omitempty"` + Ask string `json:"ask,omitempty"` + Outcome string `json:"outcome,omitempty"` + Answered string `json:"answered,omitempty"` +} + +// ConfigRetiring lists what is retiring (module narrows), or with action ask-again asks the operator again about one, +// or with cancel drops one (kept apart so it is not asked again until the value is read again). +func (c *Config) ConfigRetiring(action, module, key string) (any, error) { + c.mu.Lock() + defer c.mu.Unlock() + held, err := c.readRetiring() + if err != nil { + return nil, err + } + switch action { + case "", "list": + rows := []RetiringRow{} + for _, k := range sortedKeys(held) { + e := held[k] + if module != "" && e.Module != module { + continue + } + rows = append(rows, RetiringRow{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since, State: e.State, + Nodes: e.Nodes, Ask: e.Ask, Outcome: e.Outcome, Answered: e.Answered}) + } + return map[string]any{"retiring": rows}, nil + case "ask-again", "cancel": + e, ok := held[retiringKey(module, key)] + if !ok { + return nil, fmt.Errorf("%s's %s is not retiring: config_retiring lists what is", module, key) + } + if action == "cancel" { + e.State, e.Outcome = RetireDeclined, "cancelled by hand" + if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil { + return nil, err + } + c.clearFinding(FindingRetireUnanswered, e.Module, e.Key) + if err := c.Store.HandAct("config_retiring cancel "+module+" "+key, "a retirement cancelled by hand", "config_retiring"); err != nil { + c.logf("[mesh-config] the hand-act log does not hold the cancel: %v", err) + } + return map[string]any{"said": module + "'s " + key + " is no longer asked about until it is read again"}, nil + } + if e.State == RetireDraining || e.State == RetireApproved { + return nil, fmt.Errorf("%s's %s is %s: there is nothing to ask again", module, key, e.State) + } + s, err := c.sources() + if err != nil { + return nil, err + } + if nodes := c.nodesReading(s, e); len(nodes) > 0 { + return nil, fmt.Errorf("%s still run a build that reads %s's %s", strings.Join(nodes, ", "), module, key) + } + if err := c.ask(&e, true); err != nil { + return nil, err + } + if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil { + return nil, err + } + return map[string]any{"said": "the operator is asked again", "ask": e.Ask}, nil + } + return nil, fmt.Errorf("config_retiring lists, or does ask-again or cancel; %q is none of them", action) +} diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index e8286bef4..9a0620e0a 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -17,7 +17,7 @@ import ( // from its own manifests as built (issue 1953, ADR 0565 §3); the controller's records are no longer read. func (c *Config) sources() (Sources, error) { s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{}, - Predefined: map[string]Published{}, Values: map[string]Value{}} + Predefined: map[string]Published{}, Values: map[string]Value{}, Renames: map[string]Rename{}} var problems []string if err := c.readPlacement(&s); err != nil { return s, fmt.Errorf("the seat mesh-placement's placement cannot be read: %w", err) @@ -51,6 +51,11 @@ func (c *Config) sources() (Sources, error) { if err := c.readPulls(&s); err != nil { return s, err } + if renames, err := c.readRenames(); err != nil { + return s, fmt.Errorf("the renames cannot be read: %w", err) + } else { + s.Renames = renames + } for _, p := range problems { c.logf("[mesh-config] a placed module's declarations are not read, so its values are not resolved (they "+ "stay as last resolved): %s", p) @@ -354,6 +359,8 @@ type Resolution struct { // did after the take-over. Declared []string `json:"declared,omitempty"` Candidates []string `json:"candidates,omitempty"` + // Renamed is what the renames did in this pass. + Renamed []string `json:"renamed,omitempty"` // SourceGone is said while the stored switch still names the controller, whose records are no longer read. SourceGone string `json:"source_gone,omitempty"` } @@ -364,6 +371,11 @@ type Resolution struct { func (c *Config) Reconcile() (Resolution, error) { c.mu.Lock() defer c.mu.Unlock() + return c.reconcileLocked() +} + +// reconcileLocked is Reconcile with the lock held. +func (c *Config) reconcileLocked() (Resolution, error) { var r Resolution published, err := c.publish() if err != nil { @@ -374,6 +386,8 @@ func (c *Config) Reconcile() (Resolution, error) { if err != nil { return r, err } + // The renames move their values first, so the same pass resolves them under the new names (renames.go). + r.Renamed = c.applyRenames(&s) moved, err := c.writeResolved(s) r.Moved = moved if len(moved) > 0 { diff --git a/modules/mesh-config/cmd/mesh-config/switch_tools.go b/modules/mesh-config/cmd/mesh-config/switch_tools.go index 51475b1f2..f4de6e738 100644 --- a/modules/mesh-config/cmd/mesh-config/switch_tools.go +++ b/modules/mesh-config/cmd/mesh-config/switch_tools.go @@ -11,12 +11,18 @@ import ( // The switch's tools. const ( - DeclaredTool = "config_declared" - ResyncTool = "config_resync" + DeclaredTool = "config_declared" + ResyncTool = "config_resync" + RenamesTool = "config_renames" + RenameAgainTool = "config_rename_again" + RenameCloseTool = "config_rename_close" + RetiringTool = "config_retiring" + RenamesVerb = "renames" ) -// SeatVerbs are the seat's verbs, as the manifest claims them. -var SeatVerbs = []string{Verb} +// SeatVerbs are the seat's verbs, as the manifest claims them: `config`, and the optional `renames` the checker asks +// (design 32: the open renames and the nodes holding each window open). +var SeatVerbs = []string{Verb, RenamesVerb} func switchTools(c *Config) []stdio.Tool { return ownSwitchTools(c) } @@ -35,5 +41,48 @@ func ownSwitchTools(c *Config) []stdio.Tool { Description: "Read every source again and write what differs: the published list of predefined keys, the " + "resolved values, declared, and the indexer's config candidates.", Run: func(map[string]any) (any, error) { return c.Resync() }}, + {Name: RenamesTool, + Description: "Each config value renamed (renamed-from): from which name to which, when, the nodes its value " + + "moved on, the nodes still holding the old name open (the rename window) and how each closed one closed. " + + "Narrowed by module.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + rows, err := c.Renames(strings.TrimSpace(m), false) + return map[string]any{"renames": rows}, err + }}, + {Name: RenameAgainTool, + Description: "Move a renamed value by hand, the way a resolution does: after a refused move (its finding says " + + "why) once the cause is put right, or for a value set under the old name meanwhile.", + Input: map[string]any{"type": "object", "required": []string{"module", "to"}, "properties": map[string]any{ + "module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + to, _ := a["to"].(string) + return c.RenameAgain(strings.TrimSpace(m), strings.TrimSpace(to)) + }}, + {Name: RenameCloseTool, + Description: "Drop a node from a rename's window by hand (a node retired for good), recorded as a hand act: " + + "the old name is no longer answered for it.", + Input: map[string]any{"type": "object", "required": []string{"module", "to", "node", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"}, + "node": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}}, + Run: func(a map[string]any) (any, error) { + str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) } + return c.RenameClose(str("module"), str("to"), str("node"), str("why")) + }}, + {Name: RetiringTool, + Description: "The config values and secrets no manifest reads any more: since when, where each stands (draining " + + "while a node still runs a build that reads it, asked, unanswered, approved, declined) and its ask. With " + + "action ask-again, ask the operator once more about one; with cancel, drop one (it is asked about no more " + + "until it is read again).", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "action": map[string]any{"type": "string", "enum": []string{"list", "ask-again", "cancel"}}, + "module": map[string]any{"type": "string"}, "key": map[string]any{"type": "string"}}}, + Run: func(a map[string]any) (any, error) { + str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) } + return c.ConfigRetiring(str("action"), str("module"), str("key")) + }}, } } diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index d62def76f..ae37f8d2d 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -7,12 +7,16 @@ "name": "mesh-config", "scope": "mesh", "serves": [ - "config" + "config", + "renames" ] } ], "emits": [ - "config-changed" + "config-changed", + "events.renamed", + "events.retiring", + "events.retired" ], "uses": [ "operator-channel" @@ -20,12 +24,19 @@ "consumes": [ "gitea.pull.merged", "gitea.pull.closed", - "gitea.forge.pull-request.closed" + "gitea.forge.pull-request.closed", + "mesh-delivery-view.delivered", + "node-engine.manifest-reads-changed", + "mesh-delivery-indexer.manifest-accepted" ], "invokes": [ "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.warranted", - "gitea.gitea_get_pull_request" + "gitea.gitea_get_pull_request", + "mesh-delivery-view.node_haves", + "seat:node-engine.manifest-reads", + "mesh-delivery-indexer.indexer_config_readers", + "seat:mesh-vault.retire" ], "reads": [ "mesh-delivery-assigner.placement", @@ -38,7 +49,11 @@ ], "tools": [ "config_declared", - "config_resync" + "config_resync", + "config_renames", + "config_rename_again", + "config_rename_close", + "config_retiring" ], "state": [ "values", @@ -47,7 +62,10 @@ "proposals", "pulls", "kept", - "declared" + "declared", + "renames", + "retiring", + "findings" ], "logs": [ "history" @@ -87,5 +105,14 @@ ] } ] - } + }, + "streams": [ + { + "name": "config-events", + "events": [ + "events.**" + ], + "max-age-days": 7 + } + ] }