CAT-C1..C8, C10: mesh-config moves a renamed config value, answers both names in the rename window, retires an unread value on the operator's Approve; the keeper reads its findings

A declaration's renamed-from moves the value with its trust and opens a window per node,
closed on delivered and, for a returning node-engine, on manifest-reads-changed (ADR 0618).
A value no manifest reads is marked retiring only once no node runs a build that reads it,
asked once, removed only on the Approve (a secret through the seat mesh-vault's retire).
New state in the old sections (ADR 0619), the stream config-events, the tools config_renames,
config_rename_again, config_rename_close, config_retiring and the seat verb renames.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
2026-11-04 11:46:16 +01:00
parent 7a7db0166a
commit 376a8b0fb5
19 changed files with 2018 additions and 20 deletions
@@ -93,6 +93,12 @@ var Homes = map[string]Home{
"asks-undelivered": {Source: "asks", Controller: true, Observer: messenger},
// The seat mesh-artifact-sweep (L7): an image a want names is missing from the store, artifact-store-tools' finding.
"image-missing": {Source: "S25", Controller: true, Observer: artifactStore},
// mesh-config's own (design 32): a rename it refused, a rename window or a retirement held seven days by a node, a
// retirement asked twice and not answered.
"rename-refused": {Source: "config", Observer: configModule},
"rename-window-held": {Source: "config", Observer: configModule},
"retirement-held": {Source: "config", Observer: configModule},
"retiring-unanswered": {Source: "config", Observer: configModule},
// The kinds that end with the controller and are carried until its lease write (clauses 4, 9).
"controller-deaf": {Source: "S4", Controller: true, Carried: true},
"lease-lost": {Source: "S12", Controller: true, Carried: true},
@@ -155,6 +161,7 @@ const (
provisioner = "mesh-provisioner"
messenger = "messenger"
artifactStore = "artifact-store-tools"
configModule = "mesh-config"
)
// homeOf is the home of a condition's kind, and whether the table has one.
@@ -40,7 +40,7 @@ const HistoryKeptFor = 90 * 24 * time.Hour
// table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder,
// which observes S25, image-missing).
var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule,
"mesh-provisioner", "messenger", artifactStore}
"mesh-provisioner", "messenger", artifactStore, configModule}
// The other states it reads, as `<module>.<name>`.
const (
@@ -294,7 +294,7 @@ func TestEveryControllerKindHasItsHome(t *testing.T) {
t.Errorf("the ended probe's kind %s is %+v", kind, h)
}
}
if len(Homes) != 84 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled
if len(Homes) != 88 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled, and mesh-config's four (design 32)
t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried)
}
}
+1
View File
@@ -21,6 +21,7 @@
"messenger.findings",
"mesh-provisioner.findings",
"artifact-store-tools.findings",
"mesh-config.findings",
"mesh-healers.attempts",
"mesh-delivery-assigner.wants",
"node-engine.have",
@@ -93,6 +93,10 @@ type Config struct {
Asks AskBus
// Forge is the forge as pulls are asked of it (pulls.go); nil, a merge or a close is heard from its events alone.
Forge PullForge
// Delivery is what the delivery parts answer (renames.go, retiring.go); nil, no window closes and nothing retires.
Delivery Delivery
// Vault is the seat mesh-vault, for the retiring of a secret; nil, a secret does not retire.
Vault Vault
// RouterWithin and RouterEvery bound how long a proposal waits for the router's word on its ask, and how often
// it looks; zero is RouterAnswersWithin and RouterAnswersEvery.
RouterWithin, RouterEvery time.Duration
@@ -20,6 +20,7 @@ type memStore struct {
states map[string]map[string][]byte
logs map[string][]LogEntry
events []Change
said []saidEvent
handActs []string
// failHandAct makes the hand-act log refuse.
failHandAct bool
@@ -89,15 +90,38 @@ func (m *memStore) Read(log string) ([]LogEntry, error) {
}
func (m *memStore) Emit(event string, body any) error {
if event != ConfigChanged {
return fmt.Errorf("mesh-config emits no %s", event)
}
m.mu.Lock()
defer m.mu.Unlock()
m.events = append(m.events, body.(Change))
switch event {
case ConfigChanged:
m.events = append(m.events, body.(Change))
case EventRenamed, EventRetiring, EventRetired:
m.said = append(m.said, saidEvent{event, body})
default:
return fmt.Errorf("mesh-config emits no %s", event)
}
return nil
}
// saidEvent is an event of the family config.events as it was emitted.
type saidEvent struct {
Event string
Body any
}
// saidOf is the events of one name emitted so far.
func (m *memStore) saidOf(event string) []any {
m.mu.Lock()
defer m.mu.Unlock()
var out []any
for _, e := range m.said {
if e.Event == event {
out = append(out, e.Body)
}
}
return out
}
func (m *memStore) HandAct(what, why, cause string) error {
m.mu.Lock()
defer m.mu.Unlock()
@@ -41,6 +41,9 @@ type Declaration struct {
Predefined string `json:"predefined,omitempty"`
Required bool `json:"required,omitempty"`
Runtime bool `json:"runtime,omitempty"`
// RenamedFrom is the key's old name when it was renamed (novox/hq ADR 0609 §4, design 32): on the merge the value
// set under the old name moves to this key (renames.go).
RenamedFrom string `json:"renamed-from,omitempty"`
}
// HasDefault is whether the declaration gives a default: present and not null.
@@ -0,0 +1,133 @@
package main
// What mesh-config asks of the delivery parts and the vault, through their tools and seat verbs on the bus: never a
// bucket of theirs (novox/hq ADR 0596). Each key is the address a module's `invokes` names (mesh-sdk calladdress).
import (
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
// The addresses asked.
const (
// HavesAsk is the status view's read of the node-engines' haves, narrowed to one node and module.
HavesAsk = "mesh-delivery-view.node_haves"
// ManifestReadsAsk is the node-engine seat's verb saying what manifest forms a node-engine reads (ADR 0618 §3), asked
// of one node by the address's `@<node>`.
ManifestReadsAsk = "seat:node-engine.manifest-reads"
// ConfigReadersAsk is the indexer's own tool saying who reads a module's config keys (design 32).
ConfigReadersAsk = "mesh-delivery-indexer.indexer_config_readers"
// VaultRetireAsk is the seat mesh-vault's verb retiring one module's own secret on a node (design 32).
VaultRetireAsk = "seat:mesh-vault.retire"
)
// toolDelivery answers Delivery through the bus.
type toolDelivery struct {
ask func(key string, body any) (json.RawMessage, error)
}
// Have asks node_haves for one node and module and reads the fingerprint the node runs.
func (d toolDelivery) Have(node, module string) (string, bool, error) {
raw, err := d.ask(HavesAsk, map[string]any{"node": node, "module": module})
if err != nil {
return "", false, err
}
answer, output, ok := unwrapAnswer(raw)
if !ok {
return "", false, fmt.Errorf("node_haves refused: %s", output)
}
var kept struct {
Keys map[string]json.RawMessage `json:"keys"`
}
if err := json.Unmarshal(answer, &kept); err != nil {
return "", false, fmt.Errorf("node_haves answered what is not its listing: %s", output)
}
key, err := delivery.HaveKey(node, module)
if err != nil {
return "", false, err
}
have, ok := kept.Keys[key]
if !ok {
return "", false, nil
}
var h delivery.Have
if err := json.Unmarshal(have, &h); err != nil {
return "", false, fmt.Errorf("the have of %s on %s is not readable: %w", module, node, err)
}
return h.FP, true, nil
}
// Reads asks a node's node-engine what it reads; an answer means it states it.
func (d toolDelivery) Reads(node string) error {
_, err := d.ask(ManifestReadsAsk+"@"+node, map[string]any{})
return err
}
// Readers asks the indexer who reads a module's config keys; an indexer that does not serve the tool yet is "not known".
func (d toolDelivery) Readers(module string) ([]ReaderRow, bool, error) {
raw, err := d.ask(ConfigReadersAsk, map[string]any{"module": module})
if err != nil {
if notServed(err) {
return nil, false, nil
}
return nil, false, err
}
answer, output, ok := unwrapAnswer(raw)
if !ok {
if notServed(fmt.Errorf("%s", output)) {
return nil, false, nil
}
return nil, false, fmt.Errorf("indexer_config_readers refused: %s", output)
}
var a struct {
Readers []ReaderRow `json:"readers"`
}
if err := json.Unmarshal(answer, &a); err != nil {
return nil, false, fmt.Errorf("indexer_config_readers answered what is not its listing: %s", output)
}
return a.Readers, true, nil
}
// notServed is whether an error says the address has no holder serving it.
func notServed(err error) bool {
m := strings.ToLower(err.Error())
for _, w := range []string{"no responders", "not served", "unknown tool", "no such tool", "serves no"} {
if strings.Contains(m, w) {
return true
}
}
return false
}
// toolVault asks the seat mesh-vault to retire a secret: the hand act that warrants it is recorded first, by this
// module, naming the node, module and secret; the vault reads it back by its id.
type toolVault struct {
ask func(key string, body any) (json.RawMessage, error)
recorder handacts.Recorder
}
// Retire records the warrant and calls the vault's verb.
func (v toolVault) Retire(r RetireRequest) error {
taken, err := v.recorder.Record(handacts.Request{What: r.Node + " " + r.Module + " " + r.Name,
Why: "the operator approved retiring " + r.Module + "'s secret " + r.Name + " on " + r.Node + " (ask " + r.Ask + ")",
Cause: "the operator's Approve of ask " + r.Ask, Verb: "mesh-vault.retire"})
if err != nil {
return err
}
answer, _, _ := unwrapAnswer(taken.Answer)
var rec struct {
Act struct {
ID string `json:"id"`
} `json:"act"`
}
if json.Unmarshal(answer, &rec) != nil || rec.Act.ID == "" {
return fmt.Errorf("mesh-hand-acts recorded the warrant and named no act")
}
_, err = v.ask(VaultRetireAsk, map[string]any{"node": r.Node, "module": r.Module, "name": r.Name, "act": rec.Act.ID,
"ask": r.Ask})
return err
}
@@ -0,0 +1,138 @@
package main
// mesh-config's findings (novox/hq ADR 0564 clause 2): what it sees wrong about a rename or a retirement, kept in its
// state `findings` as the condition keeper reads them, written when what they say changes and deleted when the fault
// is gone. The keeper raises and clears the condition of the same key (mesh-conditions' Observers and Homes).
import (
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// FindingsState is the state, health.FindingsState.
const FindingsState = health.FindingsState
// The kinds of finding mesh-config writes.
const (
// FindingRenameRefused is a move mesh-config refused; cleared once the value moved or `renamed-from` left the
// trunk's manifest.
FindingRenameRefused = "rename-refused"
// FindingWindowHeld is a rename window held open seven days by a node; cleared when the window closes.
FindingWindowHeld = "rename-window-held"
// FindingRetireUnanswered is a retirement's ask unanswered after it was asked once more; cleared by an answer, a
// cancel, or the value read again.
FindingRetireUnanswered = "retiring-unanswered"
// FindingRetireHeld is a retirement waiting seven days for a node still running a build that reads the value.
FindingRetireHeld = "retirement-held"
)
// findingKey is the key of a finding of kind about a module on a node: the condition's key.
func findingKey(kind, module, node, ref string) string {
id := module
if node != "" {
id += "." + node
}
return health.Key(health.ScopeModule, id, kind+"-"+ref)
}
func finding(kind, module, node, ref, summary, headline, explanation, needs, resolved, said string, at time.Time) health.Finding {
id := module
if node != "" {
id += "." + node
}
return health.Finding{V: health.FindingVersion, Key: findingKey(kind, module, node, ref), Kind: kind, Ref: kind + "-" + ref,
Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node},
Severity: health.Warning, Resolver: health.ResolverOperator, Summary: summary, Headline: headline,
Explanation: explanation, Needs: needs, Resolved: resolved,
Evidence: []health.Evidence{{At: at.UTC(), Said: said}}, Source: "config", Observer: Module, At: at.UTC()}
}
// renameRefused is the finding of a move mesh-config refused.
func renameRefused(module, to, from, why string, at time.Time) health.Finding {
return finding(FindingRenameRefused, module, "", to,
fmt.Sprintf("the rename of %s's config value %s to %s is refused: %s", module, from, to, why),
fmt.Sprintf("%s's value %s is not moved to %s", module, from, to),
fmt.Sprintf("The new version of %s names %s as the new name of its config value %s, and mesh-config refused to move the value: %s. Nothing was moved.", module, to, from, why),
fmt.Sprintf("put the cause right, then move it with config_rename_again (module %s, to %s); the details name it.", module, to),
fmt.Sprintf("Resolved: %s's value %s is under %s", module, from, to), why, at)
}
// windowHeld is the finding of a rename window held open past HeldFinding by a node.
func windowHeld(r Rename, node string, at time.Time) health.Finding {
return finding(FindingWindowHeld, r.Module, node, r.To,
fmt.Sprintf("%s on %s has held the old config name %s open for %d days", r.Module, node, r.From, int(HeldFinding.Hours()/24)),
fmt.Sprintf("%s on %s still runs the old config name", r.Module, node),
fmt.Sprintf("The rename of %s's config value %s to %s is not complete on %s: its node-engine has not reported a build that reads %s (it may be away).", r.Module, r.From, r.To, node, r.To),
fmt.Sprintf("bring %s back, or drop it from the window with config_rename_close.", node),
fmt.Sprintf("Resolved: %s reads %s on %s", r.Module, r.To, node),
fmt.Sprintf("the window opened %s and %s has not reported a build reading %s", r.Nodes[node].Since.UTC().Format(time.RFC3339), node, r.To), at)
}
// setFinding writes a finding when it differs from the one kept.
func (c *Config) setFinding(f health.Finding) {
if _, err := health.EncodeFinding(f); err != nil {
c.logf("[mesh-config] a finding is not valid: %v", err)
return
}
if e, err := c.Store.Get(FindingsState, f.Key); err == nil && e != nil {
if old, derr := health.DecodeFinding(e.Value); derr == nil && old.Summary == f.Summary {
return // what it says has not changed
}
}
if err := c.Store.Put(FindingsState, f.Key, f); err != nil {
c.logf("[mesh-config] the finding %s is not written: %v", f.Key, err)
}
}
// clearFinding deletes the findings of a kind and ref about a module, on whatever node but those in keep.
func (c *Config) clearFinding(kind, module, ref string, keep ...string) {
keys, err := c.Store.Keys(FindingsState)
if err != nil {
return
}
want := kind + "-" + ref
for _, k := range keys {
e, err := c.Store.Get(FindingsState, k)
if err != nil || e == nil {
continue
}
f, err := health.DecodeFinding(e.Value)
if err != nil || f.Ref != want || !(f.Subject.ID == module || strings.HasPrefix(f.Subject.ID, module+".")) ||
(f.Subject.Machine != "" && slices.Contains(keep, f.Subject.Machine)) {
continue
}
if err := c.Store.Delete(FindingsState, k); err != nil {
c.logf("[mesh-config] the finding %s is not cleared: %v", k, err)
}
}
}
// retirementHeld is the finding of a retirement that waits seven days for nodes that still run a build reading the value.
func retirementHeld(e Retiring, at time.Time) health.Finding {
node := ""
if len(e.Nodes) > 0 {
node = e.Nodes[0]
}
return finding(FindingRetireHeld, e.Module, node, e.Key,
fmt.Sprintf("%s's %s has waited %d days to retire: %s still run a build that reads it", e.Module, e.Key, int(HeldFinding.Hours()/24), strings.Join(e.Nodes, ", ")),
fmt.Sprintf("%s's %s cannot be retired yet", e.Module, e.Key),
fmt.Sprintf("No manifest reads %s's %s any more, but %s still run a build that does, so the operator is not asked to remove it.", e.Module, e.Key, strings.Join(e.Nodes, ", ")),
"bring those nodes up to date, or cancel the retirement with config_retiring.",
fmt.Sprintf("Resolved: no node reads %s's %s", e.Module, e.Key),
"retiring since "+e.Since.UTC().Format(time.RFC3339)+", still read on "+strings.Join(e.Nodes, ", "), at)
}
// retireUnanswered is the finding of a retirement asked twice and not answered.
func retireUnanswered(e Retiring, at time.Time) health.Finding {
return finding(FindingRetireUnanswered, e.Module, "", e.Key,
fmt.Sprintf("the operator did not answer the ask to remove %s's %s, asked twice", e.Module, e.Key),
fmt.Sprintf("%s's %s waits for an answer", e.Module, e.Key),
fmt.Sprintf("No manifest reads %s's %s any more. The operator was asked twice whether to remove it and did not answer, so it stays.", e.Module, e.Key),
"answer the ask again with config_retiring ask-again, or cancel it.",
fmt.Sprintf("Resolved: %s's %s is answered or read again", e.Module, e.Key),
"asked "+e.Asked.UTC().Format(time.RFC3339)+" and once more", at)
}
+64 -2
View File
@@ -100,9 +100,11 @@ func (runtimeAsks) Warranted(ask string) error {
}
func main() {
c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}}
c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask},
Delivery: toolDelivery{ask: stdio.Ask}, Vault: toolVault{ask: stdio.Ask, recorder: recorder}}
go resolving(c)
go hearing(c)
go hearingDelivery(c)
if err := stdio.Serve("", Tools(c)); err != nil {
logf("%v", err)
os.Exit(1)
@@ -221,9 +223,69 @@ func hearing(c *Config) {
}
}
// hearingDelivery takes the delivery parts' words that close a rename window or retire a value, each asked again until
// the runtime takes the subscription: the status view's `delivered`, the node-engine seat's `manifest-reads-changed`
// (a returning node-engine; design 32) and the indexer's `manifest-accepted`. At its start it judges the windows once,
// since an event said while this module was down is not said again.
func hearingDelivery(c *Config) {
<-stdio.Served()
for _, h := range []struct {
pattern string
handle func(stdio.Envelope) []string
}{
{"mesh-delivery-view.delivered", func(stdio.Envelope) []string { return c.Windows("") }},
{"node-engine.manifest-reads-changed", func(e stdio.Envelope) []string {
var b struct {
Node string `json:"node"`
}
_ = json.Unmarshal(e.Body, &b)
return c.Windows(b.Node)
}},
{"mesh-delivery-indexer.manifest-accepted", func(e stdio.Envelope) []string {
var b struct {
Module string `json:"module"`
}
if json.Unmarshal(e.Body, &b) != nil {
return nil
}
return c.Retirements(b.Module)
}},
} {
go func(pattern string, handle func(stdio.Envelope) []string) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.Subscribe(pattern, func(e stdio.Envelope) error {
for _, s := range handle(e) {
logf("[mesh-config] %s", s)
}
return nil
})
if err == nil {
return
}
logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait)
time.Sleep(wait)
}
}(h.pattern, h.handle)
}
for _, s := range c.Windows("") {
logf("[mesh-config] %s", s)
}
}
// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them.
func Tools(c *Config) []stdio.Tool {
return append(switchTools(c), stdio.Tool{
renames := stdio.Tool{Name: Seat + "." + RenamesVerb,
Description: "The open renames of config values: for each, the module, the old and the new name, and the nodes " +
"still holding the old name open (the rename window). The checker asks it before it lets a manifest add or " +
"drop renamed-from (novox/hq design 32). Narrowed by module.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "one module"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
rows, err := c.Renames(strings.TrimSpace(m), true)
return map[string]any{"renames": rows}, err
}}
return append(switchTools(c), renames, stdio.Tool{
Name: Seat + "." + Verb,
Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " +
"one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " +
@@ -6,6 +6,8 @@ import (
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself
@@ -35,6 +37,9 @@ func TestEveryStateAndActHasItsTool(t *testing.T) {
PlacementRead: "config list", WantsRead: "config list", BuildsRead: "config list", ModulesRead: DeclaredTool,
ManifestsRead: DeclaredTool, IndexerPredefinedRead: "config list", CandidatesRead: ResyncTool,
"a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool,
// design 32: each new state with the tool that reads it, each act with the tool that starts it.
RenamesState: RenamesTool, RetiringState: RetiringTool, FindingsState: "config_renames and config_retiring",
"a move refused": RenameAgainTool, "a window dropped": RenameCloseTool, "an ask again": RetiringTool,
}
served := map[string]bool{}
for _, tool := range Tools(&Config{}) {
@@ -43,6 +48,21 @@ func TestEveryStateAndActHasItsTool(t *testing.T) {
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") {
t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs)
}
// The stream and its events are declared as the module emits them, and the manifest decodes whole.
if dm, err := catalogue.Decode(raw); err != nil {
t.Errorf("the manifest does not decode: %v", err)
} else {
emits := strings.Join(dm.Emits, ",")
for _, e := range []string{ConfigChanged, EventRenamed, EventRetiring, EventRetired} {
if !strings.Contains(emits, e) {
t.Errorf("the manifest does not declare the event %s", e)
}
}
if len(dm.Streams) != 1 || dm.Streams[0].Name != StreamName || dm.Streams[0].Days() != 7 || !dm.Streams[0].Carries(EventRenamed) ||
!dm.Streams[0].Carries(EventRetiring) || !dm.Streams[0].Carries(EventRetired) || dm.Streams[0].Carries(ConfigChanged) {
t.Errorf("streams %+v: want %s, seven days, carrying renamed, retiring and retired but not config-changed", dm.Streams, StreamName)
}
}
for _, verb := range SeatVerbs {
if !served[Seat+"."+verb] {
t.Errorf("the seat's verb %s is not served", verb)
@@ -759,6 +759,8 @@ func (c *Config) Decide() ([]string, error) {
said = append(said, c.decide(r.p, r.w))
}
c.recordWarrants(now)
// The retirements' asks are taken from the router's record the same way (retiring.go).
said = append(said, c.decideRetirements()...)
return said, nil
}
@@ -0,0 +1,494 @@
package main
// A config value renamed (novox/hq ADR 0609 §4, design 32 *A config value renamed*).
//
// A declaration naming `renamed-from` moves the value set under the old name to the new name, with the value's trust
// (its Value, whole), and opens a **rename window** per node: until that node's node-engine has applied a build of the
// module that reads the new name, the old name still answers with the new value (Resolve). A window closes on the
// status view's `delivered` event and, for a node that was away, on the node-engine seat's `manifest-reads-changed`
// (Windows). Its state is `renames`, key `<module>.<to>`; what it says is `events.renamed` on the stream `config-events`.
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
)
// The state, the stream's events and the bounds this part keeps.
const (
RenamesState = "renames"
// EventRenamed is `renamed`, EventRetiring `retiring` and EventRetired `retired`, the family `config.events`
// (design 26): each carried by the stream StreamName and kept seven days.
EventRenamed = "events.renamed"
EventRetiring = "events.retiring"
EventRetired = "events.retired"
StreamName = "config-events"
// HeldFinding is how long a window or a retirement may stay open before it is a finding naming the node.
HeldFinding = 7 * 24 * time.Hour
)
// Rename is one rename, as `renames` keeps it, key `<module>.<to>`.
type Rename struct {
Module string `json:"module"`
From string `json:"from"`
To string `json:"to"`
At time.Time `json:"at"`
// Moved are the nodes a value was moved on, by the last move of each.
Moved []string `json:"moved,omitempty"`
// Nodes are the windows, by node.
Nodes map[string]Window `json:"nodes"`
// Said says `renamed` was emitted; false until the bus took it.
Said bool `json:"said"`
}
// Window is one node's rename window: open until Closed is set.
type Window struct {
Since time.Time `json:"since"`
Closed *time.Time `json:"closed,omitempty"`
// How is what closed it: "applied" (the node runs a build reading the new name), "unplaced" (the module left the
// node), or "by hand" (config_rename_close).
How string `json:"how,omitempty"`
}
// Open is the nodes whose window is open, sorted.
func (r Rename) Open() []string {
var out []string
for node, w := range r.Nodes {
if w.Closed == nil {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// renameKey is a rename's key in `renames`.
func renameKey(module, to string) string { return module + "." + to }
// RenamedEvent is the body of `renamed`.
type RenamedEvent struct {
Module string `json:"module"`
From string `json:"from"`
To string `json:"to"`
At time.Time `json:"at"`
}
// readRenames is every rename kept, by key.
func (c *Config) readRenames() (map[string]Rename, error) {
out := map[string]Rename{}
err := c.each(RenamesState, func(key string, raw []byte) error {
var r Rename
if json.Unmarshal(raw, &r) != nil || r.Module == "" || renameKey(r.Module, r.To) != key {
return nil // not one this module wrote
}
if r.Nodes == nil {
r.Nodes = map[string]Window{}
}
out[key] = r
return nil
})
return out, err
}
// moduleValueKey is the key of a module's own value on a node.
func moduleValueKey(node, module, key string) string {
return Level{Kind: LevelModule, Node: node, Module: module}.ValueKey(key)
}
// applyRenames moves the values of every rename the trunk's manifests declare, opens the windows of the nodes the
// module is placed on, and writes `renamed` once: run on each resolution, under the lock, before the resolved values
// are written. s.Values and s.Renames are brought to what it did, so the same resolution sees it. A rename it refuses
// is a finding (findings.go), cleared once it moved or `renamed-from` left the manifest. It answers a line for what it
// did or could not do.
func (c *Config) applyRenames(s *Sources) []string {
held, err := c.readRenames()
if err != nil {
return []string{"the renames cannot be read: " + err.Error()}
}
now := c.now()
var said []string
seen := map[string]bool{}
for _, module := range modulesOf(s.Assignments) {
trunk, err := c.trunkDeclarations(module)
if err != nil {
continue // unresolved, said by the resolution
}
for _, to := range sortedKeys(trunk.Config) {
decl := trunk.Config[to]
if decl.RenamedFrom == "" {
continue
}
key := renameKey(module, to)
seen[key] = true
said = append(said, c.rename(s, held, trunk, to, decl, now)...)
}
}
// A rename whose `renamed-from` left the trunk's manifest goes once every window is closed.
for key, r := range held {
if seen[key] {
continue
}
if len(r.Open()) > 0 {
s.Renames[key] = r
continue
}
if err := c.Store.Delete(RenamesState, key); err != nil {
said = append(said, fmt.Sprintf("the rename %s is not removed: %v", key, err))
continue
}
delete(s.Renames, key)
c.clearFinding(FindingRenameRefused, r.Module, r.To)
c.clearFinding(FindingWindowHeld, r.Module, r.To)
}
c.sweepRenameFindings(s, now)
return said
}
// rename judges and applies one declared rename.
func (c *Config) rename(s *Sources, held map[string]Rename, trunk ModuleDeclarations, to string, decl Declaration,
now time.Time) []string {
module, from := trunk.Module, decl.RenamedFrom
key := renameKey(module, to)
nodes := nodesPlaced(s, module)
var said []string
refuse := func(why string) []string {
c.setFinding(renameRefused(module, to, from, why, now))
if r, ok := held[key]; ok {
s.Renames[key] = r
}
return []string{fmt.Sprintf("the rename of %s's %s to %s is refused: %s", module, from, to, why)}
}
if _, both := trunk.Config[from]; both {
return refuse("the manifest declares both names")
}
if from == to {
return refuse("the new name is the old one")
}
trusted := decl.Trusted
if p, ok := s.Predefined[decl.Predefined]; ok && p.Trusted {
trusted = true
}
type move struct {
node string
old, new Value
hadNew bool
oldKey string
newKey string
}
var moves []move
for _, node := range nodes {
oldKey, newKey := moduleValueKey(node, module, from), moduleValueKey(node, module, to)
ov, hadOld := s.Values[oldKey]
if !hadOld {
continue
}
nv, hadNew := s.Values[newKey]
switch {
case hadNew && string(compact(nv.Value)) != string(compact(ov.Value)):
return refuse(fmt.Sprintf("on %s both names hold a value, and they differ (%s under the old name, %s under the new)",
node, ov.Value, nv.Value))
case trusted && ov.Approved == "":
return refuse(fmt.Sprintf("on %s the value under the old name was set without the operator's Approve, and "+
"the new name is trusted (a rename keeps a value's trust, novox/hq ADR 0348 §8)", node))
}
moves = append(moves, move{node: node, old: ov, new: nv, hadNew: hadNew, oldKey: oldKey, newKey: newKey})
}
r, had := held[key]
if !had {
r = Rename{Module: module, From: from, To: to, At: now, Nodes: map[string]Window{}}
}
changed := !had
for _, m := range moves {
if !m.hadNew {
if err := c.Store.Put(ValuesState, m.newKey, m.old); err != nil {
return refuse("the value on " + m.node + " is not written under the new name: " + err.Error())
}
}
if err := c.Store.Delete(ValuesState, m.oldKey); err != nil {
return refuse("the old name's value on " + m.node + " is not removed: " + err.Error())
}
s.Values[m.newKey] = m.old
delete(s.Values, m.oldKey)
lvl := Level{Kind: LevelModule, Node: m.node, Module: module}
_ = c.Store.Append(HistoryLog, m.newKey, Change{Action: "rename", Level: &lvl, Key: to, Value: m.old.Value,
Was: m.old.Value, By: Module, Via: Module, Why: "renamed from " + from + " (renamed-from)", At: now})
if !slices.Contains(r.Moved, m.node) {
r.Moved = append(r.Moved, m.node)
sort.Strings(r.Moved)
}
changed = true
said = append(said, fmt.Sprintf("%s's %s on %s moved to %s", module, from, m.node, to))
}
for _, node := range nodes {
if _, ok := r.Nodes[node]; !ok {
r.Nodes[node] = Window{Since: now}
changed = true
}
}
c.clearFinding(FindingRenameRefused, module, to)
if !r.Said {
if err := c.Store.Emit(EventRenamed, RenamedEvent{Module: module, From: from, To: to, At: r.At}); err != nil {
said = append(said, fmt.Sprintf("renamed for %s's %s is not said yet: %v", module, to, err))
} else {
r.Said, changed = true, true
}
}
if changed {
if err := c.Store.Put(RenamesState, key, r); err != nil {
said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err))
}
}
s.Renames[key] = r
return said
}
// nodesPlaced is the nodes a module is placed on, sorted.
func nodesPlaced(s *Sources, module string) []string {
var out []string
for _, a := range s.Assignments {
if a.Module == module {
out = append(out, a.Node)
}
}
sort.Strings(out)
return slices.Compact(out)
}
// resolveOldNames adds, for each rename with a window open on a node, the old name answering the new name's value on
// that node (design 32: until then mesh-config answers both names).
func resolveOldNames(s Sources, out map[string]Resolved) {
for _, key := range sortedKeys(s.Renames) {
r := s.Renames[key]
for _, node := range r.Open() {
nv, ok := out[ResolvedKey(node, r.Module, r.To)]
if !ok {
continue
}
old := nv
old.Key = r.From
out[ResolvedKey(node, r.Module, r.From)] = old
}
}
}
// ---- the windows --------------------------------------------------------------------------------------------
// Windows closes the rename windows (and the retirements waiting for nodes) that the delivery says are done, at start
// and on each `delivered`; for node, the one a returning node-engine's `manifest-reads-changed` names, it first asks
// the node-engine seat's `manifest-reads`: a node-engine that does not serve it states nothing and its window stays
// open (design 32). It answers a line for each window closed.
func (c *Config) Windows(node string) []string {
c.mu.Lock()
defer c.mu.Unlock()
said := c.closeWindows(node)
said = append(said, c.drainRetirements(node)...)
if len(said) > 0 {
if _, err := c.reconcileLocked(); err != nil {
said = append(said, "resolving after the windows: "+err.Error())
}
}
return said
}
// closeWindows is Windows' rename half; the caller holds the lock.
func (c *Config) closeWindows(only string) []string {
renames, err := c.readRenames()
if err != nil {
return []string{"the renames cannot be read: " + err.Error()}
}
if c.Delivery == nil {
return nil
}
s, err := c.sources()
if err != nil {
return []string{"the windows cannot be judged: " + err.Error()}
}
now := c.now()
var said []string
for _, key := range sortedKeys(renames) {
r := renames[key]
changed := false
for _, node := range r.Open() {
if only != "" && only != node {
continue
}
how, why := c.windowShut(s, r, node, only != "")
if why != "" {
said = append(said, fmt.Sprintf("%s's window for %s stays open on %s: %s", r.Module, r.To, node, why))
}
if how == "" {
continue
}
w := r.Nodes[node]
w.Closed, w.How = &now, how
r.Nodes[node] = w
changed = true
said = append(said, fmt.Sprintf("%s's window for %s is closed on %s (%s)", r.Module, r.To, node, how))
}
if !changed {
continue
}
if err := c.Store.Put(RenamesState, key, r); err != nil {
said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err))
}
if len(r.Open()) == 0 {
c.clearFinding(FindingWindowHeld, r.Module, r.To)
}
}
return said
}
// windowShut says how a node's window may close now ("" when it may not) and, when it may not, why.
func (c *Config) windowShut(s Sources, r Rename, node string, returning bool) (how, why string) {
if !slices.Contains(nodesPlaced(&s, r.Module), node) {
return "unplaced", ""
}
if returning {
if err := c.Delivery.Reads(node); err != nil {
return "", "the node-engine states nothing yet (" + err.Error() + ")"
}
}
fp, found, err := c.Delivery.Have(node, r.Module)
switch {
case err != nil:
return "", "its have cannot be read: " + err.Error()
case !found || fp == "":
return "", "it reports no build of the module"
}
d, err := c.builtDeclarations(r.Module, fp)
if err != nil {
return "", "the build it runs cannot be read: " + err.Error()
}
if _, reads := d.Config[r.To]; !reads {
return "", "it still runs a build that reads " + r.From
}
return "applied", ""
}
// ---- the tools ---------------------------------------------------------------------------------------------
// RenameRow is one rename as the tools say it.
type RenameRow struct {
Module string `json:"module"`
From string `json:"from"`
To string `json:"to"`
At time.Time `json:"at"`
Moved []string `json:"moved,omitempty"`
// Open are the nodes holding the old name open; Closed the others, each with how.
Open []string `json:"open"`
Closed map[string]string `json:"closed,omitempty"`
}
// Renames lists the renames, the open ones only for the seat's verb, narrowed by module.
func (c *Config) Renames(module string, openOnly bool) ([]RenameRow, error) {
renames, err := c.readRenames()
if err != nil {
return nil, err
}
rows := []RenameRow{}
for _, key := range sortedKeys(renames) {
r := renames[key]
if module != "" && r.Module != module {
continue
}
row := RenameRow{Module: r.Module, From: r.From, To: r.To, At: r.At, Moved: r.Moved, Open: r.Open()}
if row.Open == nil {
row.Open = []string{}
}
if openOnly && len(row.Open) == 0 {
continue
}
for node, w := range r.Nodes {
if w.Closed != nil {
if row.Closed == nil {
row.Closed = map[string]string{}
}
row.Closed[node] = w.How
}
}
rows = append(rows, row)
}
return rows, nil
}
// RenameAgain moves a rename's values now, the way a resolution does, and answers what it did: the recovery for a
// refused move once its cause is put right.
func (c *Config) RenameAgain(module, to string) (any, error) {
c.mu.Lock()
defer c.mu.Unlock()
if module == "" || to == "" {
return nil, fmt.Errorf("name the module and the new name of the rename")
}
r, err := c.reconcileLocked()
if err != nil {
return r, err
}
renames, err := c.readRenames()
if err != nil {
return nil, err
}
if _, ok := renames[renameKey(module, to)]; !ok {
return nil, fmt.Errorf("no rename of %s to %s is kept: its trunk manifest declares no renamed-from for %s, or the rename is refused (config_renames and the findings say which)", module, to, to)
}
return map[string]any{"said": r.Renamed, "moved": r.Moved}, nil
}
// RenameClose drops a node from a rename's window by hand, recorded as a hand act.
func (c *Config) RenameClose(module, to, node, why string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("say why the window is dropped: a hand act keeps its why")
}
c.mu.Lock()
defer c.mu.Unlock()
key := renameKey(module, to)
renames, err := c.readRenames()
if err != nil {
return nil, err
}
r, ok := renames[key]
if !ok {
return nil, fmt.Errorf("no rename of %s to %s is kept", module, to)
}
w, ok := r.Nodes[node]
if !ok || w.Closed != nil {
return nil, fmt.Errorf("%s holds no open window for %s's %s; it holds: %s", node, module, to, strings.Join(r.Open(), ", "))
}
now := c.now()
w.Closed, w.How = &now, "by hand"
r.Nodes[node] = w
if err := c.Store.HandAct(fmt.Sprintf("config_rename_close %s %s on %s", module, to, node), why,
"the operator dropped a node from the rename window of "+key); err != nil {
return nil, fmt.Errorf("the hand-act log does not hold it, so the window is not dropped: %w", err)
}
if err := c.Store.Put(RenamesState, key, r); err != nil {
return nil, fmt.Errorf("the window is not dropped: %w", err)
}
if len(r.Open()) == 0 {
c.clearFinding(FindingWindowHeld, r.Module, r.To)
}
if _, err := c.reconcileLocked(); err != nil {
return nil, err
}
return map[string]any{"closed": node, "open": r.Open()}, nil
}
// sweepRenameFindings writes a finding for each window held open past HeldFinding, naming the node, and clears one
// whose window closed.
func (c *Config) sweepRenameFindings(s *Sources, now time.Time) {
for _, key := range sortedKeys(s.Renames) {
r := s.Renames[key]
var held []string
for _, node := range r.Open() {
if now.Sub(r.Nodes[node].Since) >= HeldFinding {
held = append(held, node)
}
}
c.clearFinding(FindingWindowHeld, r.Module, r.To, held...)
for _, node := range held {
c.setFinding(windowHeld(r, node, now))
}
}
}
@@ -0,0 +1,436 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/health"
)
// fakeDelivery is the delivery parts as mesh-config asks them: each node's have, whether a node-engine states what it
// reads, and who reads a module's keys (nil rows and known false: the indexer does not serve the tool yet).
type fakeDelivery struct {
mu sync.Mutex
haves map[string]string // "<node>.<module>" → fingerprint
notSays map[string]bool // node → its node-engine states nothing
rows []ReaderRow
known bool
}
func (d *fakeDelivery) Have(node, module string) (string, bool, error) {
d.mu.Lock()
defer d.mu.Unlock()
fp, ok := d.haves[node+"."+module]
return fp, ok, nil
}
func (d *fakeDelivery) Reads(node string) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.notSays[node] {
return errors.New("the verb manifest-reads is not served")
}
return nil
}
func (d *fakeDelivery) Readers(string) ([]ReaderRow, bool, error) {
d.mu.Lock()
defer d.mu.Unlock()
return d.rows, d.known, nil
}
// fakeVault records the retirements asked of the seat mesh-vault.
type fakeVault struct{ asked []RetireRequest }
func (v *fakeVault) Retire(r RetireRequest) error { v.asked = append(v.asked, r); return nil }
// The fingerprints of the first, second and third build of player.
var (
fp1 = "src2:" + strings.Repeat("1", 64)
fp2 = "src2:" + strings.Repeat("2", 64)
fp3 = "src2:" + strings.Repeat("3", 64)
)
const (
oldPlayer = `{"module":"player","config":{"volume":{"type":"number","default":5}}}`
newPlayer = `{"module":"player","config":{"level":{"type":"number","default":5,"renamed-from":"volume"}}}`
// donePlayer is the manifest once `renamed-from` left it.
donePlayer = `{"module":"player","config":{"level":{"type":"number","default":5}}}`
)
// aRenamingMesh is a mesh with player on g14 and novox, both running its first build (reading `volume`), a clock the
// test moves, and the delivery parts faked.
func aRenamingMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *time.Time) {
t.Helper()
m := newStore()
m.declare(t, "player", oldPlayer)
indexerList(t, m, "aaaaaaaaaaaa", theList)
for _, node := range []string{"g14", "novox"} {
m.assign(t, node, "player")
m.want(t, node, "player", fp1, oldPlayer)
}
clock := time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC)
d := &fakeDelivery{haves: map[string]string{"g14.player": fp1, "novox.player": fp1}, notSays: map[string]bool{}}
c := &Config{Store: m, Now: func() time.Time { return clock }, Logf: t.Logf, Delivery: d}
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
return c, m, d, &clock
}
func findingsOf(t *testing.T, m *memStore, kind string) []health.Finding {
t.Helper()
var out []health.Finding
for _, k := range m.keysOf(FindingsState) {
e, _ := m.Get(FindingsState, k)
f, err := health.DecodeFinding(e.Value)
if err != nil {
t.Fatalf("finding %s: %v", k, err)
}
if f.Kind == kind {
out = append(out, f)
}
}
return out
}
// **A renamed value moves with its trust, both names answer until the node-engines run the new build, and the window
// closes per node** (design 32, C20): the value set under the old name is under the new one with no operator input,
// `renamed` is said once, the old name answers the new value on a node until that node's have runs a build that reads the
// new name (on `delivered`, or for a returning node on `manifest-reads-changed` once its node-engine states what it
// reads), and the rename goes once every window is closed and `renamed-from` left the manifest.
func TestARenamedValueMovesAndTheOldNameAnswersUntilEachNodeRunsTheNewBuild(t *testing.T) {
c, m, d, _ := aRenamingMesh(t)
set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"})
m.declare(t, "player", newPlayer)
r, err := c.Reconcile()
if err != nil || len(r.Renamed) != 1 {
t.Fatalf("reconcile: %+v, %v", r, err)
}
if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" || v.By == "" {
t.Errorf("the value is %+v under the new name (set: %v)", v, ok)
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); ok {
t.Error("the old name still holds a value")
}
for _, c := range []struct{ node, key, want string }{{"g14", "level", "8"}, {"g14", "volume", "8"}, {"novox", "level", "5"}, {"novox", "volume", "5"}} {
if r := m.resolved(t, c.node, "player", c.key); string(r.Value) != c.want {
t.Errorf("%s answers %s on %s as %s, want %s", c.key, "player", c.node, r.Value, c.want)
}
}
if got := m.saidOf(EventRenamed); len(got) != 1 || got[0].(RenamedEvent).From != "volume" || got[0].(RenamedEvent).To != "level" {
t.Errorf("renamed was said as %+v", got)
}
// The history keeps the move; a second pass says and moves nothing more.
if h, _ := c.History(Call{Module: "player", Node: "g14"}); len(h) == 0 || h[len(h)-1].Action != "resolve" && h[len(h)-1].Action != "rename" {
t.Errorf("history %+v", h)
}
if _, err := c.Reconcile(); err != nil || len(m.saidOf(EventRenamed)) != 1 {
t.Errorf("a second pass said renamed %d times: %v", len(m.saidOf(EventRenamed)), err)
}
rows, _ := c.Renames("", true)
if len(rows) != 1 || strings.Join(rows[0].Open, ",") != "g14,novox" || strings.Join(rows[0].Moved, ",") != "g14" {
t.Fatalf("renames %+v", rows)
}
// A delivery while both nodes still run the first build closes nothing.
if said := c.Windows(""); len(said) != 2 || !strings.Contains(said[0], "still runs a build that reads volume") {
t.Errorf("windows said %v", said)
}
// g14 applies the new build.
m.want(t, "g14", "player", fp2, newPlayer)
d.haves["g14.player"] = fp2
if said := c.Windows(""); len(said) == 0 {
t.Error("g14 applied the new build and its window did not close")
}
if _, err := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); err != nil {
t.Fatal(err)
} else if e, _ := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); e != nil {
t.Error("g14 still answers the old name after its window closed")
}
m.resolved(t, "novox", "player", "volume") // novox still answers it
// novox applies it, but its node-engine, returning, states nothing yet: the window stays open.
m.want(t, "novox", "player", fp2, newPlayer)
d.haves["novox.player"] = fp2
d.notSays["novox"] = true
c.Windows("novox")
if rows, _ := c.Renames("", true); len(rows) != 1 || strings.Join(rows[0].Open, ",") != "novox" {
t.Errorf("a node-engine that states nothing closed its window: %+v", rows)
}
d.notSays["novox"] = false
c.Windows("novox")
if rows, _ := c.Renames("", true); len(rows) != 0 {
t.Errorf("windows still open: %+v", rows)
}
// The rename stays while `renamed-from` is in the manifest, and goes when it left.
if all, _ := c.Renames("", false); len(all) != 1 || all[0].Closed["g14"] != "applied" {
t.Errorf("renames %+v", all)
}
m.declare(t, "player", donePlayer)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
if all, _ := c.Renames("", false); len(all) != 0 {
t.Errorf("the rename stayed after renamed-from left the manifest: %+v", all)
}
}
// A refused move is a finding, cleared once it moved (C4): both names declared, both names holding different values, a
// trusted new name over a value set without the operator's Approve. Nothing is moved, nothing is said.
func TestARefusedRenameIsAFindingAndIsClearedOnceItMoves(t *testing.T) {
both := `{"module":"player","config":{"volume":{"type":"number","default":5},"level":{"type":"number","default":5,"renamed-from":"volume"}}}`
trusted := `{"module":"player","config":{"level":{"type":"number","default":5,"trusted":true,"renamed-from":"volume"}}}`
for _, tc := range []struct {
name, manifest string
both bool
why string
fixed string
}{
{"both names declared", both, false, "declares both names", newPlayer},
{"both names hold a value", newPlayer, true, "both names hold a value", newPlayer},
{"a trusted name over an unapproved value", trusted, false, "without the operator's Approve", newPlayer},
} {
t.Run(tc.name, func(t *testing.T) {
c, m, _, _ := aRenamingMesh(t)
set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"})
if tc.both {
// The new name is set meanwhile, to another value, in the state a faulty hand wrote it.
if err := m.Put(ValuesState, "module.g14.player.level", Value{Value: json.RawMessage("9"), By: "x", At: time.Now()}); err != nil {
t.Fatal(err)
}
}
m.declare(t, "player", tc.manifest)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
fs := findingsOf(t, m, FindingRenameRefused)
if len(fs) != 1 || !strings.Contains(fs[0].Summary, tc.why) || fs[0].Observer != Module {
t.Fatalf("findings %+v, want one saying %q", fs, tc.why)
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok || len(m.saidOf(EventRenamed)) != 0 {
t.Errorf("a refused rename moved the value or said renamed")
}
// Put right, it moves by hand and the finding goes.
m.declare(t, "player", tc.fixed)
if tc.both {
_ = m.Delete(ValuesState, "module.g14.player.level")
}
if _, err := c.RenameAgain("player", "level"); err != nil {
t.Fatal(err)
}
if fs := findingsOf(t, m, FindingRenameRefused); len(fs) != 0 {
t.Errorf("the finding stayed after the move: %+v", fs)
}
if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" {
t.Errorf("the value is %+v after the move", v)
}
})
}
}
// A window held open seven days is a finding naming the node, cleared when it closes; dropping a node by hand is a hand
// act with its why (C4, C5), and the seat's verb `renames` names the nodes holding a window open.
func TestAWindowHeldSevenDaysIsAFindingAndAHandCloseIsAHandAct(t *testing.T) {
c, m, d, clock := aRenamingMesh(t)
m.declare(t, "player", newPlayer)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
m.want(t, "g14", "player", fp2, newPlayer)
d.haves["g14.player"] = fp2
c.Windows("")
*clock = clock.Add(8 * 24 * time.Hour)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
fs := findingsOf(t, m, FindingWindowHeld)
if len(fs) != 1 || fs[0].Subject.Machine != "novox" || !strings.Contains(fs[0].Summary, "novox") {
t.Fatalf("findings %+v, want one naming novox", fs)
}
// The seat's verb names the node holding the window open.
var verb func(map[string]any) (any, error)
for _, tool := range Tools(c) {
if tool.Name == Seat+"."+RenamesVerb {
verb = tool.Run
}
}
got, err := verb(map[string]any{"module": "player"})
if raw, _ := json.Marshal(got); err != nil || !strings.Contains(string(raw), `"open":["novox"]`) {
t.Errorf("renames answered %s, %v", raw, err)
}
if _, err := c.RenameClose("player", "level", "novox", ""); err == nil {
t.Error("a window was dropped with no why")
}
if _, err := c.RenameClose("player", "level", "g14", "g14 is closed already"); err == nil {
t.Error("a node holding no open window was dropped")
}
m.handActs = nil
if _, err := c.RenameClose("player", "level", "novox", "novox is retired for good"); err != nil {
t.Fatal(err)
}
if len(m.handActs) != 1 || !strings.Contains(m.handActs[0], "novox is retired for good") {
t.Errorf("hand acts %v", m.handActs)
}
if fs := findingsOf(t, m, FindingWindowHeld); len(fs) != 0 {
t.Errorf("the finding stayed after the window closed: %+v", fs)
}
if rows, _ := c.Renames("", true); len(rows) != 0 {
t.Errorf("open renames %+v", rows)
}
}
// retiringMesh is a mesh where player no longer declares `volume`, a value of it is set on g14, and g14 runs the first
// build (which reads it).
func retiringMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *standIn, *fakeVault, *time.Time) {
t.Helper()
c, m, d, clock := aRenamingMesh(t)
set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"})
m.declare(t, "player", donePlayer)
r := newStandIn()
v := &fakeVault{}
c.Asks, c.Vault = r, v
d.known = true
d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{}}}
return c, m, d, r, v, clock
}
func retiringOf(t *testing.T, c *Config) []RetiringRow {
t.Helper()
got, err := c.ConfigRetiring("", "", "")
if err != nil {
t.Fatal(err)
}
return got.(map[string]any)["retiring"].([]RetiringRow)
}
// **A value no manifest reads is retired only on the operator's Approve** (C24): while a node runs a build that reads it
// it is not marked and nobody is asked; once none does, `retiring` is said and the operator is asked once; the value stays
// until the Approve, then it is removed, `retired` is said for the node it was set on, and the warrant is recorded.
func TestAValueNoManifestReadsIsRetiredOnlyOnTheOperatorsApprove(t *testing.T) {
c, m, d, r, _, _ := retiringMesh(t)
// The indexer does not serve the tool yet: nothing retires on that.
d.known = false
if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "does not say") || len(retiringOf(t, c)) != 0 {
t.Errorf("an indexer without the tool: %v", said)
}
d.known = true
// g14 still runs a build that reads volume: draining, no ask, no event.
c.Retirements("player")
if rows := retiringOf(t, c); len(rows) != 1 || rows[0].State != RetireDraining || strings.Join(rows[0].Nodes, ",") != "g14,novox" {
t.Fatalf("retiring %+v, want draining on g14 and novox", rows)
}
if len(r.asks) != 0 || len(m.saidOf(EventRetiring)) != 0 {
t.Fatalf("asked %d times and said retiring %d times while two nodes read the value", len(r.asks), len(m.saidOf(EventRetiring)))
}
// Both nodes apply the build that reads it no longer.
for _, node := range []string{"g14", "novox"} {
m.want(t, node, "player", fp3, donePlayer)
d.haves[node+".player"] = fp3
}
c.Windows("")
rows := retiringOf(t, c)
if len(rows) != 1 || rows[0].State != RetireAsked || len(r.asks) != 1 || len(m.saidOf(EventRetiring)) != 1 {
t.Fatalf("retiring %+v, %d asks, %d events", rows, len(r.asks), len(m.saidOf(EventRetiring)))
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok {
t.Fatal("the value was removed before the operator answered")
}
// A second manifest-accepted asks nobody again.
c.Retirements("player")
if len(r.asks) != 1 {
t.Errorf("asked %d times", len(r.asks))
}
r.choose(t, rows[0].Ask, OptionApprove, nil)
m.handActs = nil
if said, err := c.Decide(); err != nil || len(said) == 0 {
t.Fatalf("decide: %v, %v", said, err)
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); ok {
t.Error("the approved value is still set")
}
if got := m.saidOf(EventRetired); len(got) != 1 || got[0].(RetiredEvent).Node != "g14" || got[0].(RetiredEvent).Name != "volume" {
t.Errorf("retired was said as %+v", got)
}
if len(retiringOf(t, c)) != 0 || len(m.handActs) == 0 {
t.Errorf("retiring %+v, hand acts %v", retiringOf(t, c), m.handActs)
}
}
// A declined retirement is not asked again until the value is read again; a value read again is dropped; an ask unanswered
// is asked once more after seven days and then written as a finding, cleared by an answer or a cancel; a secret is
// retired through the seat mesh-vault with the ask that warranted it (C24, C7).
func TestARetirementDeclinedUnansweredCancelledOrASecret(t *testing.T) {
c, m, d, r, v, clock := retiringMesh(t)
for _, node := range []string{"g14", "novox"} {
m.want(t, node, "player", fp3, donePlayer)
d.haves[node+".player"] = fp3
}
c.Retirements("player")
ask := retiringOf(t, c)[0].Ask
// Unanswered: the ask expires, a week passes, it is asked once more, expires again: a finding.
r.choose(t, ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired })
c.Decide()
if rows := retiringOf(t, c); rows[0].State != RetireUnanswered || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 {
t.Fatalf("after the first expiry: %+v", rows)
}
*clock = clock.Add(8 * 24 * time.Hour)
c.Decide()
rows := retiringOf(t, c)
if rows[0].State != RetireAsked || len(r.asks) != 2 || rows[0].Ask == ask {
t.Fatalf("after a week: %+v, %d asks", rows, len(r.asks))
}
r.choose(t, rows[0].Ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired })
c.Decide()
if fs := findingsOf(t, m, FindingRetireUnanswered); len(fs) != 1 {
t.Fatalf("findings %+v after two asks unanswered", fs)
}
// Asked again by hand, then declined: the finding goes and it is not asked again.
if _, err := c.ConfigRetiring("ask-again", "player", "volume"); err != nil {
t.Fatal(err)
}
third := retiringOf(t, c)[0].Ask
r.choose(t, third, OptionDecline, nil)
c.Decide()
if rows := retiringOf(t, c); rows[0].State != RetireDeclined || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 {
t.Fatalf("after the decline: %+v", rows)
}
asked := len(r.asks)
c.Retirements("player")
if len(r.asks) != asked {
t.Error("a declined retirement was asked again")
}
// Read again: dropped.
d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{"player"}}}
if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "read again") || len(retiringOf(t, c)) != 0 {
t.Errorf("read again: %v", said)
}
// A secret: its nodes come from the indexer's answer; on the Approve the vault is asked for each, once.
d.rows = []ReaderRow{{Module: "player", Key: "token", Kind: KindSecret, Readers: []string{}, Nodes: []string{}}}
c.Retirements("player")
secret := retiringOf(t, c)[0]
if secret.Kind != KindSecret || secret.State != RetireAsked {
t.Fatalf("the secret: %+v", secret)
}
r.choose(t, secret.Ask, OptionApprove, nil)
c.Decide()
if len(v.asked) != 2 || v.asked[0].Node != "g14" || v.asked[0].Name != "token" || v.asked[0].Ask != secret.Ask {
t.Errorf("the vault was asked %+v", v.asked)
}
if len(m.saidOf(EventRetired)) != 2 || len(retiringOf(t, c)) != 0 {
t.Errorf("retired %d times, retiring %+v", len(m.saidOf(EventRetired)), retiringOf(t, c))
}
// A secret whose readers' nodes the indexer cannot name is not marked.
d.rows = []ReaderRow{{Module: "player", Key: "other", Kind: KindSecret, Readers: []string{}}}
if c.Retirements("player"); len(retiringOf(t, c)) != 0 {
t.Errorf("a secret read on unknown nodes was marked: %+v", retiringOf(t, c))
}
_ = fmt.Sprint
}
@@ -150,6 +150,8 @@ type Sources struct {
// for them (pulls.go); neither is resolved.
Pulls map[string]PullDecl
Kept map[string]KeptValue
// Renames are the renames kept, by `<module>.<to>`, whose open windows answer the old name (renames.go).
Renames map[string]Rename
}
// Unresolved is a module placed on a node whose values are not resolved, and why.
@@ -192,6 +194,7 @@ func Resolve(s Sources) map[string]Resolved {
out[ResolvedKey(a.Node, a.Module, key)] = r
}
}
resolveOldNames(s, out)
return out
}
@@ -0,0 +1,581 @@
package main
// A value no longer read (novox/hq ADR 0609, design 32 *A value no longer read*).
//
// A config value or secret that no manifest on any trunk reads any more is a **retiring value**. mesh-config learns it
// from the indexer's tool `indexer_config_readers`, asked on each of the indexer's `manifest-accepted` (Retirements);
// a tool the indexer does not serve yet answers "not known" and nothing retires. A value is marked retiring only once no
// node-engine reports applied a build that reads it (State draining until then, judged again on each `delivered`);
// mesh-config then says `retiring` and asks the operator once, through the operator channel, whether to remove it. It is
// removed only on the operator's Approve: a value by mesh-config, a secret through the seat `mesh-vault`'s verb
// `retire`. An ask unanswered after seven days is asked once more, and then written as a finding.
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// The state, and where a retirement stands.
const (
RetiringState = "retiring"
// RetireVerb is the act an answer binds.
RetireVerb = Seat + ".retire"
RetireDraining = "draining" // a node still runs a build that reads it
RetireAsked = "asked" // the operator is asked
RetireUnanswered = "unanswered" // the ask ended with no choice
RetireApproved = "approved" // the operator approved; being removed
RetireDeclined = "declined" // the operator declined, or a person cancelled: kept until the value is read again
)
// Kinds of retiring value.
const (
KindValue = "value"
KindSecret = "secret"
)
// Retiring is one retiring value, as `retiring` keeps it, key `<module>.<key>`.
type Retiring struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
State string `json:"state"`
// Nodes are the nodes that still read it (draining) or that it is removed on (asked, approved).
Nodes []string `json:"nodes,omitempty"`
// Ask is the open ask's id, Asked when the first ask was made, AskedAgain whether it was asked once more.
Ask string `json:"ask,omitempty"`
// AskedAt is when the open ask was made (it expires a day on), Asked when the first was.
AskedAt time.Time `json:"asked-at,omitempty"`
Asked time.Time `json:"asked,omitempty"`
AskedAgain bool `json:"asked-again,omitempty"`
Said bool `json:"said,omitempty"`
Outcome string `json:"outcome,omitempty"`
Answered string `json:"answered,omitempty"`
// Removed are the nodes it is removed on already.
Removed []string `json:"removed,omitempty"`
}
func retiringKey(module, key string) string { return module + "." + key }
// RetiringEvent is the body of `retiring`; RetiredEvent of `retired`.
type RetiringEvent struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
Ask string `json:"ask"`
}
// RetiredEvent is the body of `retired`: one node's value removed, with the ask that warranted it.
type RetiredEvent struct {
Module string `json:"module"`
Name string `json:"name"`
Node string `json:"node"`
Ask string `json:"ask"`
}
// ReaderRow is one key the indexer's `indexer_config_readers` answers: the modules on each trunk that declare or refer
// to it. For a secret, Nodes are the nodes whose applied build reads it, when the indexer can say.
type ReaderRow struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Readers []string `json:"readers"`
Nodes []string `json:"nodes,omitempty"`
}
// Delivery is what the delivery parts answer to this module: a node's have, a node-engine's statement of what it
// reads, and who reads a module's config keys. Each is a tool or a seat verb on the bus, never a bucket (ADR 0596).
type Delivery interface {
// Have is the fingerprint of the build node runs of module, as the status view's node_haves says it.
Have(node, module string) (fp string, found bool, err error)
// Reads asks the node-engine seat's manifest-reads for node: an error when it is not served or the node says nothing.
Reads(node string) error
// Readers asks the indexer who reads module's keys: known false when the indexer does not serve the tool yet.
Readers(module string) (rows []ReaderRow, known bool, err error)
}
// RetireRequest is what the vault's verb `retire` takes: one module's own secret on a node, with the hand act that
// warrants it.
type RetireRequest struct {
Node, Module, Name, Ask string
}
// Vault is the seat mesh-vault as this module reaches it.
type Vault interface {
Retire(r RetireRequest) error
}
func (c *Config) readRetiring() (map[string]Retiring, error) {
out := map[string]Retiring{}
err := c.each(RetiringState, func(key string, raw []byte) error {
var r Retiring
if json.Unmarshal(raw, &r) != nil || r.Module == "" || retiringKey(r.Module, r.Key) != key {
return nil
}
out[key] = r
return nil
})
return out, err
}
// Retirements judges a module's keys on the indexer's word that its manifest was accepted. It answers a line for each
// decision.
func (c *Config) Retirements(module string) []string {
c.mu.Lock()
defer c.mu.Unlock()
if c.Delivery == nil || module == "" {
return nil
}
rows, known, err := c.Delivery.Readers(module)
if err != nil {
return []string{fmt.Sprintf("who reads %s's config is not asked yet: %v", module, err)}
}
if !known {
return []string{fmt.Sprintf("the indexer does not say who reads %s's config yet; nothing retires on that", module)}
}
held, err := c.readRetiring()
if err != nil {
return []string{"the retirements cannot be read: " + err.Error()}
}
s, err := c.sources()
if err != nil {
return []string{"the retirements cannot be judged: " + err.Error()}
}
var said []string
for _, row := range rows {
if row.Module != "" && row.Module != module {
continue
}
row.Module = module
if row.Kind == "" {
row.Kind = KindValue
}
key := retiringKey(module, row.Key)
e, has := held[key]
if len(row.Readers) > 0 {
if has {
said = append(said, c.dropRetiring(e, "it is read again"))
}
continue
}
if has {
continue // already marked; it is advanced by drainRetirements and Decide
}
if row.Kind == KindValue && !valueHeld(s, module, row.Key) {
continue // nothing set under that name: nothing to remove
}
if row.Kind == KindSecret && row.Nodes == nil && !row.nodesKnown() {
continue // which nodes read a secret is not known, and nothing retires on that
}
e = Retiring{Module: module, Key: row.Key, Kind: row.Kind, Since: c.now(), State: RetireDraining, Nodes: row.Nodes}
said = append(said, c.advanceRetiring(s, &e)...)
}
return said
}
func (r ReaderRow) nodesKnown() bool { return r.Nodes != nil }
// valueHeld is whether a module's own value is set under key on any node.
func valueHeld(s Sources, module, key string) bool {
for k := range s.Values {
if l, kk, ok := parseValueKey(k); ok && l.Kind == LevelModule && l.Module == module && kk == key {
return true
}
}
return false
}
// nodesReading is the nodes whose applied build still reads a value (secrets: the nodes the indexer last named).
func (c *Config) nodesReading(s Sources, e Retiring) []string {
if e.Kind != KindValue || c.Delivery == nil {
return e.Nodes
}
var out []string
for _, node := range nodesPlaced(&s, e.Module) {
fp, found, err := c.Delivery.Have(node, e.Module)
if err != nil || !found || fp == "" {
if err != nil {
out = append(out, node) // not known is not "no longer read"
}
continue
}
d, err := c.builtDeclarations(e.Module, fp)
if err != nil {
out = append(out, node)
continue
}
if _, reads := d.Config[e.Key]; reads {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// advanceRetiring moves a draining retirement on: while a node reads the value it stays draining (and is written, so a
// delivery judges it again); once none does it is marked retiring, said, and asked. The caller holds the lock.
func (c *Config) advanceRetiring(s Sources, e *Retiring) []string {
key := retiringKey(e.Module, e.Key)
now := c.now()
if e.State == RetireDraining {
e.Nodes = c.nodesReading(s, *e)
if len(e.Nodes) > 0 {
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)}
}
if now.Sub(e.Since) >= HeldFinding {
c.setFinding(retirementHeld(*e, now))
}
return []string{fmt.Sprintf("%s's %s is not read by any manifest, and %s still run a build that reads it",
e.Module, e.Key, strings.Join(e.Nodes, ", "))}
}
c.clearFinding(FindingRetireHeld, e.Module, e.Key)
e.Nodes = nodesPlaced(&s, e.Module)
if err := c.ask(e, false); err != nil {
return []string{fmt.Sprintf("the operator is not asked about retiring %s's %s: %v", e.Module, e.Key, err)}
}
}
if !e.Said {
if err := c.Store.Emit(EventRetiring, RetiringEvent{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since,
Ask: e.Ask}); err == nil {
e.Said = true
}
}
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)}
}
return []string{fmt.Sprintf("%s's %s is retiring: the operator is asked whether to remove it", e.Module, e.Key)}
}
// drainRetirements judges the draining retirements again, after a delivery or a returning node; the caller holds the lock.
func (c *Config) drainRetirements(string) []string {
held, err := c.readRetiring()
if err != nil || len(held) == 0 || c.Delivery == nil {
return nil
}
s, err := c.sources()
if err != nil {
return []string{"the retirements cannot be judged: " + err.Error()}
}
var said []string
for _, key := range sortedKeys(held) {
e := held[key]
if e.State == RetireDraining {
said = append(said, c.advanceRetiring(s, &e)...)
}
}
return said
}
// dropRetiring ends a retirement: the value is read again, or a person cancelled it.
func (c *Config) dropRetiring(e Retiring, why string) string {
if err := c.Store.Delete(RetiringState, retiringKey(e.Module, e.Key)); err != nil {
return fmt.Sprintf("the retirement of %s's %s is not dropped: %v", e.Module, e.Key, err)
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
c.clearFinding(FindingRetireHeld, e.Module, e.Key)
return fmt.Sprintf("%s's %s is no longer retiring: %s", e.Module, e.Key, why)
}
// ---- the ask ------------------------------------------------------------------------------------------------
// retireAskID is an ask's id: the same retirement asked again is another id.
func retireAskID(e Retiring, again bool) string {
sum := sha256.Sum256([]byte(e.Module + "\x00" + e.Key + "\x00" + e.Since.UTC().Format(time.RFC3339Nano)))
id := "ret-" + hex.EncodeToString(sum[:6])
if again {
id += "-2"
}
return id
}
// act is what an answer binds: the retirement, the nodes it reaches, the ask and the answer.
func (e Retiring) act(ask, option string) asks.Act {
return asks.Act{"verb": RetireVerb, "option": option, "module": e.Module, "key": e.Key, "kind": e.Kind,
"nodes": strings.Join(e.Nodes, " "), "ask": ask}
}
// retireAsk is the ask composed from the retirement alone.
func (e Retiring) retireAsk(id string, at time.Time) asks.Ask {
nodes := strings.Join(e.Nodes, ", ")
if nodes == "" {
nodes = "no node"
}
what := "config value"
if e.Kind == KindSecret {
what = "secret"
}
a := asks.Ask{ID: id, Headline: fmt.Sprintf("Remove %s's %s %s?", e.Module, what, e.Key),
Explanation: fmt.Sprintf("No manifest reads %s's %s %s any more, and no node runs a build that reads it. Approved, it is removed on %s.", e.Module, what, e.Key, nodes),
Who: asks.Operator, Expires: at.Add(ProposalLasts), OnExpiry: "nothing is removed; mesh-config asks once more in a week",
About: "config.retire." + e.Module + "." + e.Key,
Details: fmt.Sprintf("Retiring since %s UTC. Read it with mesh-config's config_retiring.", e.Since.UTC().Format("15:04 on 2 Jan"))}
for _, o := range []struct{ id, label, does string }{
{OptionApprove, "Approve", "the " + what + " is removed on " + nodes},
{OptionDecline, "Decline", "nothing is removed; mesh-config asks no more until it is read again"},
} {
binds, _ := asks.ActDigest(e.act(id, o.id))
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
}
return a
}
// ask makes the ask for a retirement and publishes it; the caller keeps the entry.
func (c *Config) ask(e *Retiring, again bool) error {
if c.Asks == nil {
return fmt.Errorf("the operator cannot be asked from here")
}
now := c.now()
id := retireAskID(*e, again)
a := e.retireAsk(id, now)
if err := a.Check(now); err != nil {
return err
}
if err := c.Asks.Publish(asks.AskSubject(Module), a, a.ID); err != nil {
return err
}
e.Ask, e.State, e.AskedAt = id, RetireAsked, now
if e.Asked.IsZero() {
e.Asked = now
}
e.AskedAgain = e.AskedAgain || again
return nil
}
// decideRetirements takes the router's word on each open retirement ask; the caller holds the lock. It also asks once
// more an ask unanswered for seven days, writes the finding of one asked twice, and finishes an approved removal.
func (c *Config) decideRetirements() []string {
held, err := c.readRetiring()
if err != nil || len(held) == 0 {
return nil
}
now := c.now()
var said []string
for _, key := range sortedKeys(held) {
e := held[key]
switch e.State {
case RetireAsked:
said = append(said, c.decideRetirement(&e, now)...)
case RetireUnanswered:
if !e.AskedAgain && now.Sub(e.Asked) >= HeldFinding {
if err := c.ask(&e, true); err != nil {
said = append(said, fmt.Sprintf("the operator is not asked again about %s's %s: %v", e.Module, e.Key, err))
} else {
_ = c.Store.Put(RetiringState, key, e)
said = append(said, fmt.Sprintf("%s's %s: the operator is asked once more", e.Module, e.Key))
}
}
case RetireApproved:
said = append(said, c.finishRetirement(&e)...)
}
}
return said
}
func (c *Config) decideRetirement(e *Retiring, now time.Time) []string {
if c.Asks == nil {
return nil
}
key := retiringKey(e.Module, e.Key)
rec, err := c.Asks.Record(e.Ask)
if err != nil || rec == nil {
return nil
}
var r routerRecord
if json.Unmarshal(rec.Value, &r) != nil || r.State == "open" || r.Warrant == nil {
return nil
}
w := *r.Warrant
keep := func(state, outcome string) []string {
e.State, e.Outcome = state, outcome
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("%s's %s: %s, and that is not kept: %v", e.Module, e.Key, outcome, err)}
}
return []string{fmt.Sprintf("%s's %s: %s", e.Module, e.Key, outcome)}
}
if w.Asker != Module || w.Ask != e.Ask {
return keep(RetireUnanswered, "the router's record is for another ask; nothing is removed")
}
if w.Outcome != asks.OutcomeChosen {
out := keep(RetireUnanswered, "the ask "+string(w.Outcome)+"; nothing is removed")
if e.AskedAgain {
c.setFinding(retireUnanswered(*e, now))
}
return out
}
option, err := w.For(Module, e.retireAsk(e.Ask, e.AskedAt))
if err == nil {
err = option.Performs(e.act(e.Ask, option.ID))
}
if err != nil {
c.logf("[mesh-config] REFUSED the answer to the retirement ask %s: %v", e.Ask, err)
return keep(RetireUnanswered, "the answer is refused and nothing is removed: "+err.Error())
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
e.Answered = w.Says()
switch option.ID {
case OptionDecline:
return keep(RetireDeclined, "declined by the operator ("+w.Says()+"); nothing is removed")
case OptionApprove:
out := keep(RetireApproved, "approved by the operator; being removed")
return append(out, c.finishRetirement(e)...)
}
return keep(RetireUnanswered, "the option "+option.ID+" is not one a retirement offers")
}
// finishRetirement removes an approved retirement on each node it is not removed on yet: a value here, a secret through
// the vault. Each removal is said (`retired`); the entry goes when every node is done. The caller holds the lock.
func (c *Config) finishRetirement(e *Retiring) []string {
key := retiringKey(e.Module, e.Key)
var said, failed []string
for _, node := range e.Nodes {
if slices.Contains(e.Removed, node) {
continue
}
var err error
switch e.Kind {
case KindValue:
var removed bool
if removed, err = c.removeValue(*e, node); err == nil && !removed {
e.Removed = append(e.Removed, node) // nothing was set there
continue
}
default:
if c.Vault == nil {
err = fmt.Errorf("the seat mesh-vault is not reachable from here")
} else {
err = c.Vault.Retire(RetireRequest{Node: node, Module: e.Module, Name: e.Key, Ask: e.Ask})
}
}
if err != nil {
failed = append(failed, node+": "+err.Error())
continue
}
e.Removed = append(e.Removed, node)
if err := c.Store.Emit(EventRetired, RetiredEvent{Module: e.Module, Name: e.Key, Node: node, Ask: e.Ask}); err != nil {
c.logf("[mesh-config] retired for %s's %s on %s is not said: %v", e.Module, e.Key, node, err)
}
said = append(said, fmt.Sprintf("%s's %s is removed on %s", e.Module, e.Key, node))
}
if len(failed) > 0 {
e.Outcome = "not removed everywhere yet: " + strings.Join(failed, "; ")
_ = c.Store.Put(RetiringState, key, *e)
return append(said, fmt.Sprintf("%s's %s: %s", e.Module, e.Key, e.Outcome))
}
if err := c.Store.Delete(RetiringState, key); err != nil {
return append(said, fmt.Sprintf("the retirement %s is not dropped: %v", key, err))
}
if _, err := c.reconcileLocked(); err != nil {
said = append(said, "resolving after the removal: "+err.Error())
}
return said
}
// removeValue deletes a module's own value on a node, recording it in the history, the hand-act log and config-changed.
func (c *Config) removeValue(e Retiring, node string) (bool, error) {
vk := moduleValueKey(node, e.Module, e.Key)
got, err := c.Store.Get(ValuesState, vk)
if err != nil {
return false, err
}
if got == nil {
return false, nil // nothing set there, or gone already
}
var was Value
_ = json.Unmarshal(got.Value, &was)
if err := c.Store.Delete(ValuesState, vk); err != nil {
return false, err
}
lvl := Level{Kind: LevelModule, Node: node, Module: e.Module}
ch := Change{Action: "retire", Level: &lvl, Key: e.Key, Was: was.Value, By: "the operator", Via: Module,
Why: "no manifest reads it any more (retiring), removed on the operator's Approve of ask " + e.Ask, At: c.now(),
Approved: e.Answered}
c.record(vk, ch)
if err := c.Store.HandAct(handActWhat(ch), ch.Why, "the operator's Approve of ask "+e.Ask); err != nil {
c.logf("[mesh-config] the hand-act log does not hold the retirement of %s: %v", vk, err)
}
return true, nil
}
// ---- the tool -----------------------------------------------------------------------------------------------
// RetiringRow is one retirement as the tool says it.
type RetiringRow struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
State string `json:"state"`
Nodes []string `json:"nodes,omitempty"`
Ask string `json:"ask,omitempty"`
Outcome string `json:"outcome,omitempty"`
Answered string `json:"answered,omitempty"`
}
// ConfigRetiring lists what is retiring (module narrows), or with action ask-again asks the operator again about one,
// or with cancel drops one (kept apart so it is not asked again until the value is read again).
func (c *Config) ConfigRetiring(action, module, key string) (any, error) {
c.mu.Lock()
defer c.mu.Unlock()
held, err := c.readRetiring()
if err != nil {
return nil, err
}
switch action {
case "", "list":
rows := []RetiringRow{}
for _, k := range sortedKeys(held) {
e := held[k]
if module != "" && e.Module != module {
continue
}
rows = append(rows, RetiringRow{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since, State: e.State,
Nodes: e.Nodes, Ask: e.Ask, Outcome: e.Outcome, Answered: e.Answered})
}
return map[string]any{"retiring": rows}, nil
case "ask-again", "cancel":
e, ok := held[retiringKey(module, key)]
if !ok {
return nil, fmt.Errorf("%s's %s is not retiring: config_retiring lists what is", module, key)
}
if action == "cancel" {
e.State, e.Outcome = RetireDeclined, "cancelled by hand"
if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil {
return nil, err
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
if err := c.Store.HandAct("config_retiring cancel "+module+" "+key, "a retirement cancelled by hand", "config_retiring"); err != nil {
c.logf("[mesh-config] the hand-act log does not hold the cancel: %v", err)
}
return map[string]any{"said": module + "'s " + key + " is no longer asked about until it is read again"}, nil
}
if e.State == RetireDraining || e.State == RetireApproved {
return nil, fmt.Errorf("%s's %s is %s: there is nothing to ask again", module, key, e.State)
}
s, err := c.sources()
if err != nil {
return nil, err
}
if nodes := c.nodesReading(s, e); len(nodes) > 0 {
return nil, fmt.Errorf("%s still run a build that reads %s's %s", strings.Join(nodes, ", "), module, key)
}
if err := c.ask(&e, true); err != nil {
return nil, err
}
if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil {
return nil, err
}
return map[string]any{"said": "the operator is asked again", "ask": e.Ask}, nil
}
return nil, fmt.Errorf("config_retiring lists, or does ask-again or cancel; %q is none of them", action)
}
+15 -1
View File
@@ -17,7 +17,7 @@ import (
// from its own manifests as built (issue 1953, ADR 0565 §3); the controller's records are no longer read.
func (c *Config) sources() (Sources, error) {
s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{},
Predefined: map[string]Published{}, Values: map[string]Value{}}
Predefined: map[string]Published{}, Values: map[string]Value{}, Renames: map[string]Rename{}}
var problems []string
if err := c.readPlacement(&s); err != nil {
return s, fmt.Errorf("the seat mesh-placement's placement cannot be read: %w", err)
@@ -51,6 +51,11 @@ func (c *Config) sources() (Sources, error) {
if err := c.readPulls(&s); err != nil {
return s, err
}
if renames, err := c.readRenames(); err != nil {
return s, fmt.Errorf("the renames cannot be read: %w", err)
} else {
s.Renames = renames
}
for _, p := range problems {
c.logf("[mesh-config] a placed module's declarations are not read, so its values are not resolved (they "+
"stay as last resolved): %s", p)
@@ -354,6 +359,8 @@ type Resolution struct {
// did after the take-over.
Declared []string `json:"declared,omitempty"`
Candidates []string `json:"candidates,omitempty"`
// Renamed is what the renames did in this pass.
Renamed []string `json:"renamed,omitempty"`
// SourceGone is said while the stored switch still names the controller, whose records are no longer read.
SourceGone string `json:"source_gone,omitempty"`
}
@@ -364,6 +371,11 @@ type Resolution struct {
func (c *Config) Reconcile() (Resolution, error) {
c.mu.Lock()
defer c.mu.Unlock()
return c.reconcileLocked()
}
// reconcileLocked is Reconcile with the lock held.
func (c *Config) reconcileLocked() (Resolution, error) {
var r Resolution
published, err := c.publish()
if err != nil {
@@ -374,6 +386,8 @@ func (c *Config) Reconcile() (Resolution, error) {
if err != nil {
return r, err
}
// The renames move their values first, so the same pass resolves them under the new names (renames.go).
r.Renamed = c.applyRenames(&s)
moved, err := c.writeResolved(s)
r.Moved = moved
if len(moved) > 0 {
@@ -11,12 +11,18 @@ import (
// The switch's tools.
const (
DeclaredTool = "config_declared"
ResyncTool = "config_resync"
DeclaredTool = "config_declared"
ResyncTool = "config_resync"
RenamesTool = "config_renames"
RenameAgainTool = "config_rename_again"
RenameCloseTool = "config_rename_close"
RetiringTool = "config_retiring"
RenamesVerb = "renames"
)
// SeatVerbs are the seat's verbs, as the manifest claims them.
var SeatVerbs = []string{Verb}
// SeatVerbs are the seat's verbs, as the manifest claims them: `config`, and the optional `renames` the checker asks
// (design 32: the open renames and the nodes holding each window open).
var SeatVerbs = []string{Verb, RenamesVerb}
func switchTools(c *Config) []stdio.Tool { return ownSwitchTools(c) }
@@ -35,5 +41,48 @@ func ownSwitchTools(c *Config) []stdio.Tool {
Description: "Read every source again and write what differs: the published list of predefined keys, the " +
"resolved values, declared, and the indexer's config candidates.",
Run: func(map[string]any) (any, error) { return c.Resync() }},
{Name: RenamesTool,
Description: "Each config value renamed (renamed-from): from which name to which, when, the nodes its value " +
"moved on, the nodes still holding the old name open (the rename window) and how each closed one closed. " +
"Narrowed by module.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "one module"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
rows, err := c.Renames(strings.TrimSpace(m), false)
return map[string]any{"renames": rows}, err
}},
{Name: RenameAgainTool,
Description: "Move a renamed value by hand, the way a resolution does: after a refused move (its finding says " +
"why) once the cause is put right, or for a value set under the old name meanwhile.",
Input: map[string]any{"type": "object", "required": []string{"module", "to"}, "properties": map[string]any{
"module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
to, _ := a["to"].(string)
return c.RenameAgain(strings.TrimSpace(m), strings.TrimSpace(to))
}},
{Name: RenameCloseTool,
Description: "Drop a node from a rename's window by hand (a node retired for good), recorded as a hand act: " +
"the old name is no longer answered for it.",
Input: map[string]any{"type": "object", "required": []string{"module", "to", "node", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"},
"node": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}},
Run: func(a map[string]any) (any, error) {
str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) }
return c.RenameClose(str("module"), str("to"), str("node"), str("why"))
}},
{Name: RetiringTool,
Description: "The config values and secrets no manifest reads any more: since when, where each stands (draining " +
"while a node still runs a build that reads it, asked, unanswered, approved, declined) and its ask. With " +
"action ask-again, ask the operator once more about one; with cancel, drop one (it is asked about no more " +
"until it is read again).",
Input: map[string]any{"type": "object", "properties": map[string]any{
"action": map[string]any{"type": "string", "enum": []string{"list", "ask-again", "cancel"}},
"module": map[string]any{"type": "string"}, "key": map[string]any{"type": "string"}}},
Run: func(a map[string]any) (any, error) {
str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) }
return c.ConfigRetiring(str("action"), str("module"), str("key"))
}},
}
}
+34 -7
View File
@@ -7,12 +7,16 @@
"name": "mesh-config",
"scope": "mesh",
"serves": [
"config"
"config",
"renames"
]
}
],
"emits": [
"config-changed"
"config-changed",
"events.renamed",
"events.retiring",
"events.retired"
],
"uses": [
"operator-channel"
@@ -20,12 +24,19 @@
"consumes": [
"gitea.pull.merged",
"gitea.pull.closed",
"gitea.forge.pull-request.closed"
"gitea.forge.pull-request.closed",
"mesh-delivery-view.delivered",
"node-engine.manifest-reads-changed",
"mesh-delivery-indexer.manifest-accepted"
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.warranted",
"gitea.gitea_get_pull_request"
"gitea.gitea_get_pull_request",
"mesh-delivery-view.node_haves",
"seat:node-engine.manifest-reads",
"mesh-delivery-indexer.indexer_config_readers",
"seat:mesh-vault.retire"
],
"reads": [
"mesh-delivery-assigner.placement",
@@ -38,7 +49,11 @@
],
"tools": [
"config_declared",
"config_resync"
"config_resync",
"config_renames",
"config_rename_again",
"config_rename_close",
"config_retiring"
],
"state": [
"values",
@@ -47,7 +62,10 @@
"proposals",
"pulls",
"kept",
"declared"
"declared",
"renames",
"retiring",
"findings"
],
"logs": [
"history"
@@ -87,5 +105,14 @@
]
}
]
}
},
"streams": [
{
"name": "config-events",
"events": [
"events.**"
],
"max-age-days": 7
}
]
}