Join: merge A4 (wip/registry-a4) into the registry milestone

This commit is contained in:
2026-11-04 15:11:07 +01:00
45 changed files with 4066 additions and 91 deletions
@@ -93,6 +93,15 @@ var Homes = map[string]Home{
"asks-undelivered": {Source: "asks", Controller: true, Observer: messenger},
// The seat mesh-artifact-sweep (L7): an image a want names is missing from the store, artifact-store-tools' finding.
"image-missing": {Source: "S25", Controller: true, Observer: artifactStore},
// mesh-config's own (design 32): a rename it refused, a rename window or a retirement held seven days by a node, a
// retirement asked twice and not answered.
"rename-refused": {Source: "config", Observer: configModule},
"rename-window-held": {Source: "config", Observer: configModule},
"retirement-held": {Source: "config", Observer: configModule},
"retiring-unanswered": {Source: "config", Observer: configModule},
// A reader whose owner does not answer its first view past two minutes (ADR 0615 §6): written by each reader in its own
// findings (mesh-config, mesh-vault), so the kind has no one observer.
"first-view-unanswered": {Source: "views", Why: "every reader of the indexer's or the assigner's state writes its own"},
// The kinds that end with the controller and are carried until its lease write (clauses 4, 9).
"controller-deaf": {Source: "S4", Controller: true, Carried: true},
"lease-lost": {Source: "S12", Controller: true, Carried: true},
@@ -155,6 +164,8 @@ const (
provisioner = "mesh-provisioner"
messenger = "messenger"
artifactStore = "artifact-store-tools"
configModule = "mesh-config"
vaultModule = "mesh-vault"
)
// homeOf is the home of a condition's kind, and whether the table has one.
@@ -40,7 +40,7 @@ const HistoryKeptFor = 90 * 24 * time.Hour
// table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder,
// which observes S25, image-missing).
var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule,
"mesh-provisioner", "messenger", artifactStore}
"mesh-provisioner", "messenger", artifactStore, configModule, vaultModule}
// The other states it reads, as `<module>.<name>`.
const (
@@ -294,7 +294,7 @@ func TestEveryControllerKindHasItsHome(t *testing.T) {
t.Errorf("the ended probe's kind %s is %+v", kind, h)
}
}
if len(Homes) != 84 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled
if len(Homes) != 89 || carried != 37 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled, mesh-config's four (design 32) and the readers' unanswered first view
t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried)
}
}
+2
View File
@@ -21,6 +21,8 @@
"messenger.findings",
"mesh-provisioner.findings",
"artifact-store-tools.findings",
"mesh-config.findings",
"mesh-vault.findings",
"mesh-healers.attempts",
"mesh-delivery-assigner.wants",
"node-engine.have",
+20
View File
@@ -56,6 +56,26 @@ value as a set does, keeps who approved it beside the value (`approved`), and as
warrant in its hand-act log. A Decline, an expiry or the router's refusal sets nothing. `action` `proposals` lists
them, newest first; with `proposal`, one read whole, with whether the router's ask is the one composed from it.
## A value renamed, a value no longer read (design 32, novox/hq ADR 0609)
- A declaration's `renamed-from` moves the value set under the old name to the new one on each node, with its trust (it
moves whole, and a trusted new name over a value set without the Approve is refused), saying `renamed`
(module, from, to, at) once. A **rename window** opens per node and keeps the old name answering the new value in
`mesh-config_resolved` until that node's node-engine runs a build that reads the new name: judged at start, on the
status view's `delivered`, and for a returning node-engine on the node-engine seat's `manifest-reads-changed` after
`manifest-reads` answers (one that does not serve it states nothing, and its window stays open). State `renames`.
- The seat verb `renames` answers the open renames and the nodes holding each open (the checker asks it); the tools
`config_renames` (all, with the closed windows), `config_rename_again` (move by hand after a refusal) and
`config_rename_close` (drop a node by hand, a hand act).
- On the indexer's `manifest-accepted` it asks the indexer's `indexer_config_readers`; an indexer that does not serve
it answers "not known" and nothing retires. A value (or secret) nobody reads is **retiring** once no node runs a
build that reads it: `retiring` is said and the operator is asked once; the value is removed only on the Approve
(a secret through the seat mesh-vault's `retire`, with a hand act naming node, module, name and ask), `retired`
said per node. State `retiring`; the tool `config_retiring` lists, asks again and cancels.
- Findings (state `findings`, read by the condition keeper): `rename-refused`, `rename-window-held` and
`retirement-held` (seven days, naming the node), `retiring-unanswered` (asked twice).
- The events `renamed`, `retiring` and `retired` are carried by the stream `config-events`, kept seven days.
## Files it does not know
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
@@ -93,6 +93,10 @@ type Config struct {
Asks AskBus
// Forge is the forge as pulls are asked of it (pulls.go); nil, a merge or a close is heard from its events alone.
Forge PullForge
// Delivery is what the delivery parts answer (renames.go, retiring.go); nil, no window closes and nothing retires.
Delivery Delivery
// Vault is the seat mesh-vault, for the retiring of a secret; nil, a secret does not retire.
Vault Vault
// RouterWithin and RouterEvery bound how long a proposal waits for the router's word on its ask, and how often
// it looks; zero is RouterAnswersWithin and RouterAnswersEvery.
RouterWithin, RouterEvery time.Duration
@@ -20,6 +20,7 @@ type memStore struct {
states map[string]map[string][]byte
logs map[string][]LogEntry
events []Change
said []saidEvent
handActs []string
// failHandAct makes the hand-act log refuse.
failHandAct bool
@@ -89,15 +90,38 @@ func (m *memStore) Read(log string) ([]LogEntry, error) {
}
func (m *memStore) Emit(event string, body any) error {
if event != ConfigChanged {
return fmt.Errorf("mesh-config emits no %s", event)
}
m.mu.Lock()
defer m.mu.Unlock()
m.events = append(m.events, body.(Change))
switch event {
case ConfigChanged:
m.events = append(m.events, body.(Change))
case EventRenamed, EventRetiring, EventRetired:
m.said = append(m.said, saidEvent{event, body})
default:
return fmt.Errorf("mesh-config emits no %s", event)
}
return nil
}
// saidEvent is an event of the family config.events as it was emitted.
type saidEvent struct {
Event string
Body any
}
// saidOf is the events of one name emitted so far.
func (m *memStore) saidOf(event string) []any {
m.mu.Lock()
defer m.mu.Unlock()
var out []any
for _, e := range m.said {
if e.Event == event {
out = append(out, e.Body)
}
}
return out
}
func (m *memStore) HandAct(what, why, cause string) error {
m.mu.Lock()
defer m.mu.Unlock()
@@ -41,6 +41,9 @@ type Declaration struct {
Predefined string `json:"predefined,omitempty"`
Required bool `json:"required,omitempty"`
Runtime bool `json:"runtime,omitempty"`
// RenamedFrom is the key's old name when it was renamed (novox/hq ADR 0609 §4, design 32): on the merge the value
// set under the old name moves to this key (renames.go).
RenamedFrom string `json:"renamed-from,omitempty"`
}
// HasDefault is whether the declaration gives a default: present and not null.
@@ -0,0 +1,132 @@
package main
// What mesh-config asks of the delivery parts and the vault, through their tools and seat verbs on the bus: never a
// bucket of theirs (novox/hq ADR 0596). Each key is the address a module's `invokes` names (mesh-sdk calladdress).
import (
"encoding/json"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
)
// The addresses asked.
const (
// HavesAsk is the status view's read of the node-engines' haves, narrowed to one node and module.
HavesAsk = "mesh-delivery-view.node_haves"
// ManifestReadsAsk is the node-engine seat's verb saying what manifest forms a node-engine reads (ADR 0618 §3), asked
// of one node by the address's `@<node>`.
ManifestReadsAsk = "seat:node-engine.manifest-reads"
// ConfigReadersAsk is the indexer's own tool saying who reads a module's config keys (design 32).
ConfigReadersAsk = "mesh-delivery-indexer.indexer_config_readers"
// VaultRetireAsk is the seat mesh-vault's verb retiring one module's own secret on a node (design 32).
VaultRetireAsk = "seat:mesh-vault.retire"
)
// toolDelivery answers Delivery through the bus.
type toolDelivery struct {
ask func(key string, body any) (json.RawMessage, error)
}
// Have asks node_haves for one node and module and reads the fingerprint the node runs.
func (d toolDelivery) Have(node, module string) (string, bool, error) {
raw, err := d.ask(HavesAsk, map[string]any{"node": node, "module": module})
if err != nil {
return "", false, err
}
answer, output, ok := unwrapAnswer(raw)
if !ok {
return "", false, fmt.Errorf("node_haves refused: %s", output)
}
var kept struct {
Keys map[string]json.RawMessage `json:"keys"`
}
if err := json.Unmarshal(answer, &kept); err != nil {
return "", false, fmt.Errorf("node_haves answered what is not its listing: %s", output)
}
key, err := delivery.HaveKey(node, module)
if err != nil {
return "", false, err
}
have, ok := kept.Keys[key]
if !ok {
return "", false, nil
}
var h delivery.Have
if err := json.Unmarshal(have, &h); err != nil {
return "", false, fmt.Errorf("the have of %s on %s is not readable: %w", module, node, err)
}
return h.FP, true, nil
}
// Reads asks a node's node-engine what it reads; an answer means it states it.
func (d toolDelivery) Reads(node string) error {
_, err := d.ask(ManifestReadsAsk+"@"+node, map[string]any{})
return err
}
// Readers asks the indexer who reads a module's config keys; an indexer that does not serve the tool yet is "not known".
func (d toolDelivery) Readers(module string) ([]ReaderRow, bool, error) {
raw, err := d.ask(ConfigReadersAsk, map[string]any{"module": module})
if err != nil {
if notServed(err) {
return nil, false, nil
}
return nil, false, err
}
answer, output, ok := unwrapAnswer(raw)
if !ok {
if notServed(fmt.Errorf("%s", output)) {
return nil, false, nil
}
return nil, false, fmt.Errorf("indexer_config_readers refused: %s", output)
}
var a struct {
Readers []ReaderRow `json:"readers"`
}
if err := json.Unmarshal(answer, &a); err != nil {
return nil, false, fmt.Errorf("indexer_config_readers answered what is not its listing: %s", output)
}
return a.Readers, true, nil
}
// notServed is whether an error says the address has no holder serving it.
func notServed(err error) bool {
m := strings.ToLower(err.Error())
for _, w := range []string{"no responders", "not served", "unknown tool", "no such tool", "serves no"} {
if strings.Contains(m, w) {
return true
}
}
return false
}
// toolVault asks the seat mesh-vault to retire a secret: the hand act that warrants it is recorded first, by this
// module, naming the node, module, secret and the ask of the Approve; the vault reads it back by its id (`warrant`).
type toolVault struct {
ask func(key string, body any) (json.RawMessage, error)
recorder handacts.Recorder
}
// Retire records the warrant and calls the vault's verb.
func (v toolVault) Retire(r RetireRequest) error {
taken, err := v.recorder.Record(handacts.Request{What: r.Node + " " + r.Module + " " + r.Name + " " + r.Ask,
Why: "the operator approved retiring " + r.Module + "'s secret " + r.Name + " on " + r.Node + " (ask " + r.Ask + ")",
Cause: "the operator's Approve of ask " + r.Ask, Verb: "mesh-vault.retire"})
if err != nil {
return err
}
answer, _, _ := unwrapAnswer(taken.Answer)
var rec struct {
Act struct {
ID string `json:"id"`
} `json:"act"`
}
if json.Unmarshal(answer, &rec) != nil || rec.Act.ID == "" {
return fmt.Errorf("mesh-hand-acts recorded the warrant and named no act")
}
_, err = v.ask(VaultRetireAsk, map[string]any{"node": r.Node, "module": r.Module, "name": r.Name, "warrant": rec.Act.ID, "ask": r.Ask})
return err
}
@@ -0,0 +1,138 @@
package main
// mesh-config's findings (novox/hq ADR 0564 clause 2): what it sees wrong about a rename or a retirement, kept in its
// state `findings` as the condition keeper reads them, written when what they say changes and deleted when the fault
// is gone. The keeper raises and clears the condition of the same key (mesh-conditions' Observers and Homes).
import (
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// FindingsState is the state, health.FindingsState.
const FindingsState = health.FindingsState
// The kinds of finding mesh-config writes.
const (
// FindingRenameRefused is a move mesh-config refused; cleared once the value moved or `renamed-from` left the
// trunk's manifest.
FindingRenameRefused = "rename-refused"
// FindingWindowHeld is a rename window held open seven days by a node; cleared when the window closes.
FindingWindowHeld = "rename-window-held"
// FindingRetireUnanswered is a retirement's ask unanswered after it was asked once more; cleared by an answer, a
// cancel, or the value read again.
FindingRetireUnanswered = "retiring-unanswered"
// FindingRetireHeld is a retirement waiting seven days for a node still running a build that reads the value.
FindingRetireHeld = "retirement-held"
)
// findingKey is the key of a finding of kind about a module on a node: the condition's key.
func findingKey(kind, module, node, ref string) string {
id := module
if node != "" {
id += "." + node
}
return health.Key(health.ScopeModule, id, kind+"-"+ref)
}
func finding(kind, module, node, ref, summary, headline, explanation, needs, resolved, said string, at time.Time) health.Finding {
id := module
if node != "" {
id += "." + node
}
return health.Finding{V: health.FindingVersion, Key: findingKey(kind, module, node, ref), Kind: kind, Ref: kind + "-" + ref,
Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node},
Severity: health.Warning, Resolver: health.ResolverOperator, Summary: summary, Headline: headline,
Explanation: explanation, Needs: needs, Resolved: resolved,
Evidence: []health.Evidence{{At: at.UTC(), Said: said}}, Source: "config", Observer: Module, At: at.UTC()}
}
// renameRefused is the finding of a move mesh-config refused.
func renameRefused(module, to, from, why string, at time.Time) health.Finding {
return finding(FindingRenameRefused, module, "", to,
fmt.Sprintf("the rename of %s's config value %s to %s is refused: %s", module, from, to, why),
fmt.Sprintf("%s's value %s is not moved to %s", module, from, to),
fmt.Sprintf("The new version of %s names %s as the new name of its config value %s, and mesh-config refused to move the value: %s. Nothing was moved.", module, to, from, why),
fmt.Sprintf("put the cause right, then move it with config_rename_again (module %s, to %s); the details name it.", module, to),
fmt.Sprintf("Resolved: %s's value %s is under %s", module, from, to), why, at)
}
// windowHeld is the finding of a rename window held open past HeldFinding by a node.
func windowHeld(r Rename, node string, at time.Time) health.Finding {
return finding(FindingWindowHeld, r.Module, node, r.To,
fmt.Sprintf("%s on %s has held the old config name %s open for %d days", r.Module, node, r.From, int(HeldFinding.Hours()/24)),
fmt.Sprintf("%s on %s still runs the old config name", r.Module, node),
fmt.Sprintf("The rename of %s's config value %s to %s is not complete on %s: its node-engine has not reported a build that reads %s (it may be away).", r.Module, r.From, r.To, node, r.To),
fmt.Sprintf("bring %s back, or drop it from the window with config_rename_close.", node),
fmt.Sprintf("Resolved: %s reads %s on %s", r.Module, r.To, node),
fmt.Sprintf("the window opened %s and %s has not reported a build reading %s", r.Nodes[node].Since.UTC().Format(time.RFC3339), node, r.To), at)
}
// setFinding writes a finding when it differs from the one kept.
func (c *Config) setFinding(f health.Finding) {
if _, err := health.EncodeFinding(f); err != nil {
c.logf("[mesh-config] a finding is not valid: %v", err)
return
}
if e, err := c.Store.Get(FindingsState, f.Key); err == nil && e != nil {
if old, derr := health.DecodeFinding(e.Value); derr == nil && old.Summary == f.Summary {
return // what it says has not changed
}
}
if err := c.Store.Put(FindingsState, f.Key, f); err != nil {
c.logf("[mesh-config] the finding %s is not written: %v", f.Key, err)
}
}
// clearFinding deletes the findings of a kind and ref about a module, on whatever node but those in keep.
func (c *Config) clearFinding(kind, module, ref string, keep ...string) {
keys, err := c.Store.Keys(FindingsState)
if err != nil {
return
}
want := kind + "-" + ref
for _, k := range keys {
e, err := c.Store.Get(FindingsState, k)
if err != nil || e == nil {
continue
}
f, err := health.DecodeFinding(e.Value)
if err != nil || f.Ref != want || !(f.Subject.ID == module || strings.HasPrefix(f.Subject.ID, module+".")) ||
(f.Subject.Machine != "" && slices.Contains(keep, f.Subject.Machine)) {
continue
}
if err := c.Store.Delete(FindingsState, k); err != nil {
c.logf("[mesh-config] the finding %s is not cleared: %v", k, err)
}
}
}
// retirementHeld is the finding of a retirement that waits seven days for nodes that still run a build reading the value.
func retirementHeld(e Retiring, at time.Time) health.Finding {
node := ""
if len(e.Nodes) > 0 {
node = e.Nodes[0]
}
return finding(FindingRetireHeld, e.Module, node, e.Key,
fmt.Sprintf("%s's %s has waited %d days to retire: %s still run a build that reads it", e.Module, e.Key, int(HeldFinding.Hours()/24), strings.Join(e.Nodes, ", ")),
fmt.Sprintf("%s's %s cannot be retired yet", e.Module, e.Key),
fmt.Sprintf("No manifest reads %s's %s any more, but %s still run a build that does, so the operator is not asked to remove it.", e.Module, e.Key, strings.Join(e.Nodes, ", ")),
"bring those nodes up to date, or cancel the retirement with config_retiring.",
fmt.Sprintf("Resolved: no node reads %s's %s", e.Module, e.Key),
"retiring since "+e.Since.UTC().Format(time.RFC3339)+", still read on "+strings.Join(e.Nodes, ", "), at)
}
// retireUnanswered is the finding of a retirement asked twice and not answered.
func retireUnanswered(e Retiring, at time.Time) health.Finding {
return finding(FindingRetireUnanswered, e.Module, "", e.Key,
fmt.Sprintf("the operator did not answer the ask to remove %s's %s, asked twice", e.Module, e.Key),
fmt.Sprintf("%s's %s waits for an answer", e.Module, e.Key),
fmt.Sprintf("No manifest reads %s's %s any more. The operator was asked twice whether to remove it and did not answer, so it stays.", e.Module, e.Key),
"answer the ask again with config_retiring ask-again, or cancel it.",
fmt.Sprintf("Resolved: %s's %s is answered or read again", e.Module, e.Key),
"asked "+e.Asked.UTC().Format(time.RFC3339)+" and once more", at)
}
+120 -21
View File
@@ -10,6 +10,7 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
@@ -30,10 +31,19 @@ const (
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
// runtimeStore is the bus, through the runtime that launched this bundle.
// theViews are the pictures of the delivery owners' state, made when the module runs (nil in a test of the tools).
var theViews *ownerViews
// runtimeStore is the bus, through the runtime that launched this bundle. A state of the delivery owners that the
// owners' tools answer is read from the picture those keep (ownerviews.go), the rest from the bus.
type runtimeStore struct{}
func (runtimeStore) Get(state, key string) (*Entry, error) {
if theViews != nil {
if e, followed, err := theViews.get(state, key); followed {
return e, err
}
}
e, err := stdio.State(state).Get(key)
if err != nil || e == nil {
return nil, err
@@ -48,7 +58,14 @@ func (runtimeStore) Put(state, key string, value any) error {
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() }
func (runtimeStore) Keys(state string) ([]string, error) {
if theViews != nil {
if keys, followed, err := theViews.keys(state); followed {
return keys, err
}
}
return stdio.State(state).Keys()
}
func (runtimeStore) Append(log, key string, value any) error {
_, err := stdio.Log(log).Append(key, value)
@@ -100,9 +117,23 @@ func (runtimeAsks) Warranted(ask string) error {
}
func main() {
c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}}
go resolving(c)
c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask},
Delivery: toolDelivery{ask: stdio.Ask}, Vault: toolVault{ask: stdio.Ask, recorder: recorder}}
moved := make(chan struct{}, 1)
theViews = newOwnerViews(stdio.Ask, func() {
select {
case moved <- struct{}{}:
default:
}
}, func(key string, v json.RawMessage) error {
_, err := stdio.State(FindingsState).Put(key, v)
return err
},
func(key string) error { return stdio.State(FindingsState).Delete(key) })
theViews.Start(context.Background())
go resolving(c, moved)
go hearing(c)
go hearingDelivery(c)
if err := stdio.Serve("", Tools(c)); err != nil {
logf("%v", err)
os.Exit(1)
@@ -112,22 +143,9 @@ func main() {
// resolving resolves at start and again whenever a source moves, and every five minutes besides, so a change
// heard nowhere is still resolved. The runtime answers only once Serve runs, so the watches are asked again
// until they are taken.
func resolving(c *Config) {
moved := make(chan struct{}, 1)
poke := func(stdio.StateChange) error {
select {
case moved <- struct{}{}:
default:
}
return nil
}
// A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is
// every change of what a module declares where (issue 1953); the builds themselves are read, not watched.
for _, read := range []string{PlacementRead, WantsRead, ModulesRead, IndexerPredefinedRead,
CandidatesRead} {
// Asked again until the runtime takes it, each give-up said (mesh-sdk's Follow).
go stdio.State(read).Follow("", poke)
}
func resolving(c *Config, moved chan struct{}) {
// The owners' states reach `moved` through their views (ownerviews.go); the builder's builds, which has no tool yet,
// is read when a want moves, and nothing else is watched.
tick := time.NewTicker(5 * time.Minute)
defer tick.Stop()
time.Sleep(time.Second)
@@ -221,9 +239,90 @@ func hearing(c *Config) {
}
}
// hearingDelivery takes the delivery parts' words that close a rename window or retire a value, each asked again until
// the runtime takes the subscription: the status view's `delivered`, the node-engine seat's `manifest-reads-changed`
// (a returning node-engine; design 32) and the indexer's `manifest-accepted`. At its start it judges the windows once,
// since an event said while this module was down is not said again.
func hearingDelivery(c *Config) {
<-stdio.Served()
for _, h := range []struct {
pattern string
handle func(stdio.Envelope) []string
}{
{"mesh-delivery-view.delivered", func(stdio.Envelope) []string { return c.Windows("") }},
{"node-engine.manifest-reads-changed", func(e stdio.Envelope) []string {
var b struct {
Node string `json:"node"`
}
_ = json.Unmarshal(e.Body, &b)
if b.Node == "" {
// The event's subject ends with the node's token (ADR 0618 as built): mesh.seat.node-engine.event.manifest-reads-changed.<node>.
b.Node = e.Key[strings.LastIndex(e.Key, ".")+1:]
}
return c.Windows(b.Node)
}},
{"mesh-delivery-indexer.manifest-accepted", func(e stdio.Envelope) []string {
var b struct {
Module string `json:"module"`
}
if json.Unmarshal(e.Body, &b) != nil {
return nil
}
return c.Retirements(b.Module)
}},
} {
go func(pattern string, handle func(stdio.Envelope) []string) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.Subscribe(pattern, func(e stdio.Envelope) error {
for _, s := range handle(e) {
logf("[mesh-config] %s", s)
}
return nil
})
if err == nil {
return
}
logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait)
time.Sleep(wait)
}
}(h.pattern, h.handle)
}
for _, s := range c.Windows("") {
logf("[mesh-config] %s", s)
}
}
// seatRename is one open rename as the seat's definition answers it: module, from, to, at and the nodes holding it open.
type seatRename struct {
Module string `json:"module"`
From string `json:"from"`
To string `json:"to"`
At time.Time `json:"at"`
Open []string `json:"open"`
}
func seatRenames(rows []RenameRow) []seatRename {
out := make([]seatRename, 0, len(rows))
for _, r := range rows {
out = append(out, seatRename{Module: r.Module, From: r.From, To: r.To, At: r.At, Open: r.Open})
}
return out
}
// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them.
func Tools(c *Config) []stdio.Tool {
return append(switchTools(c), stdio.Tool{
renames := stdio.Tool{Name: Seat + "." + RenamesVerb,
Description: "The open renames of config values: for each, the module, the old and the new name, and the nodes " +
"still holding the old name open (the rename window). The checker asks it before it lets a manifest add or " +
"drop renamed-from (novox/hq design 32). Narrowed by module.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "one module"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
rows, err := c.Renames(strings.TrimSpace(m), true)
return map[string]any{"renames": seatRenames(rows)}, err
}}
return append(append(switchTools(c), viewTools(c, func() *ownerViews { return theViews })...), renames, stdio.Tool{
Name: Seat + "." + Verb,
Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " +
"one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " +
@@ -6,6 +6,8 @@ import (
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself
@@ -35,6 +37,10 @@ func TestEveryStateAndActHasItsTool(t *testing.T) {
PlacementRead: "config list", WantsRead: "config list", BuildsRead: "config list", ModulesRead: DeclaredTool,
ManifestsRead: DeclaredTool, IndexerPredefinedRead: "config list", CandidatesRead: ResyncTool,
"a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool,
// design 32: each new state with the tool that reads it, each act with the tool that starts it.
RenamesState: RenamesTool, RetiringState: RetiringTool, FindingsState: "config_renames and config_retiring",
"a move refused": RenameAgainTool, "a window dropped": RenameCloseTool, "an ask again": RetiringTool,
"the owners' first views": FirstViewsTool, "a first view asked again": FirstViewAgainTool,
}
served := map[string]bool{}
for _, tool := range Tools(&Config{}) {
@@ -43,6 +49,21 @@ func TestEveryStateAndActHasItsTool(t *testing.T) {
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") {
t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs)
}
// The stream and its events are declared as the module emits them, and the manifest decodes whole.
if dm, err := catalogue.Decode(raw); err != nil {
t.Errorf("the manifest does not decode: %v", err)
} else {
emits := strings.Join(dm.Emits, ",")
for _, e := range []string{ConfigChanged, EventRenamed, EventRetiring, EventRetired} {
if !strings.Contains(emits, e) {
t.Errorf("the manifest does not declare the event %s", e)
}
}
if len(dm.Streams) != 1 || dm.Streams[0].Name != StreamName || dm.Streams[0].Days() != 7 || !dm.Streams[0].Carries(EventRenamed) ||
!dm.Streams[0].Carries(EventRetiring) || !dm.Streams[0].Carries(EventRetired) || dm.Streams[0].Carries(ConfigChanged) {
t.Errorf("streams %+v: want %s, seven days, carrying renamed, retiring and retired but not config-changed", dm.Streams, StreamName)
}
}
for _, verb := range SeatVerbs {
if !served[Seat+"."+verb] {
t.Errorf("the seat's verb %s is not served", verb)
@@ -53,6 +74,11 @@ func TestEveryStateAndActHasItsTool(t *testing.T) {
t.Errorf("the manifest lists the tool %s, which is not served", name)
}
}
// The delivery owners' state is read through their tools and views (ADR 0615 §1): only the builder's builds, which has
// no tool yet, is still a read.
if len(m.Reads) != 1 || m.Reads[0] != BuildsRead {
t.Errorf("reads %v, want the builder's builds alone", m.Reads)
}
for _, list := range [][]string{m.State, m.Logs, m.Reads} {
for _, s := range list {
if _, ok := via[s]; !ok {
@@ -0,0 +1,249 @@
package main
// What mesh-config reads of the delivery owners' state, through their tools and views families (novox/hq ADR 0615 §1,
// §4 to §6, ADR 0596): never a bucket of theirs. For each of the six states it follows (the indexer's modules, manifests,
// predefined and config-candidates, the assigner's placement and wants) it asks the owner's read tool for the whole
// state at its start, keeps the picture from the owner's `views.<state>` events, and asks again only on a gap. Started
// while an owner is down it keeps asking, and past two minutes writes a finding naming the owner and the tool.
//
// The ordinary read tools are asked, not the first-view tools: those answer a few named callers (mesh-grants,
// node-tools, the delivery parts), and mesh-config is not one. The builder's `builds` has no owner tool yet (A2 built the
// indexer's and the assigner's), so it is still read as a `reads` entry.
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/views"
)
// The tools of the owners that mesh-config asks, by the state they read.
const (
IndexerModulesTool = "indexer_read_modules"
IndexerManifestsTool = "indexer_read_manifests"
IndexerPredefTool = "indexer_read_predefined"
IndexerCandidatesTool = "indexer_read_config_candidates"
AssignerPlacementTool = "assigner_read_placement"
AssignerWantsTool = "assigner_read_wants"
FirstViewsTool = "config_first_views"
FirstViewAgainTool = "config_first_view_again"
)
// followed is each state read through its owner: the `<owner>.<state>` name this module's code uses, the owner, its
// state and its tool.
var followed = []struct{ Read, Owner, State, Tool string }{
{ModulesRead, delivery.IndexerModule, "modules", IndexerModulesTool},
{ManifestsRead, delivery.IndexerModule, "manifests", IndexerManifestsTool},
{IndexerPredefinedRead, delivery.IndexerModule, delivery.IndexerPredefinedState, IndexerPredefTool},
{CandidatesRead, delivery.IndexerModule, delivery.IndexerConfigCandidatesState, IndexerCandidatesTool},
{PlacementRead, delivery.AssignerModule, "placement", AssignerPlacementTool},
{WantsRead, delivery.AssignerModule, "wants", AssignerWantsTool},
}
// ownerViews are the pictures, by the read name.
type ownerViews struct {
by map[string]*views.Follower
// store keeps the findings a first view unanswered writes.
store func(key string, value json.RawMessage) error
drop func(key string) error
}
// newOwnerViews makes a follower for each state; changed is told when any picture changes.
func newOwnerViews(ask func(key string, body any) (json.RawMessage, error), changed func(), put func(string, json.RawMessage) error,
del func(string) error) *ownerViews {
o := &ownerViews{by: map[string]*views.Follower{}, store: put, drop: del}
for _, f := range followed {
f := f
fol := &views.Follower{Owner: f.Owner, Tool: f.Owner + "." + f.Tool, State: f.State, Logf: logf,
Ask: func(_ context.Context, after string) (views.Answer, error) {
body := map[string]any{"values": true}
if after != "" {
body["after"] = after
}
raw, err := ask(f.Owner+"."+f.Tool, body)
if err != nil {
return views.Answer{}, err
}
answer, output, ok := unwrapAnswer(raw)
if !ok {
return views.Answer{}, fmt.Errorf("%s refused: %s", f.Tool, output)
}
return views.DecodeAnswer(answer)
},
Changed: func(string, json.RawMessage, bool, time.Time) { changed() }}
fol.Raise = func(s views.Status) {
if k, v, err := views.UnansweredFinding(Module, s, time.Now()); err == nil {
_ = put(k, v)
}
}
fol.Clear = func(s views.Status) {
if k, _, err := views.UnansweredFinding(Module, s, time.Now()); err == nil {
_ = del(k)
}
}
o.by[f.Read] = fol
}
return o
}
// Start asks every first view until answered (each on its goroutine) and subscribes to the owners' views events.
func (o *ownerViews) Start(ctx context.Context) {
for _, fol := range o.by {
go func(f *views.Follower) {
if err := f.Start(ctx); err != nil {
logf("[mesh-config] the first view of %s's %s ended unanswered: %v", f.Owner, f.State, err)
}
}(fol)
}
for _, f := range followed {
f := f
pattern := f.Owner + "." + views.EventName(f.State)
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.Subscribe(pattern, func(e stdio.Envelope) error {
ev, err := views.DecodeEvent(e.Body)
if err != nil {
logf("[mesh-config] %s is not a views event: %v", pattern, err)
return nil
}
fol := o.by[f.Read]
if gap, why := fol.Apply(ev); gap {
logf("[mesh-config] a gap in %s's %s (%s): its first view is asked once more", f.Owner, f.State, why)
go func() { _ = fol.Start(ctx) }()
}
return nil
})
if err == nil {
return
}
logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait)
time.Sleep(wait)
}
}()
}
}
// read is the picture of a read name, and whether the name is one followed.
func (o *ownerViews) read(state string) (*views.Follower, bool) {
f, ok := o.by[state]
return f, ok
}
// errUnanswered says a first view is not answered yet: a reader waits, and its next resolution reads it.
func errUnanswered(f *views.Follower) error {
s := f.Status()
return fmt.Errorf("the first view of %s's %s is not answered yet (%s asked %d times)", s.Owner, s.State, s.Tool, s.Asks)
}
// get is one key of a followed state.
func (o *ownerViews) get(state, key string) (*Entry, bool, error) {
f, ok := o.read(state)
if !ok {
return nil, false, nil
}
select {
case <-f.Ready():
default:
return nil, true, errUnanswered(f)
}
v, held := f.Get(key)
if !held {
return nil, true, nil
}
return &Entry{Key: key, Value: v}, true, nil
}
// keys are the keys of a followed state, sorted.
func (o *ownerViews) keys(state string) ([]string, bool, error) {
f, ok := o.read(state)
if !ok {
return nil, false, nil
}
select {
case <-f.Ready():
default:
return nil, true, errUnanswered(f)
}
var out []string
for k := range f.Values() {
out = append(out, k)
}
sort.Strings(out)
return out, true, nil
}
// statuses are the first views' statuses, by owner and state.
func (o *ownerViews) statuses() []views.Status {
var out []views.Status
for _, f := range followed {
out = append(out, o.by[f.Read].Status())
}
return out
}
// again asks one first view once more by hand ("<owner>.<state>", or all when empty).
func (o *ownerViews) again(ctx context.Context, which, why string) ([]string, error) {
var done []string
for _, f := range followed {
if which != "" && which != f.Owner+"."+f.State {
continue
}
if err := o.by[f.Read].Again(ctx, why); err != nil {
return done, err
}
done = append(done, f.Owner+"."+f.State)
}
if len(done) == 0 {
return nil, errors.New("no first view is named " + which)
}
return done, nil
}
// viewTools are the read tool of the first views and its re-read (design 25 § 12.2): the status of each, and a hand ask.
func viewTools(c *Config, o func() *ownerViews) []stdio.Tool {
return []stdio.Tool{
{Name: FirstViewsTool,
Description: "mesh-config's first views of the delivery owners' state: for each of the indexer's modules, manifests, " +
"predefined and config-candidates and the assigner's placement and wants, the tool it asks, the revision its " +
"picture holds, when it asked and was answered, how many asks (and by hand), and whether its condition is raised.",
Run: func(map[string]any) (any, error) {
ov := o()
if ov == nil {
return nil, errors.New("mesh-config reads its owners through views only when it runs as the module")
}
return map[string]any{"first-views": ov.statuses()}, nil
}},
{Name: FirstViewAgainTool,
Description: "Ask a first view once more by hand, `<owner>.<state>` (all when not given), recorded as a hand act with " +
"its why.",
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
"view": map[string]any{"type": "string", "description": "<owner>.<state>, such as mesh-delivery-assigner.wants"},
"why": map[string]any{"type": "string"}}},
Run: func(a map[string]any) (any, error) {
ov := o()
if ov == nil {
return nil, errors.New("mesh-config reads its owners through views only when it runs as the module")
}
why, _ := a["why"].(string)
view, _ := a["view"].(string)
if strings.TrimSpace(why) == "" {
return nil, errors.New("say why: a hand act keeps its why")
}
if err := c.Store.HandAct("config_first_view_again "+view, strings.TrimSpace(why), "config_first_view_again"); err != nil {
return nil, fmt.Errorf("the hand-act log does not hold it, so nothing is asked: %w", err)
}
done, err := ov.again(context.Background(), strings.TrimSpace(view), strings.TrimSpace(why))
return map[string]any{"asked-again": done}, err
}},
}
}
@@ -0,0 +1,76 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/views"
)
// CAT-C11 (ADR 0615 §1): a followed state of the delivery owners is read from the picture of its owner's tool and events:
// a key asked before the first view is answered is an error (a reader waits), never a read of the owner's bucket; after
// it, the owner's value; an event at the next revision changes it, one past it is a gap and asks the first view again; a
// state that is no owner's falls through to the bus.
func TestAFollowedStateIsReadFromItsOwnersFirstViewAndEvents(t *testing.T) {
created := time.Date(2026, 11, 4, 9, 0, 0, 0, time.UTC)
var asks atomic.Int32
hold := make(chan struct{})
ask := func(key string, body any) (json.RawMessage, error) {
if key != "mesh-delivery-assigner.assigner_read_wants" || body.(map[string]any)["values"] != true {
return nil, errors.New("asked " + key)
}
<-hold
n := asks.Add(1)
a := views.Answer{V: 1, Owner: "mesh-delivery-assigner", State: "wants", Revision: uint64(n) * 10, Created: created,
Values: map[string]json.RawMessage{"want.g14.dunst": json.RawMessage(`{"fp":"a"}`)}}
raw, _ := json.Marshal(a)
return raw, nil
}
var changed atomic.Int32
o := newOwnerViews(ask, func() { changed.Add(1) }, func(string, json.RawMessage) error { return nil }, func(string) error { return nil })
if _, followed, err := o.get(WantsRead, "want.g14.dunst"); !followed || err == nil || !strings.Contains(err.Error(), "not answered yet") {
t.Fatalf("before the first view: followed %v, %v", followed, err)
}
if _, followed, _ := o.get(BuildsRead, "x"); followed {
t.Error("the builder's builds is followed, and it has no owner tool")
}
ctx := context.Background()
go func() { _ = o.by[WantsRead].Start(ctx) }()
hold <- struct{}{}
<-o.by[WantsRead].Ready()
e, isFollowed, err := o.get(WantsRead, "want.g14.dunst")
if err != nil || !isFollowed || e == nil || string(e.Value) != `{"fp":"a"}` {
t.Fatalf("after the first view: %+v, %v, %v", e, isFollowed, err)
}
ev := func(rev uint64, key string, deleted bool) views.Event {
return views.Event{V: 1, Owner: "mesh-delivery-assigner", State: "wants", Key: key, Revision: rev, Created: created, Deleted: deleted,
Value: json.RawMessage(`{"fp":"b"}`)}
}
if gap, why := o.by[WantsRead].Apply(ev(11, "want.novox.dunst", false)); gap {
t.Fatalf("the next revision is a gap: %s", why)
}
if keys, _, _ := o.keys(WantsRead); strings.Join(keys, ",") != "want.g14.dunst,want.novox.dunst" {
t.Errorf("keys %v", keys)
}
if gap, _ := o.by[WantsRead].Apply(ev(11, "want.novox.dunst", false)); gap {
t.Error("an event already in the picture is a gap")
}
if gap, why := o.by[WantsRead].Apply(ev(13, "want.x.y", false)); !gap || !strings.Contains(why, "changes missed") {
t.Errorf("a missed revision: gap %v, %s", gap, why)
}
o.by[WantsRead].Apply(ev(12, "want.g14.dunst", true))
if e, _, _ := o.get(WantsRead, "want.g14.dunst"); e != nil {
t.Error("a deleted key is still held")
}
if changed.Load() < 3 {
t.Errorf("the resolution was told of %d changes", changed.Load())
}
if got := o.statuses(); len(got) != len(followed) {
t.Errorf("statuses %d, want %d", len(got), len(followed))
}
}
@@ -759,6 +759,8 @@ func (c *Config) Decide() ([]string, error) {
said = append(said, c.decide(r.p, r.w))
}
c.recordWarrants(now)
// The retirements' asks are taken from the router's record the same way (retiring.go).
said = append(said, c.decideRetirements()...)
return said, nil
}
@@ -0,0 +1,494 @@
package main
// A config value renamed (novox/hq ADR 0609 §4, design 32 *A config value renamed*).
//
// A declaration naming `renamed-from` moves the value set under the old name to the new name, with the value's trust
// (its Value, whole), and opens a **rename window** per node: until that node's node-engine has applied a build of the
// module that reads the new name, the old name still answers with the new value (Resolve). A window closes on the
// status view's `delivered` event and, for a node that was away, on the node-engine seat's `manifest-reads-changed`
// (Windows). Its state is `renames`, key `<module>.<to>`; what it says is `renamed` on the stream `config-events`.
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
)
// The state, the stream's events and the bounds this part keeps.
const (
RenamesState = "renames"
// EventRenamed is `renamed`, EventRetiring `retiring` and EventRetired `retired`, the family `config.events` of the
// seat's definition (design 26), under the holder's subjects: each carried by the stream StreamName, kept seven days.
EventRenamed = "renamed"
EventRetiring = "retiring"
EventRetired = "retired"
StreamName = "config-events"
// HeldFinding is how long a window or a retirement may stay open before it is a finding naming the node.
HeldFinding = 7 * 24 * time.Hour
)
// Rename is one rename, as `renames` keeps it, key `<module>.<to>`.
type Rename struct {
Module string `json:"module"`
From string `json:"from"`
To string `json:"to"`
At time.Time `json:"at"`
// Moved are the nodes a value was moved on, by the last move of each.
Moved []string `json:"moved,omitempty"`
// Nodes are the windows, by node.
Nodes map[string]Window `json:"nodes"`
// Said says `renamed` was emitted; false until the bus took it.
Said bool `json:"said"`
}
// Window is one node's rename window: open until Closed is set.
type Window struct {
Since time.Time `json:"since"`
Closed *time.Time `json:"closed,omitempty"`
// How is what closed it: "applied" (the node runs a build reading the new name), "unplaced" (the module left the
// node), or "by hand" (config_rename_close).
How string `json:"how,omitempty"`
}
// Open is the nodes whose window is open, sorted.
func (r Rename) Open() []string {
var out []string
for node, w := range r.Nodes {
if w.Closed == nil {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// renameKey is a rename's key in `renames`.
func renameKey(module, to string) string { return module + "." + to }
// RenamedEvent is the body of `renamed`.
type RenamedEvent struct {
Module string `json:"module"`
From string `json:"from"`
To string `json:"to"`
At time.Time `json:"at"`
}
// readRenames is every rename kept, by key.
func (c *Config) readRenames() (map[string]Rename, error) {
out := map[string]Rename{}
err := c.each(RenamesState, func(key string, raw []byte) error {
var r Rename
if json.Unmarshal(raw, &r) != nil || r.Module == "" || renameKey(r.Module, r.To) != key {
return nil // not one this module wrote
}
if r.Nodes == nil {
r.Nodes = map[string]Window{}
}
out[key] = r
return nil
})
return out, err
}
// moduleValueKey is the key of a module's own value on a node.
func moduleValueKey(node, module, key string) string {
return Level{Kind: LevelModule, Node: node, Module: module}.ValueKey(key)
}
// applyRenames moves the values of every rename the trunk's manifests declare, opens the windows of the nodes the
// module is placed on, and writes `renamed` once: run on each resolution, under the lock, before the resolved values
// are written. s.Values and s.Renames are brought to what it did, so the same resolution sees it. A rename it refuses
// is a finding (findings.go), cleared once it moved or `renamed-from` left the manifest. It answers a line for what it
// did or could not do.
func (c *Config) applyRenames(s *Sources) []string {
held, err := c.readRenames()
if err != nil {
return []string{"the renames cannot be read: " + err.Error()}
}
now := c.now()
var said []string
seen := map[string]bool{}
for _, module := range modulesOf(s.Assignments) {
trunk, err := c.trunkDeclarations(module)
if err != nil {
continue // unresolved, said by the resolution
}
for _, to := range sortedKeys(trunk.Config) {
decl := trunk.Config[to]
if decl.RenamedFrom == "" {
continue
}
key := renameKey(module, to)
seen[key] = true
said = append(said, c.rename(s, held, trunk, to, decl, now)...)
}
}
// A rename whose `renamed-from` left the trunk's manifest goes once every window is closed.
for key, r := range held {
if seen[key] {
continue
}
if len(r.Open()) > 0 {
s.Renames[key] = r
continue
}
if err := c.Store.Delete(RenamesState, key); err != nil {
said = append(said, fmt.Sprintf("the rename %s is not removed: %v", key, err))
continue
}
delete(s.Renames, key)
c.clearFinding(FindingRenameRefused, r.Module, r.To)
c.clearFinding(FindingWindowHeld, r.Module, r.To)
}
c.sweepRenameFindings(s, now)
return said
}
// rename judges and applies one declared rename.
func (c *Config) rename(s *Sources, held map[string]Rename, trunk ModuleDeclarations, to string, decl Declaration,
now time.Time) []string {
module, from := trunk.Module, decl.RenamedFrom
key := renameKey(module, to)
nodes := nodesPlaced(s, module)
var said []string
refuse := func(why string) []string {
c.setFinding(renameRefused(module, to, from, why, now))
if r, ok := held[key]; ok {
s.Renames[key] = r
}
return []string{fmt.Sprintf("the rename of %s's %s to %s is refused: %s", module, from, to, why)}
}
if _, both := trunk.Config[from]; both {
return refuse("the manifest declares both names")
}
if from == to {
return refuse("the new name is the old one")
}
trusted := decl.Trusted
if p, ok := s.Predefined[decl.Predefined]; ok && p.Trusted {
trusted = true
}
type move struct {
node string
old, new Value
hadNew bool
oldKey string
newKey string
}
var moves []move
for _, node := range nodes {
oldKey, newKey := moduleValueKey(node, module, from), moduleValueKey(node, module, to)
ov, hadOld := s.Values[oldKey]
if !hadOld {
continue
}
nv, hadNew := s.Values[newKey]
switch {
case hadNew && string(compact(nv.Value)) != string(compact(ov.Value)):
return refuse(fmt.Sprintf("on %s both names hold a value, and they differ (%s under the old name, %s under the new)",
node, ov.Value, nv.Value))
case trusted && ov.Approved == "":
return refuse(fmt.Sprintf("on %s the value under the old name was set without the operator's Approve, and "+
"the new name is trusted (a rename keeps a value's trust, novox/hq ADR 0348 §8)", node))
}
moves = append(moves, move{node: node, old: ov, new: nv, hadNew: hadNew, oldKey: oldKey, newKey: newKey})
}
r, had := held[key]
if !had {
r = Rename{Module: module, From: from, To: to, At: now, Nodes: map[string]Window{}}
}
changed := !had
for _, m := range moves {
if !m.hadNew {
if err := c.Store.Put(ValuesState, m.newKey, m.old); err != nil {
return refuse("the value on " + m.node + " is not written under the new name: " + err.Error())
}
}
if err := c.Store.Delete(ValuesState, m.oldKey); err != nil {
return refuse("the old name's value on " + m.node + " is not removed: " + err.Error())
}
s.Values[m.newKey] = m.old
delete(s.Values, m.oldKey)
lvl := Level{Kind: LevelModule, Node: m.node, Module: module}
_ = c.Store.Append(HistoryLog, m.newKey, Change{Action: "rename", Level: &lvl, Key: to, Value: m.old.Value,
Was: m.old.Value, By: Module, Via: Module, Why: "renamed from " + from + " (renamed-from)", At: now})
if !slices.Contains(r.Moved, m.node) {
r.Moved = append(r.Moved, m.node)
sort.Strings(r.Moved)
}
changed = true
said = append(said, fmt.Sprintf("%s's %s on %s moved to %s", module, from, m.node, to))
}
for _, node := range nodes {
if _, ok := r.Nodes[node]; !ok {
r.Nodes[node] = Window{Since: now}
changed = true
}
}
c.clearFinding(FindingRenameRefused, module, to)
if !r.Said {
if err := c.Store.Emit(EventRenamed, RenamedEvent{Module: module, From: from, To: to, At: r.At}); err != nil {
said = append(said, fmt.Sprintf("renamed for %s's %s is not said yet: %v", module, to, err))
} else {
r.Said, changed = true, true
}
}
if changed {
if err := c.Store.Put(RenamesState, key, r); err != nil {
said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err))
}
}
s.Renames[key] = r
return said
}
// nodesPlaced is the nodes a module is placed on, sorted.
func nodesPlaced(s *Sources, module string) []string {
var out []string
for _, a := range s.Assignments {
if a.Module == module {
out = append(out, a.Node)
}
}
sort.Strings(out)
return slices.Compact(out)
}
// resolveOldNames adds, for each rename with a window open on a node, the old name answering the new name's value on
// that node (design 32: until then mesh-config answers both names).
func resolveOldNames(s Sources, out map[string]Resolved) {
for _, key := range sortedKeys(s.Renames) {
r := s.Renames[key]
for _, node := range r.Open() {
nv, ok := out[ResolvedKey(node, r.Module, r.To)]
if !ok {
continue
}
old := nv
old.Key = r.From
out[ResolvedKey(node, r.Module, r.From)] = old
}
}
}
// ---- the windows --------------------------------------------------------------------------------------------
// Windows closes the rename windows (and the retirements waiting for nodes) that the delivery says are done, at start
// and on each `delivered`; for node, the one a returning node-engine's `manifest-reads-changed` names, it first asks
// the node-engine seat's `manifest-reads`: a node-engine that does not serve it states nothing and its window stays
// open (design 32). It answers a line for each window closed.
func (c *Config) Windows(node string) []string {
c.mu.Lock()
defer c.mu.Unlock()
said := c.closeWindows(node)
said = append(said, c.drainRetirements(node)...)
if len(said) > 0 {
if _, err := c.reconcileLocked(); err != nil {
said = append(said, "resolving after the windows: "+err.Error())
}
}
return said
}
// closeWindows is Windows' rename half; the caller holds the lock.
func (c *Config) closeWindows(only string) []string {
renames, err := c.readRenames()
if err != nil {
return []string{"the renames cannot be read: " + err.Error()}
}
if c.Delivery == nil {
return nil
}
s, err := c.sources()
if err != nil {
return []string{"the windows cannot be judged: " + err.Error()}
}
now := c.now()
var said []string
for _, key := range sortedKeys(renames) {
r := renames[key]
changed := false
for _, node := range r.Open() {
if only != "" && only != node {
continue
}
how, why := c.windowShut(s, r, node, only != "")
if why != "" {
said = append(said, fmt.Sprintf("%s's window for %s stays open on %s: %s", r.Module, r.To, node, why))
}
if how == "" {
continue
}
w := r.Nodes[node]
w.Closed, w.How = &now, how
r.Nodes[node] = w
changed = true
said = append(said, fmt.Sprintf("%s's window for %s is closed on %s (%s)", r.Module, r.To, node, how))
}
if !changed {
continue
}
if err := c.Store.Put(RenamesState, key, r); err != nil {
said = append(said, fmt.Sprintf("the rename %s is not kept: %v", key, err))
}
if len(r.Open()) == 0 {
c.clearFinding(FindingWindowHeld, r.Module, r.To)
}
}
return said
}
// windowShut says how a node's window may close now ("" when it may not) and, when it may not, why.
func (c *Config) windowShut(s Sources, r Rename, node string, returning bool) (how, why string) {
if !slices.Contains(nodesPlaced(&s, r.Module), node) {
return "unplaced", ""
}
if returning {
if err := c.Delivery.Reads(node); err != nil {
return "", "the node-engine states nothing yet (" + err.Error() + ")"
}
}
fp, found, err := c.Delivery.Have(node, r.Module)
switch {
case err != nil:
return "", "its have cannot be read: " + err.Error()
case !found || fp == "":
return "", "it reports no build of the module"
}
d, err := c.builtDeclarations(r.Module, fp)
if err != nil {
return "", "the build it runs cannot be read: " + err.Error()
}
if _, reads := d.Config[r.To]; !reads {
return "", "it still runs a build that reads " + r.From
}
return "applied", ""
}
// ---- the tools ---------------------------------------------------------------------------------------------
// RenameRow is one rename as the tools say it.
type RenameRow struct {
Module string `json:"module"`
From string `json:"from"`
To string `json:"to"`
At time.Time `json:"at"`
Moved []string `json:"moved,omitempty"`
// Open are the nodes holding the old name open; Closed the others, each with how.
Open []string `json:"open"`
Closed map[string]string `json:"closed,omitempty"`
}
// Renames lists the renames, the open ones only for the seat's verb, narrowed by module.
func (c *Config) Renames(module string, openOnly bool) ([]RenameRow, error) {
renames, err := c.readRenames()
if err != nil {
return nil, err
}
rows := []RenameRow{}
for _, key := range sortedKeys(renames) {
r := renames[key]
if module != "" && r.Module != module {
continue
}
row := RenameRow{Module: r.Module, From: r.From, To: r.To, At: r.At, Moved: r.Moved, Open: r.Open()}
if row.Open == nil {
row.Open = []string{}
}
if openOnly && len(row.Open) == 0 {
continue
}
for node, w := range r.Nodes {
if w.Closed != nil {
if row.Closed == nil {
row.Closed = map[string]string{}
}
row.Closed[node] = w.How
}
}
rows = append(rows, row)
}
return rows, nil
}
// RenameAgain moves a rename's values now, the way a resolution does, and answers what it did: the recovery for a
// refused move once its cause is put right.
func (c *Config) RenameAgain(module, to string) (any, error) {
c.mu.Lock()
defer c.mu.Unlock()
if module == "" || to == "" {
return nil, fmt.Errorf("name the module and the new name of the rename")
}
r, err := c.reconcileLocked()
if err != nil {
return r, err
}
renames, err := c.readRenames()
if err != nil {
return nil, err
}
if _, ok := renames[renameKey(module, to)]; !ok {
return nil, fmt.Errorf("no rename of %s to %s is kept: its trunk manifest declares no renamed-from for %s, or the rename is refused (config_renames and the findings say which)", module, to, to)
}
return map[string]any{"said": r.Renamed, "moved": r.Moved}, nil
}
// RenameClose drops a node from a rename's window by hand, recorded as a hand act.
func (c *Config) RenameClose(module, to, node, why string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("say why the window is dropped: a hand act keeps its why")
}
c.mu.Lock()
defer c.mu.Unlock()
key := renameKey(module, to)
renames, err := c.readRenames()
if err != nil {
return nil, err
}
r, ok := renames[key]
if !ok {
return nil, fmt.Errorf("no rename of %s to %s is kept", module, to)
}
w, ok := r.Nodes[node]
if !ok || w.Closed != nil {
return nil, fmt.Errorf("%s holds no open window for %s's %s; it holds: %s", node, module, to, strings.Join(r.Open(), ", "))
}
now := c.now()
w.Closed, w.How = &now, "by hand"
r.Nodes[node] = w
if err := c.Store.HandAct(fmt.Sprintf("config_rename_close %s %s on %s", module, to, node), why,
"the operator dropped a node from the rename window of "+key); err != nil {
return nil, fmt.Errorf("the hand-act log does not hold it, so the window is not dropped: %w", err)
}
if err := c.Store.Put(RenamesState, key, r); err != nil {
return nil, fmt.Errorf("the window is not dropped: %w", err)
}
if len(r.Open()) == 0 {
c.clearFinding(FindingWindowHeld, r.Module, r.To)
}
if _, err := c.reconcileLocked(); err != nil {
return nil, err
}
return map[string]any{"closed": node, "open": r.Open()}, nil
}
// sweepRenameFindings writes a finding for each window held open past HeldFinding, naming the node, and clears one
// whose window closed.
func (c *Config) sweepRenameFindings(s *Sources, now time.Time) {
for _, key := range sortedKeys(s.Renames) {
r := s.Renames[key]
var held []string
for _, node := range r.Open() {
if now.Sub(r.Nodes[node].Since) >= HeldFinding {
held = append(held, node)
}
}
c.clearFinding(FindingWindowHeld, r.Module, r.To, held...)
for _, node := range held {
c.setFinding(windowHeld(r, node, now))
}
}
}
@@ -0,0 +1,436 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/health"
)
// fakeDelivery is the delivery parts as mesh-config asks them: each node's have, whether a node-engine states what it
// reads, and who reads a module's keys (nil rows and known false: the indexer does not serve the tool yet).
type fakeDelivery struct {
mu sync.Mutex
haves map[string]string // "<node>.<module>" → fingerprint
notSays map[string]bool // node → its node-engine states nothing
rows []ReaderRow
known bool
}
func (d *fakeDelivery) Have(node, module string) (string, bool, error) {
d.mu.Lock()
defer d.mu.Unlock()
fp, ok := d.haves[node+"."+module]
return fp, ok, nil
}
func (d *fakeDelivery) Reads(node string) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.notSays[node] {
return errors.New("the verb manifest-reads is not served")
}
return nil
}
func (d *fakeDelivery) Readers(string) ([]ReaderRow, bool, error) {
d.mu.Lock()
defer d.mu.Unlock()
return d.rows, d.known, nil
}
// fakeVault records the retirements asked of the seat mesh-vault.
type fakeVault struct{ asked []RetireRequest }
func (v *fakeVault) Retire(r RetireRequest) error { v.asked = append(v.asked, r); return nil }
// The fingerprints of the first, second and third build of player.
var (
fp1 = "src2:" + strings.Repeat("1", 64)
fp2 = "src2:" + strings.Repeat("2", 64)
fp3 = "src2:" + strings.Repeat("3", 64)
)
const (
oldPlayer = `{"module":"player","config":{"volume":{"type":"number","default":5}}}`
newPlayer = `{"module":"player","config":{"level":{"type":"number","default":5,"renamed-from":"volume"}}}`
// donePlayer is the manifest once `renamed-from` left it.
donePlayer = `{"module":"player","config":{"level":{"type":"number","default":5}}}`
)
// aRenamingMesh is a mesh with player on g14 and novox, both running its first build (reading `volume`), a clock the
// test moves, and the delivery parts faked.
func aRenamingMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *time.Time) {
t.Helper()
m := newStore()
m.declare(t, "player", oldPlayer)
indexerList(t, m, "aaaaaaaaaaaa", theList)
for _, node := range []string{"g14", "novox"} {
m.assign(t, node, "player")
m.want(t, node, "player", fp1, oldPlayer)
}
clock := time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC)
d := &fakeDelivery{haves: map[string]string{"g14.player": fp1, "novox.player": fp1}, notSays: map[string]bool{}}
c := &Config{Store: m, Now: func() time.Time { return clock }, Logf: t.Logf, Delivery: d}
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
return c, m, d, &clock
}
func findingsOf(t *testing.T, m *memStore, kind string) []health.Finding {
t.Helper()
var out []health.Finding
for _, k := range m.keysOf(FindingsState) {
e, _ := m.Get(FindingsState, k)
f, err := health.DecodeFinding(e.Value)
if err != nil {
t.Fatalf("finding %s: %v", k, err)
}
if f.Kind == kind {
out = append(out, f)
}
}
return out
}
// **A renamed value moves with its trust, both names answer until the node-engines run the new build, and the window
// closes per node** (design 32, C20): the value set under the old name is under the new one with no operator input,
// `renamed` is said once, the old name answers the new value on a node until that node's have runs a build that reads the
// new name (on `delivered`, or for a returning node on `manifest-reads-changed` once its node-engine states what it
// reads), and the rename goes once every window is closed and `renamed-from` left the manifest.
func TestARenamedValueMovesAndTheOldNameAnswersUntilEachNodeRunsTheNewBuild(t *testing.T) {
c, m, d, _ := aRenamingMesh(t)
set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"})
m.declare(t, "player", newPlayer)
r, err := c.Reconcile()
if err != nil || len(r.Renamed) != 1 {
t.Fatalf("reconcile: %+v, %v", r, err)
}
if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" || v.By == "" {
t.Errorf("the value is %+v under the new name (set: %v)", v, ok)
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); ok {
t.Error("the old name still holds a value")
}
for _, c := range []struct{ node, key, want string }{{"g14", "level", "8"}, {"g14", "volume", "8"}, {"novox", "level", "5"}, {"novox", "volume", "5"}} {
if r := m.resolved(t, c.node, "player", c.key); string(r.Value) != c.want {
t.Errorf("%s answers %s on %s as %s, want %s", c.key, "player", c.node, r.Value, c.want)
}
}
if got := m.saidOf(EventRenamed); len(got) != 1 || got[0].(RenamedEvent).From != "volume" || got[0].(RenamedEvent).To != "level" {
t.Errorf("renamed was said as %+v", got)
}
// The history keeps the move; a second pass says and moves nothing more.
if h, _ := c.History(Call{Module: "player", Node: "g14"}); len(h) == 0 || h[len(h)-1].Action != "resolve" && h[len(h)-1].Action != "rename" {
t.Errorf("history %+v", h)
}
if _, err := c.Reconcile(); err != nil || len(m.saidOf(EventRenamed)) != 1 {
t.Errorf("a second pass said renamed %d times: %v", len(m.saidOf(EventRenamed)), err)
}
rows, _ := c.Renames("", true)
if len(rows) != 1 || strings.Join(rows[0].Open, ",") != "g14,novox" || strings.Join(rows[0].Moved, ",") != "g14" {
t.Fatalf("renames %+v", rows)
}
// A delivery while both nodes still run the first build closes nothing.
if said := c.Windows(""); len(said) != 2 || !strings.Contains(said[0], "still runs a build that reads volume") {
t.Errorf("windows said %v", said)
}
// g14 applies the new build.
m.want(t, "g14", "player", fp2, newPlayer)
d.haves["g14.player"] = fp2
if said := c.Windows(""); len(said) == 0 {
t.Error("g14 applied the new build and its window did not close")
}
if _, err := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); err != nil {
t.Fatal(err)
} else if e, _ := m.Get(ResolvedState, ResolvedKey("g14", "player", "volume")); e != nil {
t.Error("g14 still answers the old name after its window closed")
}
m.resolved(t, "novox", "player", "volume") // novox still answers it
// novox applies it, but its node-engine, returning, states nothing yet: the window stays open.
m.want(t, "novox", "player", fp2, newPlayer)
d.haves["novox.player"] = fp2
d.notSays["novox"] = true
c.Windows("novox")
if rows, _ := c.Renames("", true); len(rows) != 1 || strings.Join(rows[0].Open, ",") != "novox" {
t.Errorf("a node-engine that states nothing closed its window: %+v", rows)
}
d.notSays["novox"] = false
c.Windows("novox")
if rows, _ := c.Renames("", true); len(rows) != 0 {
t.Errorf("windows still open: %+v", rows)
}
// The rename stays while `renamed-from` is in the manifest, and goes when it left.
if all, _ := c.Renames("", false); len(all) != 1 || all[0].Closed["g14"] != "applied" {
t.Errorf("renames %+v", all)
}
m.declare(t, "player", donePlayer)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
if all, _ := c.Renames("", false); len(all) != 0 {
t.Errorf("the rename stayed after renamed-from left the manifest: %+v", all)
}
}
// A refused move is a finding, cleared once it moved (C4): both names declared, both names holding different values, a
// trusted new name over a value set without the operator's Approve. Nothing is moved, nothing is said.
func TestARefusedRenameIsAFindingAndIsClearedOnceItMoves(t *testing.T) {
both := `{"module":"player","config":{"volume":{"type":"number","default":5},"level":{"type":"number","default":5,"renamed-from":"volume"}}}`
trusted := `{"module":"player","config":{"level":{"type":"number","default":5,"trusted":true,"renamed-from":"volume"}}}`
for _, tc := range []struct {
name, manifest string
both bool
why string
fixed string
}{
{"both names declared", both, false, "declares both names", newPlayer},
{"both names hold a value", newPlayer, true, "both names hold a value", newPlayer},
{"a trusted name over an unapproved value", trusted, false, "without the operator's Approve", newPlayer},
} {
t.Run(tc.name, func(t *testing.T) {
c, m, _, _ := aRenamingMesh(t)
set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"})
if tc.both {
// The new name is set meanwhile, to another value, in the state a faulty hand wrote it.
if err := m.Put(ValuesState, "module.g14.player.level", Value{Value: json.RawMessage("9"), By: "x", At: time.Now()}); err != nil {
t.Fatal(err)
}
}
m.declare(t, "player", tc.manifest)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
fs := findingsOf(t, m, FindingRenameRefused)
if len(fs) != 1 || !strings.Contains(fs[0].Summary, tc.why) || fs[0].Observer != Module {
t.Fatalf("findings %+v, want one saying %q", fs, tc.why)
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok || len(m.saidOf(EventRenamed)) != 0 {
t.Errorf("a refused rename moved the value or said renamed")
}
// Put right, it moves by hand and the finding goes.
m.declare(t, "player", tc.fixed)
if tc.both {
_ = m.Delete(ValuesState, "module.g14.player.level")
}
if _, err := c.RenameAgain("player", "level"); err != nil {
t.Fatal(err)
}
if fs := findingsOf(t, m, FindingRenameRefused); len(fs) != 0 {
t.Errorf("the finding stayed after the move: %+v", fs)
}
if v, ok := storedValue(t, m, "module.g14.player.level"); !ok || string(v.Value) != "8" {
t.Errorf("the value is %+v after the move", v)
}
})
}
}
// A window held open seven days is a finding naming the node, cleared when it closes; dropping a node by hand is a hand
// act with its why (C4, C5), and the seat's verb `renames` names the nodes holding a window open.
func TestAWindowHeldSevenDaysIsAFindingAndAHandCloseIsAHandAct(t *testing.T) {
c, m, d, clock := aRenamingMesh(t)
m.declare(t, "player", newPlayer)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
m.want(t, "g14", "player", fp2, newPlayer)
d.haves["g14.player"] = fp2
c.Windows("")
*clock = clock.Add(8 * 24 * time.Hour)
if _, err := c.Reconcile(); err != nil {
t.Fatal(err)
}
fs := findingsOf(t, m, FindingWindowHeld)
if len(fs) != 1 || fs[0].Subject.Machine != "novox" || !strings.Contains(fs[0].Summary, "novox") {
t.Fatalf("findings %+v, want one naming novox", fs)
}
// The seat's verb names the node holding the window open.
var verb func(map[string]any) (any, error)
for _, tool := range Tools(c) {
if tool.Name == Seat+"."+RenamesVerb {
verb = tool.Run
}
}
got, err := verb(map[string]any{"module": "player"})
if raw, _ := json.Marshal(got); err != nil || !strings.Contains(string(raw), `"open":["novox"]`) {
t.Errorf("renames answered %s, %v", raw, err)
}
if _, err := c.RenameClose("player", "level", "novox", ""); err == nil {
t.Error("a window was dropped with no why")
}
if _, err := c.RenameClose("player", "level", "g14", "g14 is closed already"); err == nil {
t.Error("a node holding no open window was dropped")
}
m.handActs = nil
if _, err := c.RenameClose("player", "level", "novox", "novox is retired for good"); err != nil {
t.Fatal(err)
}
if len(m.handActs) != 1 || !strings.Contains(m.handActs[0], "novox is retired for good") {
t.Errorf("hand acts %v", m.handActs)
}
if fs := findingsOf(t, m, FindingWindowHeld); len(fs) != 0 {
t.Errorf("the finding stayed after the window closed: %+v", fs)
}
if rows, _ := c.Renames("", true); len(rows) != 0 {
t.Errorf("open renames %+v", rows)
}
}
// retiringMesh is a mesh where player no longer declares `volume`, a value of it is set on g14, and g14 runs the first
// build (which reads it).
func retiringMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *standIn, *fakeVault, *time.Time) {
t.Helper()
c, m, d, clock := aRenamingMesh(t)
set(t, c, Call{Key: "volume", Value: 8.0, Module: "player", Node: "g14"})
m.declare(t, "player", donePlayer)
r := newStandIn()
v := &fakeVault{}
c.Asks, c.Vault = r, v
d.known = true
d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{}}}
return c, m, d, r, v, clock
}
func retiringOf(t *testing.T, c *Config) []RetiringRow {
t.Helper()
got, err := c.ConfigRetiring("", "", "")
if err != nil {
t.Fatal(err)
}
return got.(map[string]any)["retiring"].([]RetiringRow)
}
// **A value no manifest reads is retired only on the operator's Approve** (C24): while a node runs a build that reads it
// it is not marked and nobody is asked; once none does, `retiring` is said and the operator is asked once; the value stays
// until the Approve, then it is removed, `retired` is said for the node it was set on, and the warrant is recorded.
func TestAValueNoManifestReadsIsRetiredOnlyOnTheOperatorsApprove(t *testing.T) {
c, m, d, r, _, _ := retiringMesh(t)
// The indexer does not serve the tool yet: nothing retires on that.
d.known = false
if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "does not say") || len(retiringOf(t, c)) != 0 {
t.Errorf("an indexer without the tool: %v", said)
}
d.known = true
// g14 still runs a build that reads volume: draining, no ask, no event.
c.Retirements("player")
if rows := retiringOf(t, c); len(rows) != 1 || rows[0].State != RetireDraining || strings.Join(rows[0].Nodes, ",") != "g14,novox" {
t.Fatalf("retiring %+v, want draining on g14 and novox", rows)
}
if len(r.asks) != 0 || len(m.saidOf(EventRetiring)) != 0 {
t.Fatalf("asked %d times and said retiring %d times while two nodes read the value", len(r.asks), len(m.saidOf(EventRetiring)))
}
// Both nodes apply the build that reads it no longer.
for _, node := range []string{"g14", "novox"} {
m.want(t, node, "player", fp3, donePlayer)
d.haves[node+".player"] = fp3
}
c.Windows("")
rows := retiringOf(t, c)
if len(rows) != 1 || rows[0].State != RetireAsked || len(r.asks) != 1 || len(m.saidOf(EventRetiring)) != 1 {
t.Fatalf("retiring %+v, %d asks, %d events", rows, len(r.asks), len(m.saidOf(EventRetiring)))
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); !ok {
t.Fatal("the value was removed before the operator answered")
}
// A second manifest-accepted asks nobody again.
c.Retirements("player")
if len(r.asks) != 1 {
t.Errorf("asked %d times", len(r.asks))
}
r.choose(t, rows[0].Ask, OptionApprove, nil)
m.handActs = nil
if said, err := c.Decide(); err != nil || len(said) == 0 {
t.Fatalf("decide: %v, %v", said, err)
}
if _, ok := storedValue(t, m, "module.g14.player.volume"); ok {
t.Error("the approved value is still set")
}
if got := m.saidOf(EventRetired); len(got) != 1 || got[0].(RetiredEvent).Node != "g14" || got[0].(RetiredEvent).Name != "volume" {
t.Errorf("retired was said as %+v", got)
}
if len(retiringOf(t, c)) != 0 || len(m.handActs) == 0 {
t.Errorf("retiring %+v, hand acts %v", retiringOf(t, c), m.handActs)
}
}
// A declined retirement is not asked again until the value is read again; a value read again is dropped; an ask unanswered
// is asked once more after seven days and then written as a finding, cleared by an answer or a cancel; a secret is
// retired through the seat mesh-vault with the ask that warranted it (C24, C7).
func TestARetirementDeclinedUnansweredCancelledOrASecret(t *testing.T) {
c, m, d, r, v, clock := retiringMesh(t)
for _, node := range []string{"g14", "novox"} {
m.want(t, node, "player", fp3, donePlayer)
d.haves[node+".player"] = fp3
}
c.Retirements("player")
ask := retiringOf(t, c)[0].Ask
// Unanswered: the ask expires, a week passes, it is asked once more, expires again: a finding.
r.choose(t, ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired })
c.Decide()
if rows := retiringOf(t, c); rows[0].State != RetireUnanswered || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 {
t.Fatalf("after the first expiry: %+v", rows)
}
*clock = clock.Add(8 * 24 * time.Hour)
c.Decide()
rows := retiringOf(t, c)
if rows[0].State != RetireAsked || len(r.asks) != 2 || rows[0].Ask == ask {
t.Fatalf("after a week: %+v, %d asks", rows, len(r.asks))
}
r.choose(t, rows[0].Ask, OptionApprove, func(w *asks.Warrant) { w.Outcome = asks.OutcomeExpired })
c.Decide()
if fs := findingsOf(t, m, FindingRetireUnanswered); len(fs) != 1 {
t.Fatalf("findings %+v after two asks unanswered", fs)
}
// Asked again by hand, then declined: the finding goes and it is not asked again.
if _, err := c.ConfigRetiring("ask-again", "player", "volume"); err != nil {
t.Fatal(err)
}
third := retiringOf(t, c)[0].Ask
r.choose(t, third, OptionDecline, nil)
c.Decide()
if rows := retiringOf(t, c); rows[0].State != RetireDeclined || len(findingsOf(t, m, FindingRetireUnanswered)) != 0 {
t.Fatalf("after the decline: %+v", rows)
}
asked := len(r.asks)
c.Retirements("player")
if len(r.asks) != asked {
t.Error("a declined retirement was asked again")
}
// Read again: dropped.
d.rows = []ReaderRow{{Module: "player", Key: "volume", Kind: KindValue, Readers: []string{"player"}}}
if said := c.Retirements("player"); len(said) != 1 || !strings.Contains(said[0], "read again") || len(retiringOf(t, c)) != 0 {
t.Errorf("read again: %v", said)
}
// A secret: its nodes come from the indexer's answer; on the Approve the vault is asked for each, once.
d.rows = []ReaderRow{{Module: "player", Key: "token", Kind: KindSecret, Readers: []string{}, Nodes: []string{}}}
c.Retirements("player")
secret := retiringOf(t, c)[0]
if secret.Kind != KindSecret || secret.State != RetireAsked {
t.Fatalf("the secret: %+v", secret)
}
r.choose(t, secret.Ask, OptionApprove, nil)
c.Decide()
if len(v.asked) != 2 || v.asked[0].Node != "g14" || v.asked[0].Name != "token" || v.asked[0].Ask != secret.Ask {
t.Errorf("the vault was asked %+v", v.asked)
}
if len(m.saidOf(EventRetired)) != 2 || len(retiringOf(t, c)) != 0 {
t.Errorf("retired %d times, retiring %+v", len(m.saidOf(EventRetired)), retiringOf(t, c))
}
// A secret whose readers' nodes the indexer cannot name is not marked.
d.rows = []ReaderRow{{Module: "player", Key: "other", Kind: KindSecret, Readers: []string{}}}
if c.Retirements("player"); len(retiringOf(t, c)) != 0 {
t.Errorf("a secret read on unknown nodes was marked: %+v", retiringOf(t, c))
}
_ = fmt.Sprint
}
@@ -150,6 +150,8 @@ type Sources struct {
// for them (pulls.go); neither is resolved.
Pulls map[string]PullDecl
Kept map[string]KeptValue
// Renames are the renames kept, by `<module>.<to>`, whose open windows answer the old name (renames.go).
Renames map[string]Rename
}
// Unresolved is a module placed on a node whose values are not resolved, and why.
@@ -192,6 +194,7 @@ func Resolve(s Sources) map[string]Resolved {
out[ResolvedKey(a.Node, a.Module, key)] = r
}
}
resolveOldNames(s, out)
return out
}
@@ -0,0 +1,581 @@
package main
// A value no longer read (novox/hq ADR 0609, design 32 *A value no longer read*).
//
// A config value or secret that no manifest on any trunk reads any more is a **retiring value**. mesh-config learns it
// from the indexer's tool `indexer_config_readers`, asked on each of the indexer's `manifest-accepted` (Retirements);
// a tool the indexer does not serve yet answers "not known" and nothing retires. A value is marked retiring only once no
// node-engine reports applied a build that reads it (State draining until then, judged again on each `delivered`);
// mesh-config then says `retiring` and asks the operator once, through the operator channel, whether to remove it. It is
// removed only on the operator's Approve: a value by mesh-config, a secret through the seat `mesh-vault`'s verb
// `retire`. An ask unanswered after seven days is asked once more, and then written as a finding.
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// The state, and where a retirement stands.
const (
RetiringState = "retiring"
// RetireVerb is the act an answer binds.
RetireVerb = Seat + ".retire"
RetireDraining = "draining" // a node still runs a build that reads it
RetireAsked = "asked" // the operator is asked
RetireUnanswered = "unanswered" // the ask ended with no choice
RetireApproved = "approved" // the operator approved; being removed
RetireDeclined = "declined" // the operator declined, or a person cancelled: kept until the value is read again
)
// Kinds of retiring value.
const (
KindValue = "value"
KindSecret = "secret"
)
// Retiring is one retiring value, as `retiring` keeps it, key `<module>.<key>`.
type Retiring struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
State string `json:"state"`
// Nodes are the nodes that still read it (draining) or that it is removed on (asked, approved).
Nodes []string `json:"nodes,omitempty"`
// Ask is the open ask's id, Asked when the first ask was made, AskedAgain whether it was asked once more.
Ask string `json:"ask,omitempty"`
// AskedAt is when the open ask was made (it expires a day on), Asked when the first was.
AskedAt time.Time `json:"asked-at,omitempty"`
Asked time.Time `json:"asked,omitempty"`
AskedAgain bool `json:"asked-again,omitempty"`
Said bool `json:"said,omitempty"`
Outcome string `json:"outcome,omitempty"`
Answered string `json:"answered,omitempty"`
// Removed are the nodes it is removed on already.
Removed []string `json:"removed,omitempty"`
}
func retiringKey(module, key string) string { return module + "." + key }
// RetiringEvent is the body of `retiring`; RetiredEvent of `retired`.
type RetiringEvent struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
Ask string `json:"ask"`
}
// RetiredEvent is the body of `retired`: one node's value removed, with the ask that warranted it.
type RetiredEvent struct {
Module string `json:"module"`
Name string `json:"name"`
Node string `json:"node"`
Ask string `json:"ask"`
}
// ReaderRow is one key the indexer's `indexer_config_readers` answers: the modules on each trunk that declare or refer
// to it. For a secret, Nodes are the nodes whose applied build reads it, when the indexer can say.
type ReaderRow struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Readers []string `json:"readers"`
Nodes []string `json:"nodes,omitempty"`
}
// Delivery is what the delivery parts answer to this module: a node's have, a node-engine's statement of what it
// reads, and who reads a module's config keys. Each is a tool or a seat verb on the bus, never a bucket (ADR 0596).
type Delivery interface {
// Have is the fingerprint of the build node runs of module, as the status view's node_haves says it.
Have(node, module string) (fp string, found bool, err error)
// Reads asks the node-engine seat's manifest-reads for node: an error when it is not served or the node says nothing.
Reads(node string) error
// Readers asks the indexer who reads module's keys: known false when the indexer does not serve the tool yet.
Readers(module string) (rows []ReaderRow, known bool, err error)
}
// RetireRequest is what the vault's verb `retire` takes: one module's own secret on a node, with the hand act that
// warrants it.
type RetireRequest struct {
Node, Module, Name, Ask string
}
// Vault is the seat mesh-vault as this module reaches it.
type Vault interface {
Retire(r RetireRequest) error
}
func (c *Config) readRetiring() (map[string]Retiring, error) {
out := map[string]Retiring{}
err := c.each(RetiringState, func(key string, raw []byte) error {
var r Retiring
if json.Unmarshal(raw, &r) != nil || r.Module == "" || retiringKey(r.Module, r.Key) != key {
return nil
}
out[key] = r
return nil
})
return out, err
}
// Retirements judges a module's keys on the indexer's word that its manifest was accepted. It answers a line for each
// decision.
func (c *Config) Retirements(module string) []string {
c.mu.Lock()
defer c.mu.Unlock()
if c.Delivery == nil || module == "" {
return nil
}
rows, known, err := c.Delivery.Readers(module)
if err != nil {
return []string{fmt.Sprintf("who reads %s's config is not asked yet: %v", module, err)}
}
if !known {
return []string{fmt.Sprintf("the indexer does not say who reads %s's config yet; nothing retires on that", module)}
}
held, err := c.readRetiring()
if err != nil {
return []string{"the retirements cannot be read: " + err.Error()}
}
s, err := c.sources()
if err != nil {
return []string{"the retirements cannot be judged: " + err.Error()}
}
var said []string
for _, row := range rows {
if row.Module != "" && row.Module != module {
continue
}
row.Module = module
if row.Kind == "" {
row.Kind = KindValue
}
key := retiringKey(module, row.Key)
e, has := held[key]
if len(row.Readers) > 0 {
if has {
said = append(said, c.dropRetiring(e, "it is read again"))
}
continue
}
if has {
continue // already marked; it is advanced by drainRetirements and Decide
}
if row.Kind == KindValue && !valueHeld(s, module, row.Key) {
continue // nothing set under that name: nothing to remove
}
if row.Kind == KindSecret && row.Nodes == nil && !row.nodesKnown() {
continue // which nodes read a secret is not known, and nothing retires on that
}
e = Retiring{Module: module, Key: row.Key, Kind: row.Kind, Since: c.now(), State: RetireDraining, Nodes: row.Nodes}
said = append(said, c.advanceRetiring(s, &e)...)
}
return said
}
func (r ReaderRow) nodesKnown() bool { return r.Nodes != nil }
// valueHeld is whether a module's own value is set under key on any node.
func valueHeld(s Sources, module, key string) bool {
for k := range s.Values {
if l, kk, ok := parseValueKey(k); ok && l.Kind == LevelModule && l.Module == module && kk == key {
return true
}
}
return false
}
// nodesReading is the nodes whose applied build still reads a value (secrets: the nodes the indexer last named).
func (c *Config) nodesReading(s Sources, e Retiring) []string {
if e.Kind != KindValue || c.Delivery == nil {
return e.Nodes
}
var out []string
for _, node := range nodesPlaced(&s, e.Module) {
fp, found, err := c.Delivery.Have(node, e.Module)
if err != nil || !found || fp == "" {
if err != nil {
out = append(out, node) // not known is not "no longer read"
}
continue
}
d, err := c.builtDeclarations(e.Module, fp)
if err != nil {
out = append(out, node)
continue
}
if _, reads := d.Config[e.Key]; reads {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// advanceRetiring moves a draining retirement on: while a node reads the value it stays draining (and is written, so a
// delivery judges it again); once none does it is marked retiring, said, and asked. The caller holds the lock.
func (c *Config) advanceRetiring(s Sources, e *Retiring) []string {
key := retiringKey(e.Module, e.Key)
now := c.now()
if e.State == RetireDraining {
e.Nodes = c.nodesReading(s, *e)
if len(e.Nodes) > 0 {
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)}
}
if now.Sub(e.Since) >= HeldFinding {
c.setFinding(retirementHeld(*e, now))
}
return []string{fmt.Sprintf("%s's %s is not read by any manifest, and %s still run a build that reads it",
e.Module, e.Key, strings.Join(e.Nodes, ", "))}
}
c.clearFinding(FindingRetireHeld, e.Module, e.Key)
e.Nodes = nodesPlaced(&s, e.Module)
if err := c.ask(e, false); err != nil {
return []string{fmt.Sprintf("the operator is not asked about retiring %s's %s: %v", e.Module, e.Key, err)}
}
}
if !e.Said {
if err := c.Store.Emit(EventRetiring, RetiringEvent{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since,
Ask: e.Ask}); err == nil {
e.Said = true
}
}
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)}
}
return []string{fmt.Sprintf("%s's %s is retiring: the operator is asked whether to remove it", e.Module, e.Key)}
}
// drainRetirements judges the draining retirements again, after a delivery or a returning node; the caller holds the lock.
func (c *Config) drainRetirements(string) []string {
held, err := c.readRetiring()
if err != nil || len(held) == 0 || c.Delivery == nil {
return nil
}
s, err := c.sources()
if err != nil {
return []string{"the retirements cannot be judged: " + err.Error()}
}
var said []string
for _, key := range sortedKeys(held) {
e := held[key]
if e.State == RetireDraining {
said = append(said, c.advanceRetiring(s, &e)...)
}
}
return said
}
// dropRetiring ends a retirement: the value is read again, or a person cancelled it.
func (c *Config) dropRetiring(e Retiring, why string) string {
if err := c.Store.Delete(RetiringState, retiringKey(e.Module, e.Key)); err != nil {
return fmt.Sprintf("the retirement of %s's %s is not dropped: %v", e.Module, e.Key, err)
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
c.clearFinding(FindingRetireHeld, e.Module, e.Key)
return fmt.Sprintf("%s's %s is no longer retiring: %s", e.Module, e.Key, why)
}
// ---- the ask ------------------------------------------------------------------------------------------------
// retireAskID is an ask's id: the same retirement asked again is another id.
func retireAskID(e Retiring, again bool) string {
sum := sha256.Sum256([]byte(e.Module + "\x00" + e.Key + "\x00" + e.Since.UTC().Format(time.RFC3339Nano)))
id := "ret-" + hex.EncodeToString(sum[:6])
if again {
id += "-2"
}
return id
}
// act is what an answer binds: the retirement, the nodes it reaches, the ask and the answer.
func (e Retiring) act(ask, option string) asks.Act {
return asks.Act{"verb": RetireVerb, "option": option, "module": e.Module, "key": e.Key, "kind": e.Kind,
"nodes": strings.Join(e.Nodes, " "), "ask": ask}
}
// retireAsk is the ask composed from the retirement alone.
func (e Retiring) retireAsk(id string, at time.Time) asks.Ask {
nodes := strings.Join(e.Nodes, ", ")
if nodes == "" {
nodes = "no node"
}
what := "config value"
if e.Kind == KindSecret {
what = "secret"
}
a := asks.Ask{ID: id, Headline: fmt.Sprintf("Remove %s's %s %s?", e.Module, what, e.Key),
Explanation: fmt.Sprintf("No manifest reads %s's %s %s any more, and no node runs a build that reads it. Approved, it is removed on %s.", e.Module, what, e.Key, nodes),
Who: asks.Operator, Expires: at.Add(ProposalLasts), OnExpiry: "nothing is removed; mesh-config asks once more in a week",
About: "config.retire." + e.Module + "." + e.Key,
Details: fmt.Sprintf("Retiring since %s UTC. Read it with mesh-config's config_retiring.", e.Since.UTC().Format("15:04 on 2 Jan"))}
for _, o := range []struct{ id, label, does string }{
{OptionApprove, "Approve", "the " + what + " is removed on " + nodes},
{OptionDecline, "Decline", "nothing is removed; mesh-config asks no more until it is read again"},
} {
binds, _ := asks.ActDigest(e.act(id, o.id))
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
}
return a
}
// ask makes the ask for a retirement and publishes it; the caller keeps the entry.
func (c *Config) ask(e *Retiring, again bool) error {
if c.Asks == nil {
return fmt.Errorf("the operator cannot be asked from here")
}
now := c.now()
id := retireAskID(*e, again)
a := e.retireAsk(id, now)
if err := a.Check(now); err != nil {
return err
}
if err := c.Asks.Publish(asks.AskSubject(Module), a, a.ID); err != nil {
return err
}
e.Ask, e.State, e.AskedAt = id, RetireAsked, now
if e.Asked.IsZero() {
e.Asked = now
}
e.AskedAgain = e.AskedAgain || again
return nil
}
// decideRetirements takes the router's word on each open retirement ask; the caller holds the lock. It also asks once
// more an ask unanswered for seven days, writes the finding of one asked twice, and finishes an approved removal.
func (c *Config) decideRetirements() []string {
held, err := c.readRetiring()
if err != nil || len(held) == 0 {
return nil
}
now := c.now()
var said []string
for _, key := range sortedKeys(held) {
e := held[key]
switch e.State {
case RetireAsked:
said = append(said, c.decideRetirement(&e, now)...)
case RetireUnanswered:
if !e.AskedAgain && now.Sub(e.Asked) >= HeldFinding {
if err := c.ask(&e, true); err != nil {
said = append(said, fmt.Sprintf("the operator is not asked again about %s's %s: %v", e.Module, e.Key, err))
} else {
_ = c.Store.Put(RetiringState, key, e)
said = append(said, fmt.Sprintf("%s's %s: the operator is asked once more", e.Module, e.Key))
}
}
case RetireApproved:
said = append(said, c.finishRetirement(&e)...)
}
}
return said
}
func (c *Config) decideRetirement(e *Retiring, now time.Time) []string {
if c.Asks == nil {
return nil
}
key := retiringKey(e.Module, e.Key)
rec, err := c.Asks.Record(e.Ask)
if err != nil || rec == nil {
return nil
}
var r routerRecord
if json.Unmarshal(rec.Value, &r) != nil || r.State == "open" || r.Warrant == nil {
return nil
}
w := *r.Warrant
keep := func(state, outcome string) []string {
e.State, e.Outcome = state, outcome
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("%s's %s: %s, and that is not kept: %v", e.Module, e.Key, outcome, err)}
}
return []string{fmt.Sprintf("%s's %s: %s", e.Module, e.Key, outcome)}
}
if w.Asker != Module || w.Ask != e.Ask {
return keep(RetireUnanswered, "the router's record is for another ask; nothing is removed")
}
if w.Outcome != asks.OutcomeChosen {
out := keep(RetireUnanswered, "the ask "+string(w.Outcome)+"; nothing is removed")
if e.AskedAgain {
c.setFinding(retireUnanswered(*e, now))
}
return out
}
option, err := w.For(Module, e.retireAsk(e.Ask, e.AskedAt))
if err == nil {
err = option.Performs(e.act(e.Ask, option.ID))
}
if err != nil {
c.logf("[mesh-config] REFUSED the answer to the retirement ask %s: %v", e.Ask, err)
return keep(RetireUnanswered, "the answer is refused and nothing is removed: "+err.Error())
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
e.Answered = w.Says()
switch option.ID {
case OptionDecline:
return keep(RetireDeclined, "declined by the operator ("+w.Says()+"); nothing is removed")
case OptionApprove:
out := keep(RetireApproved, "approved by the operator; being removed")
return append(out, c.finishRetirement(e)...)
}
return keep(RetireUnanswered, "the option "+option.ID+" is not one a retirement offers")
}
// finishRetirement removes an approved retirement on each node it is not removed on yet: a value here, a secret through
// the vault. Each removal is said (`retired`); the entry goes when every node is done. The caller holds the lock.
func (c *Config) finishRetirement(e *Retiring) []string {
key := retiringKey(e.Module, e.Key)
var said, failed []string
for _, node := range e.Nodes {
if slices.Contains(e.Removed, node) {
continue
}
var err error
switch e.Kind {
case KindValue:
var removed bool
if removed, err = c.removeValue(*e, node); err == nil && !removed {
e.Removed = append(e.Removed, node) // nothing was set there
continue
}
default:
if c.Vault == nil {
err = fmt.Errorf("the seat mesh-vault is not reachable from here")
} else {
err = c.Vault.Retire(RetireRequest{Node: node, Module: e.Module, Name: e.Key, Ask: e.Ask})
}
}
if err != nil {
failed = append(failed, node+": "+err.Error())
continue
}
e.Removed = append(e.Removed, node)
if err := c.Store.Emit(EventRetired, RetiredEvent{Module: e.Module, Name: e.Key, Node: node, Ask: e.Ask}); err != nil {
c.logf("[mesh-config] retired for %s's %s on %s is not said: %v", e.Module, e.Key, node, err)
}
said = append(said, fmt.Sprintf("%s's %s is removed on %s", e.Module, e.Key, node))
}
if len(failed) > 0 {
e.Outcome = "not removed everywhere yet: " + strings.Join(failed, "; ")
_ = c.Store.Put(RetiringState, key, *e)
return append(said, fmt.Sprintf("%s's %s: %s", e.Module, e.Key, e.Outcome))
}
if err := c.Store.Delete(RetiringState, key); err != nil {
return append(said, fmt.Sprintf("the retirement %s is not dropped: %v", key, err))
}
if _, err := c.reconcileLocked(); err != nil {
said = append(said, "resolving after the removal: "+err.Error())
}
return said
}
// removeValue deletes a module's own value on a node, recording it in the history, the hand-act log and config-changed.
func (c *Config) removeValue(e Retiring, node string) (bool, error) {
vk := moduleValueKey(node, e.Module, e.Key)
got, err := c.Store.Get(ValuesState, vk)
if err != nil {
return false, err
}
if got == nil {
return false, nil // nothing set there, or gone already
}
var was Value
_ = json.Unmarshal(got.Value, &was)
if err := c.Store.Delete(ValuesState, vk); err != nil {
return false, err
}
lvl := Level{Kind: LevelModule, Node: node, Module: e.Module}
ch := Change{Action: "retire", Level: &lvl, Key: e.Key, Was: was.Value, By: "the operator", Via: Module,
Why: "no manifest reads it any more (retiring), removed on the operator's Approve of ask " + e.Ask, At: c.now(),
Approved: e.Answered}
c.record(vk, ch)
if err := c.Store.HandAct(handActWhat(ch), ch.Why, "the operator's Approve of ask "+e.Ask); err != nil {
c.logf("[mesh-config] the hand-act log does not hold the retirement of %s: %v", vk, err)
}
return true, nil
}
// ---- the tool -----------------------------------------------------------------------------------------------
// RetiringRow is one retirement as the tool says it.
type RetiringRow struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
State string `json:"state"`
Nodes []string `json:"nodes,omitempty"`
Ask string `json:"ask,omitempty"`
Outcome string `json:"outcome,omitempty"`
Answered string `json:"answered,omitempty"`
}
// ConfigRetiring lists what is retiring (module narrows), or with action ask-again asks the operator again about one,
// or with cancel drops one (kept apart so it is not asked again until the value is read again).
func (c *Config) ConfigRetiring(action, module, key string) (any, error) {
c.mu.Lock()
defer c.mu.Unlock()
held, err := c.readRetiring()
if err != nil {
return nil, err
}
switch action {
case "", "list":
rows := []RetiringRow{}
for _, k := range sortedKeys(held) {
e := held[k]
if module != "" && e.Module != module {
continue
}
rows = append(rows, RetiringRow{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since, State: e.State,
Nodes: e.Nodes, Ask: e.Ask, Outcome: e.Outcome, Answered: e.Answered})
}
return map[string]any{"retiring": rows}, nil
case "ask-again", "cancel":
e, ok := held[retiringKey(module, key)]
if !ok {
return nil, fmt.Errorf("%s's %s is not retiring: config_retiring lists what is", module, key)
}
if action == "cancel" {
e.State, e.Outcome = RetireDeclined, "cancelled by hand"
if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil {
return nil, err
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
if err := c.Store.HandAct("config_retiring cancel "+module+" "+key, "a retirement cancelled by hand", "config_retiring"); err != nil {
c.logf("[mesh-config] the hand-act log does not hold the cancel: %v", err)
}
return map[string]any{"said": module + "'s " + key + " is no longer asked about until it is read again"}, nil
}
if e.State == RetireDraining || e.State == RetireApproved {
return nil, fmt.Errorf("%s's %s is %s: there is nothing to ask again", module, key, e.State)
}
s, err := c.sources()
if err != nil {
return nil, err
}
if nodes := c.nodesReading(s, e); len(nodes) > 0 {
return nil, fmt.Errorf("%s still run a build that reads %s's %s", strings.Join(nodes, ", "), module, key)
}
if err := c.ask(&e, true); err != nil {
return nil, err
}
if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil {
return nil, err
}
return map[string]any{"said": "the operator is asked again", "ask": e.Ask}, nil
}
return nil, fmt.Errorf("config_retiring lists, or does ask-again or cancel; %q is none of them", action)
}
+15 -1
View File
@@ -17,7 +17,7 @@ import (
// from its own manifests as built (issue 1953, ADR 0565 §3); the controller's records are no longer read.
func (c *Config) sources() (Sources, error) {
s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{},
Predefined: map[string]Published{}, Values: map[string]Value{}}
Predefined: map[string]Published{}, Values: map[string]Value{}, Renames: map[string]Rename{}}
var problems []string
if err := c.readPlacement(&s); err != nil {
return s, fmt.Errorf("the seat mesh-placement's placement cannot be read: %w", err)
@@ -51,6 +51,11 @@ func (c *Config) sources() (Sources, error) {
if err := c.readPulls(&s); err != nil {
return s, err
}
if renames, err := c.readRenames(); err != nil {
return s, fmt.Errorf("the renames cannot be read: %w", err)
} else {
s.Renames = renames
}
for _, p := range problems {
c.logf("[mesh-config] a placed module's declarations are not read, so its values are not resolved (they "+
"stay as last resolved): %s", p)
@@ -354,6 +359,8 @@ type Resolution struct {
// did after the take-over.
Declared []string `json:"declared,omitempty"`
Candidates []string `json:"candidates,omitempty"`
// Renamed is what the renames did in this pass.
Renamed []string `json:"renamed,omitempty"`
// SourceGone is said while the stored switch still names the controller, whose records are no longer read.
SourceGone string `json:"source_gone,omitempty"`
}
@@ -364,6 +371,11 @@ type Resolution struct {
func (c *Config) Reconcile() (Resolution, error) {
c.mu.Lock()
defer c.mu.Unlock()
return c.reconcileLocked()
}
// reconcileLocked is Reconcile with the lock held.
func (c *Config) reconcileLocked() (Resolution, error) {
var r Resolution
published, err := c.publish()
if err != nil {
@@ -374,6 +386,8 @@ func (c *Config) Reconcile() (Resolution, error) {
if err != nil {
return r, err
}
// The renames move their values first, so the same pass resolves them under the new names (renames.go).
r.Renamed = c.applyRenames(&s)
moved, err := c.writeResolved(s)
r.Moved = moved
if len(moved) > 0 {
@@ -11,12 +11,18 @@ import (
// The switch's tools.
const (
DeclaredTool = "config_declared"
ResyncTool = "config_resync"
DeclaredTool = "config_declared"
ResyncTool = "config_resync"
RenamesTool = "config_renames"
RenameAgainTool = "config_rename_again"
RenameCloseTool = "config_rename_close"
RetiringTool = "config_retiring"
RenamesVerb = "renames"
)
// SeatVerbs are the seat's verbs, as the manifest claims them.
var SeatVerbs = []string{Verb}
// SeatVerbs are the seat's verbs, as the manifest claims them: `config`, and the optional `renames` the checker asks
// (design 32: the open renames and the nodes holding each window open).
var SeatVerbs = []string{Verb, RenamesVerb}
func switchTools(c *Config) []stdio.Tool { return ownSwitchTools(c) }
@@ -35,5 +41,48 @@ func ownSwitchTools(c *Config) []stdio.Tool {
Description: "Read every source again and write what differs: the published list of predefined keys, the " +
"resolved values, declared, and the indexer's config candidates.",
Run: func(map[string]any) (any, error) { return c.Resync() }},
{Name: RenamesTool,
Description: "Each config value renamed (renamed-from): from which name to which, when, the nodes its value " +
"moved on, the nodes still holding the old name open (the rename window) and how each closed one closed. " +
"Narrowed by module.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "one module"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
rows, err := c.Renames(strings.TrimSpace(m), false)
return map[string]any{"renames": rows}, err
}},
{Name: RenameAgainTool,
Description: "Move a renamed value by hand, the way a resolution does: after a refused move (its finding says " +
"why) once the cause is put right, or for a value set under the old name meanwhile.",
Input: map[string]any{"type": "object", "required": []string{"module", "to"}, "properties": map[string]any{
"module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"}}},
Run: func(a map[string]any) (any, error) {
m, _ := a["module"].(string)
to, _ := a["to"].(string)
return c.RenameAgain(strings.TrimSpace(m), strings.TrimSpace(to))
}},
{Name: RenameCloseTool,
Description: "Drop a node from a rename's window by hand (a node retired for good), recorded as a hand act: " +
"the old name is no longer answered for it.",
Input: map[string]any{"type": "object", "required": []string{"module", "to", "node", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string"}, "to": map[string]any{"type": "string", "description": "the new name"},
"node": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}},
Run: func(a map[string]any) (any, error) {
str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) }
return c.RenameClose(str("module"), str("to"), str("node"), str("why"))
}},
{Name: RetiringTool,
Description: "The config values and secrets no manifest reads any more: since when, where each stands (draining " +
"while a node still runs a build that reads it, asked, unanswered, approved, declined) and its ask. With " +
"action ask-again, ask the operator once more about one; with cancel, drop one (it is asked about no more " +
"until it is read again).",
Input: map[string]any{"type": "object", "properties": map[string]any{
"action": map[string]any{"type": "string", "enum": []string{"list", "ask-again", "cancel"}},
"module": map[string]any{"type": "string"}, "key": map[string]any{"type": "string"}}},
Run: func(a map[string]any) (any, error) {
str := func(k string) string { v, _ := a[k].(string); return strings.TrimSpace(v) }
return c.ConfigRetiring(str("action"), str("module"), str("key"))
}},
}
}
@@ -0,0 +1,790 @@
//go:build integration
package main
// mesh-config whole against a real bus (design 32, C20 and C24): the built binary, played over its real stdin and stdout
// as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real nats-server's key-value buckets,
// watches handed over as the runtime hands them, mesh/event delivering the events it subscribed to, mesh/ask answered
// as the status view, the indexer, the node-engine seat and mesh-hand-acts answer it, mesh/seat.* as the router does).
//
// A renamed value moves with no input, keeps its value, and both names answer until the node-engine reports the new
// build on `delivered`; the module restarted inside the window keeps the window and says `renamed` once; a refused move
// writes its finding. A value no manifest reads is not asked about while a node reads it, is asked once when none does,
// stays until the operator's Approve and is then removed, `retired` said.
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/meshconfig"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
type wire struct {
ID json.RawMessage `json:"id,omitempty"`
Method string `json:"method,omitempty"`
Params json.RawMessage `json:"params,omitempty"`
Result json.RawMessage `json:"result,omitempty"`
Error *struct {
Message string `json:"message"`
} `json:"error,omitempty"`
}
// player is the node's tool runtime, as far as mesh-config needs it, over a real bus.
type player struct {
t *testing.T
ctx context.Context
js jetstream.JetStream
router *standIn
mu sync.Mutex
in io.Writer
next int
pending map[string]chan wire
// haves is the status view's node_haves, by "<node>.<module>"; readers the indexer's answer.
haves map[string]string
readers []ReaderRow
said []saidOnBus
subs []string
// owned is each delivery owner's state as the owner's tool answers it: its values and the revision of its last change.
owned map[string]*ownedState
}
// ownedState is one state of an owner (the indexer's modules, the assigner's wants...), kept from its bucket.
type ownedState struct {
owner, state string
revision uint64
values map[string]json.RawMessage
}
var theCreated = time.Date(2026, 11, 4, 9, 0, 0, 0, time.UTC)
// own follows the bucket of an owner's state, and says each change after the first listing on the owner's views family
// to the module (once it subscribed), as the owner does (ADR 0615 §4).
func (p *player) own(read string) {
p.t.Helper()
owner, state, _ := strings.Cut(read, ".")
kv, err := p.kv(read)
if err != nil {
p.t.Fatal(err)
}
w, err := kv.WatchAll(p.ctx)
if err != nil {
p.t.Fatal(err)
}
st := &ownedState{owner: owner, state: state, values: map[string]json.RawMessage{}}
p.owned[read] = st
listed := make(chan struct{})
go func() {
current := true
for e := range w.Updates() {
if e == nil {
current = false
close(listed)
continue
}
p.mu.Lock()
deleted := e.Operation() != jetstream.KeyValuePut
if deleted {
delete(st.values, e.Key())
} else {
st.values[e.Key()] = append(json.RawMessage(nil), e.Value()...)
}
if !current {
st.revision++
ev := map[string]any{"v": 1, "owner": owner, "state": state, "key": e.Key(), "revision": st.revision, "created": theCreated}
if deleted {
ev["deleted"] = true
} else {
ev["value"] = json.RawMessage(e.Value())
}
p.next++
id := fmt.Sprintf("o-%d", p.next)
body, _ := json.Marshal(ev)
raw, _ := json.Marshal(map[string]any{"id": id, "jsonrpc": "2.0", "method": "mesh/event", "params": map[string]any{
"envelope": map[string]any{"key": owner + ".views." + state, "node": "novox", "body": json.RawMessage(body)}}})
_, _ = p.in.Write(append(raw, '\n'))
}
p.mu.Unlock()
}
}()
select {
case <-listed:
case <-time.After(10 * time.Second):
p.t.Fatalf("%s was not listed", read)
}
}
// ownerAnswer is an owner's read tool asked with values: the state whole at its revision.
func (p *player) ownerAnswer(key string) (any, bool) {
for read, st := range p.owned {
if key == st.owner+"."+toolOf(st) {
_ = read
return map[string]any{"v": 1, "owner": st.owner, "state": st.state, "revision": st.revision, "created": theCreated,
"values": st.values, "at": time.Now()}, true
}
}
return nil, false
}
func toolOf(st *ownedState) string {
prefix := "indexer_read_"
if st.owner == "mesh-delivery-assigner" {
prefix = "assigner_read_"
}
return prefix + strings.ReplaceAll(st.state, "-", "_")
}
type saidOnBus struct {
Key string
Body json.RawMessage
}
func (p *player) send(m map[string]any) {
raw, _ := json.Marshal(m)
p.mu.Lock()
defer p.mu.Unlock()
_, _ = p.in.Write(append(raw, '\n'))
}
func (p *player) answer(id json.RawMessage, result any, err error) {
if err != nil {
p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}})
return
}
p.send(map[string]any{"id": id, "result": result})
}
func (p *player) serve(out io.Reader) {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 1<<20), 1<<24)
for scanner.Scan() {
var m wire
if json.Unmarshal(scanner.Bytes(), &m) != nil {
continue
}
if m.Method == "" {
var key string
_ = json.Unmarshal(m.ID, &key)
p.mu.Lock()
ch := p.pending[key]
delete(p.pending, key)
p.mu.Unlock()
if ch != nil {
ch <- m
}
continue
}
go p.handle(m)
}
}
func (p *player) kv(state string) (jetstream.KeyValue, error) {
if owner, name, read := strings.Cut(state, "."); read {
return p.js.KeyValue(p.ctx, owner+"_"+name)
}
return p.js.KeyValue(p.ctx, Module+"_"+state)
}
func (p *player) handle(m wire) {
var a struct {
State string `json:"state"`
Key string `json:"key"`
Value json.RawMessage `json:"value"`
Revision uint64 `json:"revision"`
Subject string `json:"subject"`
Body json.RawMessage `json:"body"`
Pattern string `json:"pattern"`
ID string `json:"id"`
Bucket string `json:"bucket"`
}
_ = json.Unmarshal(m.Params, &a)
kv, kvErr := p.kv(a.State)
switch m.Method {
case "mesh/state.get":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
e, err := kv.Get(p.ctx, a.Key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
p.answer(m.ID, nil, nil)
return
}
if err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil)
case "mesh/state.create", "mesh/state.update", "mesh/state.put":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
var rev uint64
var err error
switch m.Method {
case "mesh/state.create":
rev, err = kv.Create(p.ctx, a.Key, a.Value)
case "mesh/state.update":
rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision)
default:
rev, err = kv.Put(p.ctx, a.Key, a.Value)
}
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}})
return
}
p.answer(m.ID, map[string]any{"revision": rev}, err)
case "mesh/state.keys":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
keys, err := kv.Keys(p.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
keys, err = []string{}, nil
}
p.answer(m.ID, keys, err)
case "mesh/state.delete":
err := kvErr
if err == nil {
err = kv.Delete(p.ctx, a.Key)
}
p.answer(m.ID, map[string]any{}, err)
case "mesh/state.watch":
if kvErr != nil {
p.answer(m.ID, nil, kvErr)
return
}
w, err := kv.WatchAll(p.ctx)
if err != nil {
p.answer(m.ID, nil, err)
return
}
var watchID string
_ = json.Unmarshal(m.ID, &watchID)
current := true
for e := range w.Updates() {
if e == nil {
current = false
p.answer(m.ID, map[string]any{}, nil)
continue
}
op := "put"
if e.Operation() != jetstream.KeyValuePut {
op = "delete"
}
p.mu.Lock()
p.next++
id := fmt.Sprintf("w-%d", p.next)
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID,
"change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())),
"revision": e.Revision(), "current": current}}})
}
case "mesh/log.append":
p.answer(m.ID, map[string]any{"sequence": 1}, nil)
case "mesh/log.read":
p.answer(m.ID, map[string]any{"entries": []any{}, "next": 0, "last": 0}, nil)
case "mesh/publish":
var e struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &e)
p.mu.Lock()
p.said = append(p.said, saidOnBus{e.Key, e.Body})
p.mu.Unlock()
p.answer(m.ID, map[string]any{}, nil)
case "mesh/subscribe":
p.mu.Lock()
p.subs = append(p.subs, a.Pattern)
p.mu.Unlock()
p.answer(m.ID, map[string]any{}, nil)
case "mesh/seat.publish":
var ask asks.Ask
if err := json.Unmarshal(a.Body, &ask); err != nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, map[string]any{"sequence": 1}, p.router.Publish(a.Subject, ask, a.ID))
case "mesh/seat.record":
e, err := p.router.Record(a.Key)
if e == nil {
p.answer(m.ID, nil, err)
return
}
p.answer(m.ID, e, err)
case "mesh/seat.kinds":
p.answer(m.ID, p.router.kinds, nil)
case "mesh/ask":
var q struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &q)
p.answer(m.ID, p.asked(q.Key, q.Body), nil)
default:
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
}
}
// asked answers a tool or a seat verb mesh-config asks, as its holder does.
func (p *player) asked(key string, body json.RawMessage) any {
p.mu.Lock()
defer p.mu.Unlock()
if a, ok := p.ownerAnswer(key); ok {
return a
}
switch {
case key == HavesAsk:
var q struct{ Node, Module string }
_ = json.Unmarshal(body, &q)
hk, _ := delivery.HaveKey(q.Node, q.Module)
keys := map[string]any{}
if fp, ok := p.haves[q.Node+"."+q.Module]; ok {
keys[hk] = delivery.Have{V: delivery.Version, FP: fp, Phase: "applied", Health: "healthy"}
}
return map[string]any{"keys": keys}
case strings.HasPrefix(key, ManifestReadsAsk):
return map[string]any{}
case key == ConfigReadersAsk:
return map[string]any{"readers": p.readers}
case key == "seat:mesh-hand-acts.hand-act" || key == "seat:mesh-hand-acts.warranted":
return map[string]any{"said": "recorded"}
}
return map[string]any{"error": "no responders available for " + key}
}
func nullIfEmpty(b []byte) []byte {
if len(b) == 0 {
return []byte("null")
}
return b
}
func (p *player) call(name string, args map[string]any) (json.RawMessage, error) {
p.mu.Lock()
p.next++
id := fmt.Sprintf("c-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args,
"_meta": map[string]any{"mesh/caller": "person.jochen"}}})
select {
case m := <-ch:
if m.Error != nil {
return nil, errors.New(m.Error.Message)
}
var r struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
IsError bool `json:"isError"`
}
if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 {
return nil, fmt.Errorf("%s answered %s", name, m.Result)
}
if r.IsError {
return nil, errors.New(r.Content[0].Text)
}
return json.RawMessage(r.Content[0].Text), nil
case <-time.After(60 * time.Second):
return nil, fmt.Errorf("%s did not answer in 60s", name)
}
}
// event hands the module one event it subscribed to, as the runtime does, and waits for its answer.
func (p *player) event(key string, body any) {
p.t.Helper()
raw, _ := json.Marshal(body)
p.mu.Lock()
p.next++
id := fmt.Sprintf("e-%d", p.next)
ch := make(chan wire, 1)
p.pending[id] = ch
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{"key": key, "node": "novox",
"body": json.RawMessage(raw)}}})
select {
case m := <-ch:
if m.Error != nil {
p.t.Fatalf("%s was not taken: %s", key, m.Error.Message)
}
case <-time.After(30 * time.Second):
p.t.Fatalf("%s was not answered in 30s", key)
}
}
// conforms decodes what the module published under key strictly into the seat definition's type (a field the type does
// not know is a fault here, though a reader drops it) and checks the subject the runtime publishes it on is the one the
// definition names: CAT-V4, the definition's fixtures matched by a real event.
func (p *player) conforms(key string, into any, subject func(holder string) string) {
p.t.Helper()
p.mu.Lock()
defer p.mu.Unlock()
found := false
for _, s := range p.said {
if s.Key != key {
continue
}
found = true
dec := json.NewDecoder(strings.NewReader(string(s.Body)))
dec.DisallowUnknownFields()
if err := dec.Decode(into); err != nil {
p.t.Errorf("the published %s %s does not match the seat's definition: %v", key, s.Body, err)
}
if got, want := "mesh.mod."+Module+".event."+s.Key, subject(Module); got != want {
p.t.Errorf("%s is published on %s, the definition names %s", key, got, want)
}
}
if !found {
p.t.Errorf("%s was never published", key)
}
}
func (p *player) publishedCount(key string) int {
p.mu.Lock()
defer p.mu.Unlock()
n := 0
for _, s := range p.said {
if s.Key == key {
n++
}
}
return n
}
func (p *player) put(state, key string, v any) {
p.t.Helper()
kv, err := p.kv(state)
if err != nil {
p.t.Fatal(err)
}
raw, _ := json.Marshal(v)
if _, err := kv.Put(p.ctx, key, raw); err != nil {
p.t.Fatal(err)
}
}
func (p *player) get(state, key string) ([]byte, bool) {
kv, err := p.kv(state)
if err != nil {
return nil, false
}
e, err := kv.Get(p.ctx, key)
if err != nil {
return nil, false
}
return e.Value(), true
}
// viewsReady waits until every first view of the module is answered (config_first_views).
func (p *player) viewsReady() {
p.t.Helper()
eventually(p.t, "every first view is answered", func() bool {
raw, err := p.call(FirstViewsTool, nil)
if err != nil {
return false
}
var a struct {
Views []struct {
Revision uint64 `json:"revision"`
Answered time.Time `json:"Answered"`
} `json:"first-views"`
}
if json.Unmarshal(raw, &a) != nil || len(a.Views) != len(followed) {
return false
}
for _, v := range a.Views {
if v.Answered.IsZero() {
return false
}
}
return true
})
}
func eventually(t *testing.T, what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if ok() {
return
}
}
t.Fatalf("never: %s", what)
}
// manifestIn writes a module's manifest into the indexer's buckets as the indexer writes it, and names it as the trunk's.
func (p *player) manifestIn(module, raw string) string {
p.t.Helper()
digest := delivery.ManifestDigest([]byte(raw))
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
if err != nil {
p.t.Fatal(err)
}
p.put("mesh-delivery-indexer.manifests", digest, json.RawMessage(value))
p.put("mesh-delivery-indexer.modules", module, map[string]any{"v": 1, "manifest": digest, "repository": "novox/" + module, "commit": "c1"})
return digest
}
// buildIn writes a build of a module with a manifest, and the want that names it on a node.
func (p *player) buildIn(node, module, fp, raw string) {
p.t.Helper()
digest := delivery.ManifestDigest([]byte(raw))
value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
p.put("mesh-delivery-indexer.manifests", digest, json.RawMessage(value))
bk, _ := delivery.BuilderBuildKey(module, fp)
p.put("mesh-delivery-builder.builds", bk, map[string]any{"v": 1, "state": "done", "manifest": digest})
p.put("mesh-delivery-assigner.wants", "want."+node+"."+module, map[string]any{"v": 1, "fp": fp, "self": true})
}
func (p *player) placed(node string, modules ...string) {
p.t.Helper()
pl := delivery.Placement{V: delivery.Version, Node: node}
for _, m := range modules {
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
}
values, err := delivery.PlacementValues(pl)
if err != nil {
p.t.Fatal(err)
}
for k, raw := range values {
p.put("mesh-delivery-assigner.placement", k, json.RawMessage(raw))
}
}
func resolvedValue(p *player, node, module, key string) (string, bool) {
raw, ok := p.get(ResolvedState, ResolvedKey(node, module, key))
if !ok {
return "", false
}
var r Resolved
_ = json.Unmarshal(raw, &r)
return string(r.Value), true
}
// start runs the built binary on the bus and serves it as the runtime.
func start(t *testing.T, bin string, js jetstream.JetStream, router *standIn, haves map[string]string, readers []ReaderRow) *player {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module)
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
p := &player{t: t, ctx: ctx, js: js, router: router, in: stdin, pending: map[string]chan wire{}, haves: haves, readers: readers,
owned: map[string]*ownedState{}}
for _, f := range followed {
p.own(f.Read)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
go p.serve(stdout)
t.Cleanup(cancel)
return p
}
// 1966/C9 (design 32, C20 and C24): the module whole on a real bus.
func TestConfigWholeRenamesAndRetiresAgainstTheBus(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
bin := filepath.Join(t.TempDir(), Module)
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("mesh-config does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
buckets := []string{"mesh-delivery-assigner_placement", "mesh-delivery-assigner_wants", "mesh-delivery-builder_builds",
"mesh-delivery-indexer_modules", "mesh-delivery-indexer_manifests", "mesh-delivery-indexer_predefined",
"mesh-delivery-indexer_config-candidates"}
for _, s := range []string{ValuesState, ResolvedState, PredefinedState, ProposalsState, PullsState, KeptState, DeclaredState,
RenamesState, RetiringState, FindingsState} {
buckets = append(buckets, Module+"_"+s)
}
for _, b := range buckets {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil {
t.Fatal(err)
}
}
router := newStandIn()
haves := map[string]string{"g14.player": fp1, "novox.player": fp1, "g14.radio": fp1}
p := start(t, bin, js, router, haves, nil)
p.viewsReady()
// The mesh: player and radio on g14, player also on novox, all on their first build; dock declares both names.
const oldRadio = `{"module":"radio","config":{"band":{"type":"number","default":1}}}`
const newRadio = `{"module":"radio","config":{"gain":{"type":"number","default":1}}}`
const dock = `{"module":"dock","config":{"a":{"type":"number","default":1},"b":{"type":"number","default":1,"renamed-from":"a"}}}`
p.manifestIn("player", oldPlayer)
p.manifestIn("radio", oldRadio)
p.buildIn("g14", "player", fp1, oldPlayer)
p.buildIn("novox", "player", fp1, oldPlayer)
p.buildIn("g14", "radio", fp1, oldRadio)
p.placed("g14", "player", "radio")
p.placed("novox", "player")
if _, err := p.call(ResyncTool, nil); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second) // the watches start after the first call
set := func(module, key string, value any) {
t.Helper()
if _, err := p.call(Seat+"."+Verb, map[string]any{"action": "set", "module": module, "node": "g14", "key": key,
"value": value, "why": "the operator asked for it"}); err != nil {
t.Fatal(err)
}
}
set("player", "volume", 8)
set("radio", "band", 3)
// C20: the new manifest renames volume to level; the value moves, both names answer.
p.manifestIn("player", newPlayer)
eventually(t, "the value moves to the new name", func() bool {
raw, ok := p.get(ValuesState, "module.g14.player.level")
return ok && strings.Contains(string(raw), `"value":8`)
})
if _, ok := p.get(ValuesState, "module.g14.player.volume"); ok {
t.Error("the old name still holds a value")
}
eventually(t, "both names answer on g14", func() bool {
a, okA := resolvedValue(p, "g14", "player", "level")
b, okB := resolvedValue(p, "g14", "player", "volume")
return okA && okB && a == "8" && b == "8"
})
eventually(t, "renamed is said", func() bool { return p.publishedCount(EventRenamed) == 1 })
p.conforms(EventRenamed, &meshconfig.RenamedHolderEvent{}, meshconfig.HolderSubjectRenamed)
// A refused move writes its finding: dock declares both names.
p.manifestIn("dock", dock)
p.buildIn("g14", "dock", fp1, dock)
p.placed("g14", "player", "radio", "dock")
eventually(t, "the refused rename's finding", func() bool {
keys, _ := (func() ([]string, error) { kv, _ := p.kv(FindingsState); return kv.Keys(p.ctx) })()
for _, k := range keys {
raw, _ := p.get(FindingsState, k)
if f, err := health.DecodeFinding(raw); err == nil && f.Kind == FindingRenameRefused && strings.Contains(f.Summary, "dock") {
return true
}
}
return false
})
// The module restarted inside the window keeps it, and says renamed no more.
said := p.publishedCount(EventRenamed)
cancelOld := p.ctx
_ = cancelOld
p2 := start(t, bin, js, router, haves, nil)
p2.viewsReady()
if _, err := p2.call(ResyncTool, nil); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
if v, ok := resolvedValue(p2, "g14", "player", "volume"); !ok || v != "8" {
t.Errorf("after the restart the old name answers %q (%v), want 8", v, ok)
}
if n := p2.publishedCount(EventRenamed); n != 0 || said != 1 {
t.Errorf("renamed was said %d times before the restart and %d after", said, n)
}
raw, _ := p2.get(RenamesState, "player.level")
var rn Rename
if json.Unmarshal(raw, &rn) != nil || strings.Join(rn.Open(), ",") != "g14,novox" {
t.Fatalf("the window after the restart: %s", raw)
}
// The seat's verb answers the open window in the definition's shape (CAT-C10).
raw, err = p2.call(Seat+"."+RenamesVerb, map[string]any{"module": "player"})
if err != nil {
t.Fatal(err)
}
var open meshconfig.RenamesAnswer
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&open); err != nil || len(open.Renames) != 1 || open.Renames[0].To != "level" ||
strings.Join(open.Renames[0].Open, ",") != "g14,novox" {
t.Fatalf("renames answered %s (%v), want player's level open on g14 and novox", raw, err)
}
// g14 applies the build that reads the new name; the delivery says so; its window closes, novox's stays.
p2.buildIn("g14", "player", fp2, newPlayer)
p2.mu.Lock()
p2.haves["g14.player"] = fp2
p2.mu.Unlock()
p2.event("mesh-delivery-view.delivered", map[string]any{"revision": "r1"})
eventually(t, "g14's window closes and the old name goes", func() bool {
_, old := resolvedValue(p2, "g14", "player", "volume")
v, still := resolvedValue(p2, "novox", "player", "volume")
return !old && still && v == "5"
})
// C24: radio's manifest drops band; g14 still runs the build that reads it: no ask. Then it runs the next: one ask.
p2.manifestIn("radio", newRadio)
p2.mu.Lock()
p2.readers = []ReaderRow{{Module: "radio", Key: "band", Kind: KindValue, Readers: []string{}}}
p2.mu.Unlock()
p2.event("mesh-delivery-indexer.manifest-accepted", map[string]any{"module": "radio"})
eventually(t, "the retirement waits on g14", func() bool {
raw, ok := p2.get(RetiringState, "radio.band")
return ok && strings.Contains(string(raw), RetireDraining)
})
if len(router.asks) != 0 || p2.publishedCount(EventRetiring) != 0 {
t.Fatalf("asked %d times, retiring said %d times while g14 reads the value", len(router.asks), p2.publishedCount(EventRetiring))
}
p2.buildIn("g14", "radio", fp2, newRadio)
p2.mu.Lock()
p2.haves["g14.radio"] = fp2
p2.mu.Unlock()
p2.event("mesh-delivery-view.delivered", map[string]any{"revision": "r2"})
eventually(t, "the operator is asked once", func() bool {
router.mu.Lock()
defer router.mu.Unlock()
return len(router.asks) == 1 && p2.publishedCount(EventRetiring) == 1
})
if _, ok := p2.get(ValuesState, "module.g14.radio.band"); !ok {
t.Fatal("the value was removed before the operator answered")
}
var id string
router.mu.Lock()
for k := range router.asks {
id = k
}
router.mu.Unlock()
w := router.choose(t, id, OptionApprove, nil)
p2.event(asks.Seat+".decided."+Module, w)
eventually(t, "the approved value is removed and retired is said", func() bool {
_, still := p2.get(ValuesState, "module.g14.radio.band")
return !still && p2.publishedCount(EventRetired) == 1
})
p2.conforms(EventRetiring, &meshconfig.RetiringHolderEvent{}, meshconfig.HolderSubjectRetiring)
p2.conforms(EventRetired, &meshconfig.RetiredHolderEvent{}, meshconfig.HolderSubjectRetired)
eventually(t, "the retirement is dropped", func() bool { _, ok := p2.get(RetiringState, "radio.band"); return !ok })
}
+12 -2
View File
@@ -2,13 +2,23 @@ module mesh-config
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
require (
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+13
View File
@@ -1,7 +1,17 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
@@ -10,5 +20,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+51 -14
View File
@@ -7,12 +7,16 @@
"name": "mesh-config",
"scope": "mesh",
"serves": [
"config"
"config",
"renames"
]
}
],
"emits": [
"config-changed"
"config-changed",
"renamed",
"retiring",
"retired"
],
"uses": [
"operator-channel"
@@ -20,25 +24,44 @@
"consumes": [
"gitea.pull.merged",
"gitea.pull.closed",
"gitea.forge.pull-request.closed"
"gitea.forge.pull-request.closed",
"mesh-delivery-view.delivered",
"node-engine.manifest-reads-changed",
"mesh-delivery-indexer.manifest-accepted",
"mesh-delivery-indexer.views.modules",
"mesh-delivery-indexer.views.manifests",
"mesh-delivery-indexer.views.predefined",
"mesh-delivery-indexer.views.config-candidates",
"mesh-delivery-assigner.views.placement",
"mesh-delivery-assigner.views.wants"
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.warranted",
"gitea.gitea_get_pull_request"
"gitea.gitea_get_pull_request",
"mesh-delivery-view.node_haves",
"seat:node-engine.manifest-reads",
"mesh-delivery-indexer.indexer_config_readers",
"seat:mesh-vault.retire",
"mesh-delivery-indexer.indexer_read_modules",
"mesh-delivery-indexer.indexer_read_manifests",
"mesh-delivery-indexer.indexer_read_predefined",
"mesh-delivery-indexer.indexer_read_config_candidates",
"mesh-delivery-assigner.assigner_read_placement",
"mesh-delivery-assigner.assigner_read_wants"
],
"reads": [
"mesh-delivery-assigner.placement",
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.predefined",
"mesh-delivery-indexer.config-candidates"
"mesh-delivery-builder.builds"
],
"tools": [
"config_declared",
"config_resync"
"config_resync",
"config_renames",
"config_rename_again",
"config_rename_close",
"config_retiring",
"config_first_views",
"config_first_view_again"
],
"state": [
"values",
@@ -47,7 +70,10 @@
"proposals",
"pulls",
"kept",
"declared"
"declared",
"renames",
"retiring",
"findings"
],
"logs": [
"history"
@@ -87,5 +113,16 @@
]
}
]
}
},
"streams": [
{
"name": "config-events",
"events": [
"renamed",
"retiring",
"retired"
],
"max-age-days": 7
}
]
}
+9
View File
@@ -229,6 +229,15 @@ restart; it is on until switched off, and `own_drift` shows it.
- `provisioner.retirement` (ADR 0230 §35) — `waiting`, `settled`, `approved`, `rejected`, `retired`,
`reenabled`, `deleted`, naming holders, never a value.
## Retiring an own secret: `retire` (design 32)
`retire` removes one module's own secret on a node because no manifest reads it any more, and says `own-retired`
(node, module, name, ask, caller). Two proofs: the caller is on the vault's trusted list `retire-callers` (the holder of
the seat `mesh-config` alone, a file the mesh writes from the config value, read at start; a list that cannot be read
allows nobody), and `warrant` is the id of a hand act that caller recorded through `mesh-hand-acts` for this verb,
naming exactly the node, module, name and the ask of the operator's Approve, within a day. A bus account and the tool
runner's `ledgers` are never retired this way. A waiter on `rotated` or `own-replaced` does not wake on it.
## The tools
`secret_export`, `secret_holders`, `secret_verify` answer as the TypeScript vault's did, and its custody
@@ -128,6 +128,23 @@ func enrolmentTools(env *Env) []toolDef {
}
return v.AbandonRotation(caller, arg(a, "node"))
}},
{Name: Seat + "." + RetireVerb,
Description: "Retire one module's own secret on a node because no manifest reads it any more (design 32): removed " +
"whole, and own-retired said (node, module, name, the ask, the caller; never a value). Answered to the holder of " +
"the seat mesh-config alone, as the vault's list retire-callers names it, and only for a hand act it recorded " +
"through mesh-hand-acts for this verb naming exactly this node, module and name and the ask of the operator's Approve " +
"(`warrant`, the act's id), read back here.",
Input: map[string]any{"node": node, "module": str("the module whose secret it is"), "name": str("the secret's name"),
"warrant": str("the operator's warrant for this node, module and name: the id of the hand act recorded through mesh-hand-acts"),
"ask": str("the id of the operator's ask the warrant answers")},
run: func(caller string, a map[string]any) (any, error) {
v, err := env.ready()
if err != nil {
return nil, err
}
return v.RetireOwn(caller, RetireArgs{Node: arg(a, "node"), Module: arg(a, "module"), Name: arg(a, "name"),
Warrant: arg(a, "warrant"), Ask: arg(a, "ask")})
}},
{Name: "node_logins",
Description: "Every node's key of the vault's state node-logins: the login its node-engine dials as and, while a rotation " +
"runs, the other login and its stage; with the rotations queued and the enrolment tokens held. Never a value or a hash.",
@@ -728,6 +728,10 @@ func (v *Vault) RotateFor(caller, secret string, one Narrow, by, why string) (Ch
return Changed{}, fmt.Errorf("not rotated: %s is a bus account the controller mints and hands here (ADR 0374 "+
"clause 4): only a password whose hash the bus's user list holds can log in, so the vault makes no new "+
"value for it; `module issue` again hands a new one and replaces this in place", secret)
case isLedgersSecret(secret):
return Changed{}, fmt.Errorf("not rotated: %s is the tool runner's own secret of the ledger accounts of its machine, "+
"which the vault rewrites on each ledger account minted or retired (ADR 0597 §9); a person's rotation would "+
"replace every module's credential with a value no ledger account holds", secret)
case rec.Origin == OriginIssued:
return Changed{}, fmt.Errorf("not rotated: %s is issued by its provider's software, which the mesh cannot "+
"make or rotate (ADR 0305 §9): change it in that software, and `secret issued` takes the value it holds now",
@@ -26,6 +26,12 @@ func ledgersSecret(node string) (string, error) {
return OwnSecret(node, toolRunnerModule, buscompose.LedgersSecret)
}
// isLedgersSecret says secret is a machine's `ledgers`: `own/<node>/node-tools/ledgers`.
func isLedgersSecret(secret string) bool {
parts := strings.Split(secret, "/")
return len(parts) == 4 && parts[0] == "own" && parts[2] == toolRunnerModule && parts[3] == buscompose.LedgersSecret
}
// toolRunnerNodes are the machines whose tool runner is wanted, sorted.
func (v *Vault) toolRunnerNodes() []string {
v.minting.mu.Lock()
+40 -23
View File
@@ -19,6 +19,8 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-delivery/sdk/views"
)
func say(format string, args ...any) {
@@ -61,7 +63,7 @@ func build() *Env {
Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, WantRead: readWant,
HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{},
AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), EnrolCallers: askCallers(os.Getenv("MESH_VAULT_ENROLMENT_CALLERS")),
Logins: busNodeLogins{}, ReadAct: readHandAct, TokenBase: TokenBaseFromFile(os.Getenv("MESH_VAULT_STATE")),
RetireCallers: retireCallers(os.Getenv("MESH_VAULT_RETIRE_CALLERS")), Logins: busNodeLogins{}, ReadAct: readHandAct, TokenBase: TokenBaseFromFile(os.Getenv("MESH_VAULT_STATE")),
Bound: Bound{
Get: func(key string) ([]byte, error) {
e, err := stdio.State(BoundState).Get(key)
@@ -108,21 +110,24 @@ func readHandAct(id string) (*HandActSeen, error) {
return nil, nil
}
// watchLeavers deletes a node's key of node-logins when its node key of the placement is deleted (noderotation.go): a
// watch, never a poll. It answers whether the watch began; the minute pass (LoginsPass) catches what it missed.
// watchLeavers begins the picture of the placement and deletes a node's key of node-logins when its node key of the
// placement is deleted (noderotation.go): the owner's event, never a poll. It answers whether it began; the minute pass
// (LoginsPass) catches what it missed.
func watchLeavers(env *Env) bool {
if env.Vault == nil {
return true
}
err := stdio.State(PlacementRead).Watch("*.node", func(c stdio.StateChange) error {
if c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0 {
env.Vault.NodeLeft(strings.TrimSuffix(c.Key, ".node"))
}
return nil
})
if err != nil {
say("the placement's nodes are not watched for one that leaves; the minute pass finds it: %v", err)
return false
if placementView == nil {
placementView = newPlacementView(stdio.Ask, func(key string) {
if node, ok := leaverKey(key); ok {
env.Vault.NodeLeft(node)
}
}, func(key string, v json.RawMessage) error {
_, err := stdio.State(FindingsState).Put(key, v)
return err
},
func(key string) error { return stdio.State(FindingsState).Delete(key) })
beginPlacementView(placementView)
}
return true
}
@@ -232,17 +237,9 @@ func watchCredentialAsks(env *Env, kick chan<- struct{}) bool {
return true
}
// placement reads the seat mesh-placement's placement through the runtime under the vault's own account (its manifest reads it).
var placement = PlacementReader{
Keys: func() ([]string, error) { return stdio.State(PlacementRead).Keys() },
Read: func(key string) ([]byte, error) {
e, err := stdio.State(PlacementRead).Get(key)
if err != nil || e == nil {
return nil, err
}
return e.Value, nil
},
}
// placement reads the seat mesh-placement's placement through its owner's tool and events (placementview.go), never the
// owner's bucket.
var placement = viewPlacement(func() *views.Follower { return placementView })
// readWant is one wanted account of the grants part, read afresh (issue-account).
func readWant(user string) ([]byte, bool, error) {
@@ -368,6 +365,26 @@ func main() {
}
}
// retireCallers reads the vault's list `retire-callers`, the file the mesh writes from its trusted config value
// (stdio.ParseCallers): a list that cannot be read allows nobody, and says why.
func retireCallers(path string) stdio.Callers {
if path == "" {
say("the vault was started without its retire-callers list (MESH_VAULT_RETIRE_CALLERS): retire answers nobody")
return nil
}
raw, err := os.ReadFile(path)
if err != nil {
say("the retire-callers list cannot be read, so retire answers nobody: %v", err)
return nil
}
callers, err := stdio.ParseCallers(raw)
if err != nil {
say("%v: retire answers nobody", err)
return nil
}
return callers
}
// askCallers are the callers the vault's manifest names for secret-ask besides the controller (novox/hq ADR 0563 §8):
// MESH_VAULT_SECRET_ASK_CALLERS, comma-separated, each a module (on any machine), <machine>.<module> or person.<name>.
func askCallers(list string) stdio.Callers {
@@ -31,6 +31,7 @@ import (
"golang.org/x/crypto/bcrypt"
"golang.org/x/crypto/nacl/box"
"git.novox.be/novox/mesh-contracts/go/seats/meshvault"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/seal"
"git.novox.be/novox/mesh-sdk/go/testbus"
@@ -62,6 +63,70 @@ type player struct {
said []string
// ask answers a mesh/ask (a call of another module's tool); nil answers none, as a call with no responder.
ask func(key string, body json.RawMessage) (any, error)
// The assigner's placement as its owner's tool and views family say it (ADR 0615): the values, and the revision of the
// last change, kept from the bucket.
placeMu sync.Mutex
placeValues map[string]json.RawMessage
placeRevision uint64
}
var placeCreated = time.Date(2026, 11, 4, 9, 0, 0, 0, time.UTC)
// followPlacement keeps the bucket of the placement as the assigner's owner does: listed once, then each change said on
// its views family to the vault. Call it before the player serves.
func (p *player) followPlacement() {
p.t.Helper()
kv, err := p.js.KeyValue(p.ctx, "mesh-delivery-assigner_placement")
if err != nil {
p.t.Fatal(err)
}
w, err := kv.WatchAll(p.ctx)
if err != nil {
p.t.Fatal(err)
}
p.placeValues = map[string]json.RawMessage{}
listed := make(chan struct{})
go func() {
current := true
for e := range w.Updates() {
if e == nil {
current = false
close(listed)
continue
}
p.placeMu.Lock()
deleted := e.Operation() != jetstream.KeyValuePut
if deleted {
delete(p.placeValues, e.Key())
} else {
p.placeValues[e.Key()] = append(json.RawMessage(nil), e.Value()...)
}
if !current {
p.placeRevision++
ev := map[string]any{"v": 1, "owner": "mesh-delivery-assigner", "state": "placement", "key": e.Key(),
"revision": p.placeRevision, "created": placeCreated}
if deleted {
ev["deleted"] = true
} else {
ev["value"] = json.RawMessage(e.Value())
}
body, _ := json.Marshal(ev)
p.mu.Lock()
p.next++
id := fmt.Sprintf("o-%d", p.next)
p.mu.Unlock()
p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{
"key": "mesh-delivery-assigner.views.placement", "node": "novox", "body": json.RawMessage(body)}}})
}
p.placeMu.Unlock()
}
}()
select {
case <-listed:
case <-time.After(10 * time.Second):
p.t.Fatal("the placement was not listed")
}
}
// bucketOf is the bucket a state names: the vault's own by its local name, another's as `<owner>.<name>`.
@@ -223,6 +288,15 @@ func (p *player) handle(m wire) {
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &ask)
if ask.Key == PlacementTool && p.placeValues != nil {
p.placeMu.Lock()
answer := map[string]any{"v": 1, "owner": "mesh-delivery-assigner", "state": "placement", "revision": p.placeRevision,
"created": placeCreated, "values": p.placeValues, "at": time.Now()}
raw, _ := json.Marshal(answer)
p.placeMu.Unlock()
p.answer(m.ID, json.RawMessage(raw), nil)
return
}
if p.ask != nil {
result, err := p.ask(ask.Key, ask.Body)
p.answer(m.ID, result, err)
@@ -300,6 +374,16 @@ func (p *player) hashOf(t *testing.T, user string) *AccountHash {
return &a
}
// strictly decodes raw into v, a field the type does not know being a fault (the definition's type is the contract).
func strictly(t *testing.T, raw []byte, v any) {
t.Helper()
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(v); err != nil {
t.Errorf("%s does not match the seat's definition: %v", raw, err)
}
}
func eventually(t *testing.T, what string, ok func() bool) {
t.Helper()
for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) {
@@ -347,10 +431,30 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) {
if err := os.WriteFile(filepath.Join(root, "export.json"), export, 0o600); err != nil {
t.Fatal(err)
}
// An own secret of dunst on g14 is held before the vault starts, for the retire case below (CAT-V1).
seedStore, err := OpenStore(filepath.Join(dir, "store"))
if err != nil {
t.Fatal(err)
}
seedKey, err := OwnKey(filepath.Join(dir, "state"), seedStore)
if err != nil {
t.Fatal(err)
}
seed := &Vault{Store: seedStore, Own: seedKey, Now: time.Now, Log: t.Logf, HashCost: bcrypt.MinCost}
if _, err := seed.SetOperatorKey(asPEM(t, base64.StdEncoding.EncodeToString(opPub[:])), "the test"); err != nil {
t.Fatal(err)
}
if _, err := seed.keep("own/g14/dunst/token", "a-token-value", "the test", "g14.dunst", OriginMade); err != nil {
t.Fatal(err)
}
callers := filepath.Join(dir, "retire-callers.json")
if err := os.WriteFile(callers, []byte(`{"callers":["mesh-config"]}`), 0o600); err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_VAULT_STORE="+filepath.Join(dir, "store"), "MESH_VAULT_STATE="+filepath.Join(dir, "state"),
"MESH_VAULT_ROOT="+root, "MESH_VAULT_LEDGER="+filepath.Join(dir, "ledger"), "MESH_RECEIVES="+filepath.Join(dir, "mesh.json"),
"MESH_NODE=novox", "MESH_MODULE=mesh-vault")
"MESH_VAULT_RETIRE_CALLERS="+callers, "MESH_NODE=novox", "MESH_MODULE=mesh-vault")
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
@@ -359,6 +463,16 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) {
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{t: t, ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
// mesh-hand-acts holds the acts mesh-config recorded: one warranting the token's retirement, one for another name.
p.ask = func(key string, _ json.RawMessage) (any, error) {
if key != "mesh-hand-acts.hand-acts" {
return nil, errors.New("nats: no responders available for request")
}
return map[string]any{"acts": []HandActSeen{
{ID: "act-ok", By: "novox.mesh-config", At: time.Now(), Verb: "mesh-vault.retire", Args: []string{"g14 dunst token ret-1"}},
{ID: "act-other", By: "novox.mesh-config", At: time.Now(), Verb: "mesh-vault.retire", Args: []string{"g14 dunst other ret-1"}}}}, nil
}
p.followPlacement()
go p.serve(stdout)
status := func() MintState {
@@ -418,4 +532,45 @@ func TestTheVaultWholeMintsOnTheBusWhoeverHoldsTheLease(t *testing.T) {
if !strings.Contains(said, AccountIssuedEvent+` {"`) || !strings.Contains(said, `"user":"ace.mod-x"`) || strings.Contains(said, held.Password) {
t.Errorf("the vault said: %s", said)
}
// CAT-V1 (design 32): retire on the real bus. A waiter on rotated and own-replaced hears nothing of it.
waiter, err := nc.SubscribeSync("mesh.mod.mesh-vault.event.>")
if err != nil {
t.Fatal(err)
}
for _, c := range []struct{ caller, act string }{{"g14.claude-code", "act-ok"}, {"novox.mesh-config", "act-other"}, {"novox.mesh-config", ""}} {
if _, err := p.call(Seat+".retire", c.caller, map[string]any{"node": "g14", "module": "dunst", "name": "token", "warrant": c.act}); err == nil {
t.Errorf("%s retired the token with act %q", c.caller, c.act)
}
}
if m, err := waiter.NextMsg(300 * time.Millisecond); err == nil {
t.Fatalf("a refused retire said %s", m.Subject)
}
// The call is the definition's input, the answer its answer, and the event its body on its subject (CAT-V4).
input, _ := json.Marshal(meshvault.RetireInput{Node: "g14", Module: "dunst", Name: "token", Warrant: "act-ok", Ask: "ret-1"})
var args map[string]any
_ = json.Unmarshal(input, &args)
raw, err = p.call(Seat+"."+meshvault.VerbRetire, "novox.mesh-config", args)
if err != nil {
t.Fatalf("retire with its warrant: %v", err)
}
var answered meshvault.RetireAnswer
strictly(t, raw, &answered)
if !answered.Retired {
t.Errorf("retire answered %s", raw)
}
m, err := waiter.NextMsg(10 * time.Second)
if err != nil || m.Subject != meshvault.HolderSubjectOwnRetired("mesh-vault") {
t.Fatalf("retire said %v, %v", m, err)
}
var event meshvault.OwnRetiredHolderEvent
strictly(t, m.Data, &event)
if event.Ask != "ret-1" || event.Caller != "novox.mesh-config" || event.Node != "g14" || event.Module != "dunst" || event.Name != "token" {
t.Errorf("own-retired said %s", m.Data)
}
for _, other := range []string{"rotated", OwnReplacedEvent} {
if extra, err := waiter.NextMsg(300 * time.Millisecond); err == nil {
t.Errorf("a waiter on %s heard %s", other, extra.Subject)
}
}
}
@@ -196,6 +196,21 @@ func TestTheVaultMintsAWantedAccountOnceAndNothingItHolds(t *testing.T) {
t.Errorf("wanted again, m is not back in ledgers (%d) with its hash", len(got))
}
}},
// 1966/CAT-V2 (ADR 0597 §9): a person's rotate of a machine's `ledgers` is refused as a bus account's is: it
// would replace every module's credential with a value no ledger account holds.
{"a person's rotate of ledgers is refused", func(t *testing.T, v testVault, accounts *fakeAccounts, priv string) {
w := nodeToolsWant("ace")
v.WantHeard(w.User, wantRaw(t, w), false)
v.LedgersPass()
before, _ := ledgersOf(t, v, priv)
if _, err := v.Rotate("person.jochen", "own/ace/node-tools/ledgers", "jochen", "a test"); err == nil ||
!strings.Contains(err.Error(), "ledger accounts of its machine") {
t.Errorf("a person rotated ledgers: %v", err)
}
if after, _ := ledgersOf(t, v, priv); len(after) != len(before) {
t.Errorf("a refused rotation changed ledgers: %d, was %d", len(after), len(before))
}
}},
} {
t.Run(c.name, func(t *testing.T) {
v, accounts, priv := mintingVault(t)
+5 -3
View File
@@ -455,6 +455,11 @@ func (v *Vault) HandedOf(node string) ([]HandedJudged, error) {
}
plan := v.plansOf(node)
placed, placedErr := plan.placedModules()
if placedErr != nil {
// Never dropped: an answer that leaves out the modules placed there, with no word of it, would read as "the others
// are not placed" (issue 1966 review note).
return nil, fmt.Errorf("the placement of %s could not be read, so the modules placed there are not all judged: %w", node, placedErr)
}
for _, m := range placed {
modules[m] = true
}
@@ -462,9 +467,6 @@ func (v *Vault) HandedOf(node string) ([]HandedJudged, error) {
for _, module := range slices.Sorted(maps.Keys(modules)) {
out = append(out, plan.judged(module))
}
if placedErr != nil && len(out) == 0 {
return nil, fmt.Errorf("the placement of %s could not be read and the vault keeps no own secret there: %w", node, placedErr)
}
return out, nil
}
@@ -270,6 +270,7 @@ func TestOwnLeavesOutOnlyAModuleWhosePlanHandsIt(t *testing.T) {
}
v := newVault(t, nil)
v.Askers = Declared{Read: plans.read, Store: v.Store}
v.Placement = placementsOf(t, map[string][]string{"anchor": {"grafana", "web", "db"}})
pub, _ := aSealingKey(t)
for _, m := range []MoveOf{
{Node: "anchor", Module: "grafana", Name: "admin", Origin: "made", Generation: "g-1", SealingKey: pub},
@@ -314,4 +315,14 @@ func TestOwnLeavesOutOnlyAModuleWhosePlanHandsIt(t *testing.T) {
if len(judged) != 4 {
t.Errorf("own_handed names %v, want builder, db, grafana and web", judged)
}
// Review note of issue 1966: a placement that cannot be read is an error the tool answers, never a list that leaves out
// the modules placed there while the vault keeps an own secret of others.
v.Placement = PlacementReader{Keys: func() ([]string, error) { return nil, errors.New("the bus did not answer") },
Read: func(string) ([]byte, error) { return nil, errors.New("the bus did not answer") }}
if got, err := v.HandedOf("anchor"); err == nil || !strings.Contains(err.Error(), "placement of anchor could not be read") || got != nil {
t.Errorf("an unreadable placement answered %v, %v", got, err)
}
if _, err := toolNamed(t, definitions(&Env{Vault: v.Vault}), "own_handed").Run(map[string]any{"node": "anchor"}); err == nil {
t.Error("own_handed answered with the placement unread")
}
}
@@ -0,0 +1,136 @@
package main
// The seat's verb `retire` (design 32 *A value no longer read*, novox/hq ADR 0609): one module's own secret on one node
// removed because no manifest reads it any more, asked by the holder of the seat mesh-config alone after the operator's
// Approve. Two proofs, both required:
//
// - the caller is on the vault's list `retire-callers`, a trusted config value of the vault (ADR 0363's form, as
// mesh-mail-reader's `callers`), which names the holder of mesh-config alone, checked against the caller the tool
// runner proved (stdio.Callers); and
// - the warrant (the verb's argument, as the seat's definition names it): the id of a hand act the caller recorded
// through mesh-hand-acts for this verb, read back here, recent, naming exactly this node, module and secret and the
// ask whose Approve it answers (ADR 0259 §6). An act for another name warrants nothing.
//
// The vault then removes the record whole and says `own-retired` (node, module, name, the ask's id, the caller it proved)
// beside `rotated` and `own-replaced`, under its own subject: a waiter on either does not wake on it. Never a value.
import (
"fmt"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// OwnRetiredEvent is what the vault says when it retired an own secret.
const OwnRetiredEvent = "own-retired"
// RetireVerb is the seat's verb, and the verb of the hand act that warrants it (`mesh-vault.retire`).
const RetireVerb = "retire"
// RetireArgs are `retire`'s arguments, as the seat's definition names them: the secret by node, module and name, the
// warrant (the id of the hand act that warrants it) and the ask of the operator's Approve it answers.
type RetireArgs struct {
Node, Module, Name, Warrant, Ask string
}
// OwnRetiredBody is the body of own-retired, exactly the seat definition's (node, module, name, ask, caller).
type OwnRetiredBody struct {
Node string `json:"node"`
Module string `json:"module"`
Name string `json:"name"`
Ask string `json:"ask"`
Caller string `json:"caller"`
}
// OwnRetired is what one `retire` answers, the seat definition's answer.
type OwnRetired struct {
Retired bool `json:"retired"`
}
// fromRetireCallers refuses every caller the vault's list `retire-callers` does not name.
func (v *Vault) fromRetireCallers(caller string) error {
if !v.RetireCallers.Allows(caller) {
if strings.TrimSpace(caller) == "" {
caller = stdio.Unproven
}
return fmt.Errorf("retire is asked by the holder of the seat mesh-config alone, as the vault's list retire-callers "+
"names it; %s is not on it (design 32)", caller)
}
return nil
}
// warrantsRetire reads the hand act back and refuses what it does not warrant for exactly this secret; it answers the
// ask the act names.
func (v *Vault) warrantsRetire(caller string, a RetireArgs) (ask string, err error) {
if strings.TrimSpace(a.Ask) == "" {
return "", fmt.Errorf("retire names the ask of the operator's Approve it answers (`ask`)")
}
if strings.TrimSpace(a.Warrant) == "" {
return "", fmt.Errorf("retire carries the hand act that warrants it (`warrant`): record it through mesh-hand-acts first (design 32)")
}
if v.ReadAct == nil {
return "", fmt.Errorf("the vault reads no hand acts, so it retires nothing")
}
act, err := v.ReadAct(a.Warrant)
if err != nil {
return "", fmt.Errorf("the hand act %s could not be read back from mesh-hand-acts: %w", a.Warrant, err)
}
switch {
case act == nil:
return "", fmt.Errorf("mesh-hand-acts holds no hand act %s: record one first", a.Warrant)
case act.By != caller:
return "", fmt.Errorf("the hand act %s was recorded by %s, not by %s who asks", a.Warrant, act.By, caller)
case v.Now().Sub(act.At) > HandActWithin:
return "", fmt.Errorf("the hand act %s is older than %s: record a new one", a.Warrant, HandActWithin)
case act.Verb != Seat+"."+RetireVerb:
return "", fmt.Errorf("the hand act %s is %q, not %s.%s: it warrants no retirement", a.Warrant, act.Verb, Seat, RetireVerb)
}
var named []string
for _, arg := range act.Args {
named = append(named, strings.Fields(arg)...)
}
// The warrant is bound to the node, module and name it was asked for, and names the ask: those four words, no others.
if len(named) != 4 || named[0] != a.Node || named[1] != a.Module || named[2] != a.Name || named[3] != a.Ask {
return "", fmt.Errorf("the hand act %s names %q, not %s %s %s and ask %s: a warrant retires nothing else",
a.Warrant, strings.Join(named, " "), a.Node, a.Module, a.Name, a.Ask)
}
return named[3], nil
}
// RetireOwn removes one module's own secret on a node on the warrant of a hand act, and says `own-retired`.
func (v *Vault) RetireOwn(caller string, a RetireArgs) (OwnRetired, error) {
if err := v.fromRetireCallers(caller); err != nil {
return OwnRetired{}, err
}
secret, err := OwnSecret(a.Node, a.Module, a.Name)
if err != nil {
return OwnRetired{}, err
}
ask, err := v.warrantsRetire(caller, a)
if err != nil {
return OwnRetired{}, err
}
switch {
case Covered(secret) || isLedgersSecret(secret):
return OwnRetired{}, fmt.Errorf("%s is a bus account or the tool runner's ledgers, which the vault retires and rewrites with the wants (ADR 0230, ADR 0597 §9), never a retirement of a value no manifest reads", secret)
}
_, held, err := v.Store.Read(secret)
if err != nil {
return OwnRetired{}, err
}
if !held {
return OwnRetired{}, fmt.Errorf("the vault holds no secret named %s, so there is nothing to retire", secret)
}
if err := v.Store.Delete(secret); err != nil {
return OwnRetired{}, fmt.Errorf("%s is not retired: %w", secret, err)
}
out := OwnRetired{Retired: true}
v.log("retired %s on the warrant of the hand act %s, asked by %s (ask %s)", secret, a.Warrant, caller, ask)
if v.Emit != nil {
body := map[string]any{"node": a.Node, "module": a.Module, "name": a.Name, "ask": ask, "caller": caller}
if err := v.Emit(OwnRetiredEvent, body); err != nil {
return out, fmt.Errorf("%s is retired, but own-retired was not said: %w", secret, err)
}
}
return out, nil
}
@@ -0,0 +1,198 @@
package main
// The vault's reading of the seat mesh-placement's placement, through its owner (novox/hq ADR 0615 §1, §4 to §6, ADR
// 0596): the assigner's tool `assigner_read_placement` asked for the whole state at start, the owner's `views.placement`
// events after it, and the tool once more only on a gap. Started while the assigner is down it keeps asking and says so;
// it never reads the assigner's bucket instead. (The vault's other reads, the node-engines' haves, the grants part's
// accounts and the provisioner's plans and provisions, are not the indexer's or the assigner's state and stay as they are.)
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/views"
)
// PlacementTool is the assigner's ordinary read tool of its placement; PlacementViewsEvent the event of its views family.
const (
PlacementTool = delivery.AssignerModule + ".assigner_read_placement"
PlacementViewsEvent = delivery.AssignerModule + ".views.placement"
FirstViewsTool = "vault_first_views"
FirstViewAgainTool = "vault_first_view_again"
)
// FindingsState is the vault's state of findings, which the condition keeper reads (ADR 0564): a finding names no secret.
const FindingsState = "findings"
// placementView is the picture; nil until main begins it.
var placementView *views.Follower
// newPlacementView makes the follower: ask calls the tool, gone is told of a deleted key.
func newPlacementView(ask func(key string, body any) (json.RawMessage, error), gone func(key string),
put func(key string, value json.RawMessage) error, drop func(key string) error) *views.Follower {
return &views.Follower{Owner: delivery.AssignerModule, Tool: PlacementTool, State: "placement", Logf: say,
Ask: func(_ context.Context, after string) (views.Answer, error) {
body := map[string]any{"values": true}
if after != "" {
body["after"] = after
}
raw, err := ask(PlacementTool, body)
if err != nil {
return views.Answer{}, err
}
return views.DecodeAnswer(unwrapToolAnswer(raw))
},
// Past its bound the vault's finding names the owner and the tool, and the condition keeper raises it (the
// answer deletes it); the vault keeps asking meanwhile.
Raise: func(s views.Status) {
say("the vault has no first view of the assigner's placement: %s has not answered since %s (%s); it keeps asking", s.Tool,
s.Asked.Format(time.RFC3339), s.Error)
if k, v, err := views.UnansweredFinding(Seat, s, time.Now()); err == nil {
if err := put(k, v); err != nil {
say("the finding %s is not written: %v", k, err)
}
}
},
Clear: func(s views.Status) {
if k, _, err := views.UnansweredFinding(Seat, s, time.Now()); err == nil {
_ = drop(k)
}
},
Changed: func(key string, _ json.RawMessage, deleted bool, _ time.Time) {
if deleted {
gone(key)
}
}}
}
// unwrapToolAnswer takes a tool's answer out of the protocol's content list when it comes in one.
func unwrapToolAnswer(raw json.RawMessage) []byte {
var r struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if json.Unmarshal(raw, &r) == nil && len(r.Content) == 1 {
return []byte(r.Content[0].Text)
}
return raw
}
// beginPlacementView asks the first view until answered and hears the owner's events.
func beginPlacementView(f *views.Follower) {
go func() {
if err := f.Start(context.Background()); err != nil {
say("the first view of the assigner's placement ended unanswered: %v", err)
}
}()
go func() {
<-stdio.Served()
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
err := stdio.Subscribe(PlacementViewsEvent, func(e stdio.Envelope) error {
ev, err := views.DecodeEvent(e.Body)
if err != nil {
say("%s is not a views event: %v", PlacementViewsEvent, err)
return nil
}
if gap, why := f.Apply(ev); gap {
say("a gap in the assigner's placement (%s): its first view is asked once more", why)
go func() { _ = f.Start(context.Background()) }()
}
return nil
})
if err == nil {
return
}
say("does not hear %s yet (%v); asking again in %s", PlacementViewsEvent, err, wait)
time.Sleep(wait)
}
}()
}
// viewPlacement is the placement reader over the picture: a key asked before the first view is answered is an error, so
// the caller waits (the "not yet" of the vault's other reads), never a read of the owner's bucket.
func viewPlacement(f func() *views.Follower) PlacementReader {
ready := func() (*views.Follower, error) {
fol := f()
if fol == nil {
return nil, fmt.Errorf("the vault reads the assigner's placement only when it runs as the module")
}
select {
case <-fol.Ready():
return fol, nil
default:
s := fol.Status()
return nil, fmt.Errorf("the first view of the assigner's placement is not answered yet (%s asked %d times)", s.Tool, s.Asks)
}
}
return PlacementReader{
Keys: func() ([]string, error) {
fol, err := ready()
if err != nil {
return nil, err
}
var keys []string
for k := range fol.Values() {
keys = append(keys, k)
}
sort.Strings(keys)
return keys, nil
},
Read: func(key string) ([]byte, error) {
fol, err := ready()
if err != nil {
return nil, err
}
v, ok := fol.Get(key)
if !ok {
return nil, nil
}
return v, nil
},
}
}
// leaverKey is whether a deleted key of the placement is a node's own key, `<node>.node`.
func leaverKey(key string) (string, bool) {
node, ok := strings.CutSuffix(key, ".node")
return node, ok && node != ""
}
// viewTools are the read tool of the first view and its re-read, as every reader has them (design 25 § 12.2).
func viewTools(env *Env) []toolDef {
return []toolDef{
{Name: FirstViewsTool,
Description: "The vault's first view of the assigner's placement: the tool it asks, the revision its picture holds, when it " +
"asked and was answered, how many asks (and by hand), and whether it is waiting.",
run: func(string, map[string]any) (any, error) {
if placementView == nil {
return nil, fmt.Errorf("the vault reads the assigner's placement only when it runs as the module")
}
return map[string]any{"first-views": []views.Status{placementView.Status()}}, nil
}},
{Name: FirstViewAgainTool,
Description: "Ask the first view of the assigner's placement once more by hand, with why, recorded in the vault's log.",
Input: map[string]any{"why": str("why the first view is asked again")},
run: func(caller string, a map[string]any) (any, error) {
if placementView == nil {
return nil, fmt.Errorf("the vault reads the assigner's placement only when it runs as the module")
}
why := strings.TrimSpace(arg(a, "why"))
if why == "" {
return nil, fmt.Errorf("say why: a first view asked by hand keeps its why")
}
if err := fromAGrantedCaller(caller, FirstViewAgainTool); err != nil {
return nil, err
}
say("the first view of the assigner's placement is asked again by %s: %s", caller, why)
return map[string]any{"asked-again": true}, placementView.Again(context.Background(), why)
}},
}
}
@@ -0,0 +1,59 @@
package main
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/views"
)
// ADR 0615 §6: the vault started while the assigner is down keeps asking, never reads its bucket, and past the bound
// writes a finding naming the owner and the tool for the condition keeper; the answer deletes it. Before the answer a read
// of the placement is an error.
func TestAnUnansweredFirstViewOfThePlacementIsAFindingTheAnswerClears(t *testing.T) {
var mu sync.Mutex
up := false
held := map[string]json.RawMessage{}
ask := func(string, any) (json.RawMessage, error) {
mu.Lock()
defer mu.Unlock()
if !up {
return nil, errors.New("nats: no responders available for request")
}
raw, _ := json.Marshal(views.Answer{V: 1, Owner: "mesh-delivery-assigner", State: "placement", Revision: 1, Created: time.Now(),
Values: map[string]json.RawMessage{"g14.node": json.RawMessage(`{}`)}})
return raw, nil
}
put := func(k string, v json.RawMessage) error { mu.Lock(); held[k] = v; mu.Unlock(); return nil }
drop := func(k string) error { mu.Lock(); delete(held, k); mu.Unlock(); return nil }
f := newPlacementView(ask, func(string) {}, put, drop)
f.Bound, f.Wait = 30*time.Millisecond, 10*time.Millisecond
reader := viewPlacement(func() *views.Follower { return f })
if _, err := reader.Keys(); err == nil {
t.Fatal("the placement was read before its first view")
}
done := make(chan error, 1)
go func() { done <- f.Start(context.Background()) }()
count := func() int { mu.Lock(); defer mu.Unlock(); return len(held) }
for end := time.Now().Add(5 * time.Second); count() == 0 && time.Now().Before(end); time.Sleep(10 * time.Millisecond) {
}
if count() != 1 {
t.Fatalf("past its bound the vault wrote %d findings, want one", count())
}
mu.Lock()
up = true
mu.Unlock()
if err := <-done; err != nil {
t.Fatal(err)
}
if count() != 0 {
t.Errorf("the answer left %d findings", count())
}
if keys, err := reader.Keys(); err != nil || len(keys) != 1 || keys[0] != "g14.node" {
t.Errorf("keys %v, %v", keys, err)
}
}
@@ -4,6 +4,8 @@ import (
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// 774/AC9 (novox/hq ADR 0561 §2, ADR 0230 §15, §18): a wanted account whose key is deleted is retired (its hash taken
@@ -80,3 +82,90 @@ func TestAWantGoneRetiresItsAccountOnlyWithinTheBounds(t *testing.T) {
t.Errorf("after a person's retirement %d keys remain", len(accounts.keys))
}
}
// 1966/CAT-V1 (design 32): `retire` removes one module's own secret on a node, and says own-retired, only for a caller on
// the vault's list retire-callers who carries a hand act it recorded for this verb naming exactly this node, module and
// secret. Without the act, with another name's, with another caller's or an older one, from a caller not on the list, or
// for a bus account, nothing is removed and nothing is said; a waiter on rotated or own-replaced is not woken.
func TestRetireRemovesAnOwnSecretOnlyOnTheWarrantOfTheListedCaller(t *testing.T) {
v := newVault(t, nil)
const caller = "novox.mesh-config"
v.RetireCallers = stdio.Callers{"mesh-config"}
acts := map[string]*HandActSeen{}
v.ReadAct = func(id string) (*HandActSeen, error) { return acts[id], nil }
act := func(id, by, verb, what string, age time.Duration) {
acts[id] = &HandActSeen{ID: id, By: by, At: v.Now().Add(-age), Verb: verb, Args: []string{what}}
}
act("ok", caller, "mesh-vault.retire", "g14 dunst token ret-1", time.Minute)
act("other-name", caller, "mesh-vault.retire", "g14 dunst other ret-1", time.Minute)
act("other-node", caller, "mesh-vault.retire", "ace dunst token ret-1", time.Minute)
act("other-caller", "g14.claude-code", "mesh-vault.retire", "g14 dunst token ret-1", time.Minute)
act("other-verb", caller, "mesh-vault.rotate", "g14 dunst token ret-1", time.Minute)
act("ledgers", caller, "mesh-vault.retire", "g14 node-tools ledgers ret-1", time.Minute)
act("no-ask", caller, "mesh-vault.retire", "g14 dunst token", time.Minute)
act("old", caller, "mesh-vault.retire", "g14 dunst token ret-1", 25*time.Hour)
const secret = "own/g14/dunst/token"
for _, name := range []string{secret, "own/g14/dunst/other", "own/g14/node-tools/ledgers"} {
if _, err := v.keep(name, "a-value-of-"+name, "the test", "g14.dunst", OriginMade); err != nil {
t.Fatal(err)
}
}
for _, tc := range []struct {
name, caller string
args RetireArgs
want string
}{
{"no act", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Ask: "ret-1"}, "carries the hand act"},
{"an act that does not exist", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "nope", Ask: "ret-1"}, "holds no hand act"},
{"an act for another name", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-name", Ask: "ret-1"}, "a warrant retires nothing else"},
{"an act for another node", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-node", Ask: "ret-1"}, "a warrant retires nothing else"},
{"an act of another caller", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-caller", Ask: "ret-1"}, "was recorded by g14.claude-code"},
{"an act for another verb", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-verb", Ask: "ret-1"}, "warrants no retirement"},
{"an ask other than the act's", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-2"}, "a warrant retires nothing else"},
{"no ask", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok"}, "names the ask"},
{"an act that names no ask", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "no-ask", Ask: "ret-1"}, "a warrant retires nothing else"},
{"an act older than a day", caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "old", Ask: "ret-1"}, "older than"},
{"a caller not on the list", "g14.claude-code", RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "other-caller", Ask: "ret-1"}, "retire-callers"},
{"an unproven caller", stdio.Unproven, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-1"}, "retire-callers"},
{"the tool runner's ledgers", caller, RetireArgs{Node: "g14", Module: "node-tools", Name: "ledgers", Warrant: "ledgers", Ask: "ret-1"}, "bus account or the tool runner's ledgers"},
} {
t.Run(tc.name, func(t *testing.T) {
before := len(v.bus.events)
if _, err := v.RetireOwn(tc.caller, tc.args); err == nil || !strings.Contains(err.Error(), tc.want) {
t.Errorf("answered %v, want a refusal saying %q", err, tc.want)
}
if _, held, _ := v.Store.Read(secret); !held {
t.Error("a refused retire removed the secret")
}
if n := len(v.bus.events) - before; n != 0 {
t.Errorf("a refused retire said %d events", n)
}
})
}
before := len(v.bus.events)
got, err := v.RetireOwn(caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-1"})
if err != nil || !got.Retired {
t.Fatalf("retire answered %+v, %v", got, err)
}
if _, held, _ := v.Store.Read(secret); held {
t.Error("the secret is still held")
}
if _, held, _ := v.Store.Read("own/g14/dunst/other"); !held {
t.Error("another secret of the module was removed")
}
if n := eventsNamed(v, before, OwnRetiredEvent); n != 1 || len(v.bus.events)-before != 1 {
t.Fatalf("said %d own-retired among %d events, want one and no other", n, len(v.bus.events)-before)
}
if body, _ := v.bus.events[before]["body"].(map[string]any); len(body) != 5 || body["ask"] != "ret-1" || body["caller"] != caller || body["node"] != "g14" ||
body["module"] != "dunst" || body["name"] != "token" {
t.Errorf("own-retired said %+v", body)
}
if OwnRetiredEvent == OwnReplacedEvent || OwnRetiredEvent == "rotated" {
t.Error("own-retired shares a name with an event a waiter hears")
}
// Asked again for what is gone: nothing to retire.
if _, err := v.RetireOwn(caller, RetireArgs{Node: "g14", Module: "dunst", Name: "token", Warrant: "ok", Ask: "ret-1"}); err == nil ||
!strings.Contains(err.Error(), "nothing to retire") {
t.Errorf("a second retire answered %v", err)
}
}
@@ -88,6 +88,7 @@ func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) {
}
return map[string]any{"acts": []HandActSeen{{ID: "act-1", By: "person.jochen", At: time.Now(), Verb: "rotate-node", Args: []string{"g14"}}}}, nil
}
p.followPlacement()
go p.serve(stdout)
// The bus node (novox, running nats) and the node that rotates (g14); no lease key anywhere (issue 1126: the controller
+1 -1
View File
@@ -79,7 +79,7 @@ func str(description string) map[string]any {
}
func definitions(env *Env) []toolDef {
return append(enrolmentTools(env), []toolDef{
return append(append(enrolmentTools(env), viewTools(env)...), []toolDef{
{Name: Seat + ".get",
Description: "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its " +
"generation; answered only to an asker the vault allows, as the runtime proved it (ADR 0349 §11). " +
@@ -77,6 +77,9 @@ type Vault struct {
// EnrolCallers are the callers of the enrolment verbs `enrolment-token` and `node-account` and of `sign` for an
// enrolment: the seat mesh-enrolment (enrolment.go). Nil: that module alone.
EnrolCallers stdio.Callers
// RetireCallers are the callers of the seat's verb `retire`: the vault's trusted config value `retire-callers`, the
// holder of the seat mesh-config alone (ownretire.go). Nil: nobody.
RetireCallers stdio.Callers
// TokenBase says where the bus is and its certificate's fingerprint, for the token shown (enrolment.go): nil shows none.
TokenBase func() (TokenBase, error)
// Logins is the vault's state node-logins (noderotation.go): nil writes none.
@@ -146,14 +146,14 @@ func TestNoBucketHoldsASecret(t *testing.T) {
if err := json.Unmarshal(raw, &manifest); err != nil {
t.Fatal(err)
}
if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound","node-logins"]` {
if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound","node-logins","findings"]` {
t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the "+
"buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+
"credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6) and each node's login in use (ADR 0583)", s)
}
if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` {
if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-provisioner.plans","mesh-provisioner.provisions"]` {
t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+
"grants part's wanted accounts and the placement alone, never the controller's lease (issue 1126)", r)
"grants part's wanted accounts alone (the placement is the assigner's, through its tool and events, ADR 0615), never the controller's lease (issue 1126)", r)
}
sources, _ := filepath.Glob("*.go")
bucket := regexp.MustCompile(`stdio\.State\(|nats-io|jetstream|KeyValue`)
@@ -195,11 +195,9 @@ func TestNoBucketHoldsASecret(t *testing.T) {
code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Keys(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Watch(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Get(", "", 1))
// And the seat's placement, where modules stand (placed.go): its keys and one key, read.
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1))
// And one watch of its nodes' keys, for a node that leaves: its node-logins key goes with it (noderotation.go).
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Watch(", "", 1))
// And its findings (ADR 0615 §6): the finding of a first view unanswered, written and deleted, never a secret.
code = []byte(strings.Replace(string(code), "stdio.State(FindingsState).Put(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(FindingsState).Delete(", "", 1))
// And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and
// one read of the seat's provisions, which hold none.
for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} {
@@ -0,0 +1,3 @@
{
"callers": ["mesh-config"]
}
+2
View File
@@ -3,6 +3,8 @@ module mesh-vault
go 1.26.0
require (
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/go v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103174208-b7abe203a929
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
+44 -7
View File
@@ -24,15 +24,16 @@
"signed",
"rotation-begun",
"node-account-ready",
"rotation-done"
"rotation-done",
"own-retired"
],
"invokes": [
"seat:mesh-hand-acts.hand-acts"
"seat:mesh-hand-acts.hand-acts",
"mesh-delivery-assigner.assigner_read_placement"
],
"reads": [
"node-engine.have",
"mesh-grants.accounts",
"mesh-delivery-assigner.placement",
"mesh-provisioner.plans",
"mesh-provisioner.provisions"
],
@@ -43,7 +44,8 @@
},
"signers",
"bound",
"node-logins"
"node-logins",
"findings"
],
"receives": {
"secret": "${dir:grants}/mesh.json"
@@ -142,6 +144,19 @@
"mode": "0700",
"owner": "mesh-vault",
"unknown": "program"
},
{
"id": "retire-callers",
"trusted": true,
"type": "file",
"path": "${dir:state}/retire-callers.json",
"mode": "0600",
"owner": "mesh-vault",
"merge": "json",
"template": "files/retire-callers.json",
"takes-config": [
"retire-callers"
]
}
],
"build": {
@@ -163,8 +178,14 @@
"MESH_VAULT_STORE": "${dir:store}",
"MESH_VAULT_STATE": "${dir:state}",
"MESH_VAULT_SECRET_ASK_CALLERS": "node-tools",
"MESH_VAULT_ENROLMENT_CALLERS": "mesh-enrolment"
"MESH_VAULT_ENROLMENT_CALLERS": "mesh-enrolment",
"MESH_VAULT_RETIRE_CALLERS": "${dir:state}/retire-callers.json"
}
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
},
@@ -191,7 +212,9 @@
"vault_signers",
"vault_signed",
"signers_reassert",
"node_logins"
"node_logins",
"vault_first_views",
"vault_first_view_again"
],
"claims": [
{
@@ -211,8 +234,22 @@
"node-account",
"rotate-node",
"node-account-confirm",
"abandon"
"abandon",
"retire"
]
}
],
"config": {
"retire-callers": {
"type": "list",
"description": "Who may call the seat's verb retire, which removes one module's own secret on a node because no manifest reads it any more: a module's name (on every machine), <machine>.<module>, or person.<name>. The holder of the seat mesh-config alone by default (design 32); the verb also needs a hand act the caller recorded for exactly that node, module and secret.",
"trusted": true,
"default": [
"mesh-config"
]
}
},
"consumes": [
"mesh-delivery-assigner.views.placement"
]
}