Files
mesh-catalog/modules/mesh-config/cmd/mesh-config/retiring.go
T
jschoubben 9d83987815 Join item 1: the catalogue's modules take the router's ask from operator-channel's contract and client
mesh-sdk holds neither asks nor asker, nor the vault's answer types, and mesh-delivery's
sdk/views is mesh-sdk's go/views (a1-progress.md, the rewrite map for the join): thirteen
modules switch to the operatorchannel contract and client, mesh-sdk's channel package and
meshvault, with go.mod requiring each at the group pin (nats: meshbroker too). The gate
holds the checks mesh-sdk dropped (an ask's id is its key's asking, at least one option),
with a row that fails without them; claude-code refuses a router record whose times do
not parse, as its JSON decode did, with a row.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 15:33:27 +01:00

583 lines
22 KiB
Go

package main
// A value no longer read (novox/hq ADR 0609, design 32 *A value no longer read*).
//
// A config value or secret that no manifest on any trunk reads any more is a **retiring value**. mesh-config learns it
// from the indexer's tool `indexer_config_readers`, asked on each of the indexer's `manifest-accepted` (Retirements);
// a tool the indexer does not serve yet answers "not known" and nothing retires. A value is marked retiring only once no
// node-engine reports applied a build that reads it (State draining until then, judged again on each `delivered`);
// mesh-config then says `retiring` and asks the operator once, through the operator channel, whether to remove it. It is
// removed only on the operator's Approve: a value by mesh-config, a secret through the seat `mesh-vault`'s verb
// `retire`. An ask unanswered after seven days is asked once more, and then written as a finding.
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
)
// The state, and where a retirement stands.
const (
RetiringState = "retiring"
// RetireVerb is the act an answer binds.
RetireVerb = Seat + ".retire"
RetireDraining = "draining" // a node still runs a build that reads it
RetireAsked = "asked" // the operator is asked
RetireUnanswered = "unanswered" // the ask ended with no choice
RetireApproved = "approved" // the operator approved; being removed
RetireDeclined = "declined" // the operator declined, or a person cancelled: kept until the value is read again
)
// Kinds of retiring value.
const (
KindValue = "value"
KindSecret = "secret"
)
// Retiring is one retiring value, as `retiring` keeps it, key `<module>.<key>`.
type Retiring struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
State string `json:"state"`
// Nodes are the nodes that still read it (draining) or that it is removed on (asked, approved).
Nodes []string `json:"nodes,omitempty"`
// Ask is the open ask's id, Asked when the first ask was made, AskedAgain whether it was asked once more.
Ask string `json:"ask,omitempty"`
// AskedAt is when the open ask was made (it expires a day on), Asked when the first was.
AskedAt time.Time `json:"asked-at,omitempty"`
Asked time.Time `json:"asked,omitempty"`
AskedAgain bool `json:"asked-again,omitempty"`
Said bool `json:"said,omitempty"`
Outcome string `json:"outcome,omitempty"`
Answered string `json:"answered,omitempty"`
// Removed are the nodes it is removed on already.
Removed []string `json:"removed,omitempty"`
}
func retiringKey(module, key string) string { return module + "." + key }
// RetiringEvent is the body of `retiring`; RetiredEvent of `retired`.
type RetiringEvent struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
Ask string `json:"ask"`
}
// RetiredEvent is the body of `retired`: one node's value removed, with the ask that warranted it.
type RetiredEvent struct {
Module string `json:"module"`
Name string `json:"name"`
Node string `json:"node"`
Ask string `json:"ask"`
}
// ReaderRow is one key the indexer's `indexer_config_readers` answers: the modules on each trunk that declare or refer
// to it. For a secret, Nodes are the nodes whose applied build reads it, when the indexer can say.
type ReaderRow struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Readers []string `json:"readers"`
Nodes []string `json:"nodes,omitempty"`
}
// Delivery is what the delivery parts answer to this module: a node's have, a node-engine's statement of what it
// reads, and who reads a module's config keys. Each is a tool or a seat verb on the bus, never a bucket (ADR 0596).
type Delivery interface {
// Have is the fingerprint of the build node runs of module, as the status view's node_haves says it.
Have(node, module string) (fp string, found bool, err error)
// Reads asks the node-engine seat's manifest-reads for node: an error when it is not served or the node says nothing.
Reads(node string) error
// Readers asks the indexer who reads module's keys: known false when the indexer does not serve the tool yet.
Readers(module string) (rows []ReaderRow, known bool, err error)
}
// RetireRequest is what the vault's verb `retire` takes: one module's own secret on a node, with the hand act that
// warrants it.
type RetireRequest struct {
Node, Module, Name, Ask string
}
// Vault is the seat mesh-vault as this module reaches it.
type Vault interface {
Retire(r RetireRequest) error
}
func (c *Config) readRetiring() (map[string]Retiring, error) {
out := map[string]Retiring{}
err := c.each(RetiringState, func(key string, raw []byte) error {
var r Retiring
if json.Unmarshal(raw, &r) != nil || r.Module == "" || retiringKey(r.Module, r.Key) != key {
return nil
}
out[key] = r
return nil
})
return out, err
}
// Retirements judges a module's keys on the indexer's word that its manifest was accepted. It answers a line for each
// decision.
func (c *Config) Retirements(module string) []string {
c.mu.Lock()
defer c.mu.Unlock()
if c.Delivery == nil || module == "" {
return nil
}
rows, known, err := c.Delivery.Readers(module)
if err != nil {
return []string{fmt.Sprintf("who reads %s's config is not asked yet: %v", module, err)}
}
if !known {
return []string{fmt.Sprintf("the indexer does not say who reads %s's config yet; nothing retires on that", module)}
}
held, err := c.readRetiring()
if err != nil {
return []string{"the retirements cannot be read: " + err.Error()}
}
s, err := c.sources()
if err != nil {
return []string{"the retirements cannot be judged: " + err.Error()}
}
var said []string
for _, row := range rows {
if row.Module != "" && row.Module != module {
continue
}
row.Module = module
if row.Kind == "" {
row.Kind = KindValue
}
key := retiringKey(module, row.Key)
e, has := held[key]
if len(row.Readers) > 0 {
if has {
said = append(said, c.dropRetiring(e, "it is read again"))
}
continue
}
if has {
continue // already marked; it is advanced by drainRetirements and Decide
}
if row.Kind == KindValue && !valueHeld(s, module, row.Key) {
continue // nothing set under that name: nothing to remove
}
if row.Kind == KindSecret && row.Nodes == nil && !row.nodesKnown() {
continue // which nodes read a secret is not known, and nothing retires on that
}
e = Retiring{Module: module, Key: row.Key, Kind: row.Kind, Since: c.now(), State: RetireDraining, Nodes: row.Nodes}
said = append(said, c.advanceRetiring(s, &e)...)
}
return said
}
func (r ReaderRow) nodesKnown() bool { return r.Nodes != nil }
// valueHeld is whether a module's own value is set under key on any node.
func valueHeld(s Sources, module, key string) bool {
for k := range s.Values {
if l, kk, ok := parseValueKey(k); ok && l.Kind == LevelModule && l.Module == module && kk == key {
return true
}
}
return false
}
// nodesReading is the nodes whose applied build still reads a value (secrets: the nodes the indexer last named).
func (c *Config) nodesReading(s Sources, e Retiring) []string {
if e.Kind != KindValue || c.Delivery == nil {
return e.Nodes
}
var out []string
for _, node := range nodesPlaced(&s, e.Module) {
fp, found, err := c.Delivery.Have(node, e.Module)
if err != nil || !found || fp == "" {
if err != nil {
out = append(out, node) // not known is not "no longer read"
}
continue
}
d, err := c.builtDeclarations(e.Module, fp)
if err != nil {
out = append(out, node)
continue
}
if _, reads := d.Config[e.Key]; reads {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// advanceRetiring moves a draining retirement on: while a node reads the value it stays draining (and is written, so a
// delivery judges it again); once none does it is marked retiring, said, and asked. The caller holds the lock.
func (c *Config) advanceRetiring(s Sources, e *Retiring) []string {
key := retiringKey(e.Module, e.Key)
now := c.now()
if e.State == RetireDraining {
e.Nodes = c.nodesReading(s, *e)
if len(e.Nodes) > 0 {
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)}
}
if now.Sub(e.Since) >= HeldFinding {
c.setFinding(retirementHeld(*e, now))
}
return []string{fmt.Sprintf("%s's %s is not read by any manifest, and %s still run a build that reads it",
e.Module, e.Key, strings.Join(e.Nodes, ", "))}
}
c.clearFinding(FindingRetireHeld, e.Module, e.Key)
e.Nodes = nodesPlaced(&s, e.Module)
if err := c.ask(e, false); err != nil {
return []string{fmt.Sprintf("the operator is not asked about retiring %s's %s: %v", e.Module, e.Key, err)}
}
}
if !e.Said {
if err := c.Store.Emit(EventRetiring, RetiringEvent{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since,
Ask: e.Ask}); err == nil {
e.Said = true
}
}
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)}
}
return []string{fmt.Sprintf("%s's %s is retiring: the operator is asked whether to remove it", e.Module, e.Key)}
}
// drainRetirements judges the draining retirements again, after a delivery or a returning node; the caller holds the lock.
func (c *Config) drainRetirements(string) []string {
held, err := c.readRetiring()
if err != nil || len(held) == 0 || c.Delivery == nil {
return nil
}
s, err := c.sources()
if err != nil {
return []string{"the retirements cannot be judged: " + err.Error()}
}
var said []string
for _, key := range sortedKeys(held) {
e := held[key]
if e.State == RetireDraining {
said = append(said, c.advanceRetiring(s, &e)...)
}
}
return said
}
// dropRetiring ends a retirement: the value is read again, or a person cancelled it.
func (c *Config) dropRetiring(e Retiring, why string) string {
if err := c.Store.Delete(RetiringState, retiringKey(e.Module, e.Key)); err != nil {
return fmt.Sprintf("the retirement of %s's %s is not dropped: %v", e.Module, e.Key, err)
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
c.clearFinding(FindingRetireHeld, e.Module, e.Key)
return fmt.Sprintf("%s's %s is no longer retiring: %s", e.Module, e.Key, why)
}
// ---- the ask ------------------------------------------------------------------------------------------------
// retireAskID is an ask's id: the same retirement asked again is another id.
func retireAskID(e Retiring, again bool) string {
sum := sha256.Sum256([]byte(e.Module + "\x00" + e.Key + "\x00" + e.Since.UTC().Format(time.RFC3339Nano)))
id := "ret-" + hex.EncodeToString(sum[:6])
if again {
id += "-2"
}
return id
}
// act is what an answer binds: the retirement, the nodes it reaches, the ask and the answer.
func (e Retiring) act(ask, option string) occ.Act {
return occ.Act{"verb": RetireVerb, "option": option, "module": e.Module, "key": e.Key, "kind": e.Kind,
"nodes": strings.Join(e.Nodes, " "), "ask": ask}
}
// retireAsk is the ask composed from the retirement alone.
func (e Retiring) retireAsk(id string, at time.Time) oc.Ask {
nodes := strings.Join(e.Nodes, ", ")
if nodes == "" {
nodes = "no node"
}
what := "config value"
if e.Kind == KindSecret {
what = "secret"
}
a := oc.Ask{ID: id, Headline: fmt.Sprintf("Remove %s's %s %s?", e.Module, what, e.Key),
Explanation: fmt.Sprintf("No manifest reads %s's %s %s any more, and no node runs a build that reads it. Approved, it is removed on %s.", e.Module, what, e.Key, nodes),
Who: occ.Operator, Expires: occ.FormatTime(at.Add(ProposalLasts)), OnExpiry: "nothing is removed; mesh-config asks once more in a week",
About: "config.retire." + e.Module + "." + e.Key,
Details: fmt.Sprintf("Retiring since %s UTC. Read it with mesh-config's config_retiring.", e.Since.UTC().Format("15:04 on 2 Jan"))}
for _, o := range []struct{ id, label, does string }{
{OptionApprove, "Approve", "the " + what + " is removed on " + nodes},
{OptionDecline, "Decline", "nothing is removed; mesh-config asks no more until it is read again"},
} {
binds, _ := occ.ActDigest(e.act(id, o.id))
a.Options = append(a.Options, oc.Option{ID: o.id, Label: o.label, Does: o.does, Level: occ.Approve, Binds: binds})
}
return a
}
// ask makes the ask for a retirement and publishes it; the caller keeps the entry.
func (c *Config) ask(e *Retiring, again bool) error {
if c.Asks == nil {
return fmt.Errorf("the operator cannot be asked from here")
}
now := c.now()
id := retireAskID(*e, again)
a := e.retireAsk(id, now)
if err := occ.Check(a, now); err != nil {
return err
}
if err := c.Asks.Publish(oc.AcceptSubjectAsk(Module), a, a.ID); err != nil {
return err
}
e.Ask, e.State, e.AskedAt = id, RetireAsked, now
if e.Asked.IsZero() {
e.Asked = now
}
e.AskedAgain = e.AskedAgain || again
return nil
}
// decideRetirements takes the router's word on each open retirement ask; the caller holds the lock. It also asks once
// more an ask unanswered for seven days, writes the finding of one asked twice, and finishes an approved removal.
func (c *Config) decideRetirements() []string {
held, err := c.readRetiring()
if err != nil || len(held) == 0 {
return nil
}
now := c.now()
var said []string
for _, key := range sortedKeys(held) {
e := held[key]
switch e.State {
case RetireAsked:
said = append(said, c.decideRetirement(&e, now)...)
case RetireUnanswered:
if !e.AskedAgain && now.Sub(e.Asked) >= HeldFinding {
if err := c.ask(&e, true); err != nil {
said = append(said, fmt.Sprintf("the operator is not asked again about %s's %s: %v", e.Module, e.Key, err))
} else {
_ = c.Store.Put(RetiringState, key, e)
said = append(said, fmt.Sprintf("%s's %s: the operator is asked once more", e.Module, e.Key))
}
}
case RetireApproved:
said = append(said, c.finishRetirement(&e)...)
}
}
return said
}
func (c *Config) decideRetirement(e *Retiring, now time.Time) []string {
if c.Asks == nil {
return nil
}
key := retiringKey(e.Module, e.Key)
rec, err := c.Asks.Record(e.Ask)
if err != nil || rec == nil {
return nil
}
var r routerRecord
if json.Unmarshal(rec.Value, &r) != nil || r.State == "open" || r.Warrant == nil {
return nil
}
w := *r.Warrant
keep := func(state, outcome string) []string {
e.State, e.Outcome = state, outcome
if err := c.Store.Put(RetiringState, key, *e); err != nil {
return []string{fmt.Sprintf("%s's %s: %s, and that is not kept: %v", e.Module, e.Key, outcome, err)}
}
return []string{fmt.Sprintf("%s's %s: %s", e.Module, e.Key, outcome)}
}
if w.Asker != Module || w.Ask != e.Ask {
return keep(RetireUnanswered, "the router's record is for another ask; nothing is removed")
}
if w.Outcome != occ.OutcomeChosen {
out := keep(RetireUnanswered, "the ask "+string(w.Outcome)+"; nothing is removed")
if e.AskedAgain {
c.setFinding(retireUnanswered(*e, now))
}
return out
}
option, err := occ.For(w, Module, e.retireAsk(e.Ask, e.AskedAt))
if err == nil {
err = occ.Performs(option, e.act(e.Ask, option.ID))
}
if err != nil {
c.logf("[mesh-config] REFUSED the answer to the retirement ask %s: %v", e.Ask, err)
return keep(RetireUnanswered, "the answer is refused and nothing is removed: "+err.Error())
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
e.Answered = occ.Says(w)
switch option.ID {
case OptionDecline:
return keep(RetireDeclined, "declined by the operator ("+occ.Says(w)+"); nothing is removed")
case OptionApprove:
out := keep(RetireApproved, "approved by the operator; being removed")
return append(out, c.finishRetirement(e)...)
}
return keep(RetireUnanswered, "the option "+option.ID+" is not one a retirement offers")
}
// finishRetirement removes an approved retirement on each node it is not removed on yet: a value here, a secret through
// the vault. Each removal is said (`retired`); the entry goes when every node is done. The caller holds the lock.
func (c *Config) finishRetirement(e *Retiring) []string {
key := retiringKey(e.Module, e.Key)
var said, failed []string
for _, node := range e.Nodes {
if slices.Contains(e.Removed, node) {
continue
}
var err error
switch e.Kind {
case KindValue:
var removed bool
if removed, err = c.removeValue(*e, node); err == nil && !removed {
e.Removed = append(e.Removed, node) // nothing was set there
continue
}
default:
if c.Vault == nil {
err = fmt.Errorf("the seat mesh-vault is not reachable from here")
} else {
err = c.Vault.Retire(RetireRequest{Node: node, Module: e.Module, Name: e.Key, Ask: e.Ask})
}
}
if err != nil {
failed = append(failed, node+": "+err.Error())
continue
}
e.Removed = append(e.Removed, node)
if err := c.Store.Emit(EventRetired, RetiredEvent{Module: e.Module, Name: e.Key, Node: node, Ask: e.Ask}); err != nil {
c.logf("[mesh-config] retired for %s's %s on %s is not said: %v", e.Module, e.Key, node, err)
}
said = append(said, fmt.Sprintf("%s's %s is removed on %s", e.Module, e.Key, node))
}
if len(failed) > 0 {
e.Outcome = "not removed everywhere yet: " + strings.Join(failed, "; ")
_ = c.Store.Put(RetiringState, key, *e)
return append(said, fmt.Sprintf("%s's %s: %s", e.Module, e.Key, e.Outcome))
}
if err := c.Store.Delete(RetiringState, key); err != nil {
return append(said, fmt.Sprintf("the retirement %s is not dropped: %v", key, err))
}
if _, err := c.reconcileLocked(); err != nil {
said = append(said, "resolving after the removal: "+err.Error())
}
return said
}
// removeValue deletes a module's own value on a node, recording it in the history, the hand-act log and config-changed.
func (c *Config) removeValue(e Retiring, node string) (bool, error) {
vk := moduleValueKey(node, e.Module, e.Key)
got, err := c.Store.Get(ValuesState, vk)
if err != nil {
return false, err
}
if got == nil {
return false, nil // nothing set there, or gone already
}
var was Value
_ = json.Unmarshal(got.Value, &was)
if err := c.Store.Delete(ValuesState, vk); err != nil {
return false, err
}
lvl := Level{Kind: LevelModule, Node: node, Module: e.Module}
ch := Change{Action: "retire", Level: &lvl, Key: e.Key, Was: was.Value, By: "the operator", Via: Module,
Why: "no manifest reads it any more (retiring), removed on the operator's Approve of ask " + e.Ask, At: c.now(),
Approved: e.Answered}
c.record(vk, ch)
if err := c.Store.HandAct(handActWhat(ch), ch.Why, "the operator's Approve of ask "+e.Ask); err != nil {
c.logf("[mesh-config] the hand-act log does not hold the retirement of %s: %v", vk, err)
}
return true, nil
}
// ---- the tool -----------------------------------------------------------------------------------------------
// RetiringRow is one retirement as the tool says it.
type RetiringRow struct {
Module string `json:"module"`
Key string `json:"key"`
Kind string `json:"kind"`
Since time.Time `json:"since"`
State string `json:"state"`
Nodes []string `json:"nodes,omitempty"`
Ask string `json:"ask,omitempty"`
Outcome string `json:"outcome,omitempty"`
Answered string `json:"answered,omitempty"`
}
// ConfigRetiring lists what is retiring (module narrows), or with action ask-again asks the operator again about one,
// or with cancel drops one (kept apart so it is not asked again until the value is read again).
func (c *Config) ConfigRetiring(action, module, key string) (any, error) {
c.mu.Lock()
defer c.mu.Unlock()
held, err := c.readRetiring()
if err != nil {
return nil, err
}
switch action {
case "", "list":
rows := []RetiringRow{}
for _, k := range sortedKeys(held) {
e := held[k]
if module != "" && e.Module != module {
continue
}
rows = append(rows, RetiringRow{Module: e.Module, Key: e.Key, Kind: e.Kind, Since: e.Since, State: e.State,
Nodes: e.Nodes, Ask: e.Ask, Outcome: e.Outcome, Answered: e.Answered})
}
return map[string]any{"retiring": rows}, nil
case "ask-again", "cancel":
e, ok := held[retiringKey(module, key)]
if !ok {
return nil, fmt.Errorf("%s's %s is not retiring: config_retiring lists what is", module, key)
}
if action == "cancel" {
e.State, e.Outcome = RetireDeclined, "cancelled by hand"
if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil {
return nil, err
}
c.clearFinding(FindingRetireUnanswered, e.Module, e.Key)
if err := c.Store.HandAct("config_retiring cancel "+module+" "+key, "a retirement cancelled by hand", "config_retiring"); err != nil {
c.logf("[mesh-config] the hand-act log does not hold the cancel: %v", err)
}
return map[string]any{"said": module + "'s " + key + " is no longer asked about until it is read again"}, nil
}
if e.State == RetireDraining || e.State == RetireApproved {
return nil, fmt.Errorf("%s's %s is %s: there is nothing to ask again", module, key, e.State)
}
s, err := c.sources()
if err != nil {
return nil, err
}
if nodes := c.nodesReading(s, e); len(nodes) > 0 {
return nil, fmt.Errorf("%s still run a build that reads %s's %s", strings.Join(nodes, ", "), module, key)
}
if err := c.ask(&e, true); err != nil {
return nil, err
}
if err := c.Store.Put(RetiringState, retiringKey(module, key), e); err != nil {
return nil, err
}
return map[string]any{"said": "the operator is asked again", "ask": e.Ask}, nil
}
return nil, fmt.Errorf("config_retiring lists, or does ask-again or cancel; %q is none of them", action)
}