mesh-vault: retire a holder by ADR 0230, never delete it on a pass; refuse a contributions file that says nothing (review 250)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

modules/mesh-vault/cmd/mesh-vault/ledger.go

@@ -1,10 +1,18 @@
 package main
 
-// The custody ledger (novox/hq ADR 0085, design 24), carried over from the TypeScript vault as it was:
-// for each pair credential the controller delivers to this vault as the `secret` provider, who holds it,
-// since when, its fingerprint and every rotation — and never the value. The controller still mints and
-// seals those until the secrets move to the vault (design 57's V3, V4); this pass records what it delivered,
-// so `secret_holders` and `secret_verify` answer as before. Its files are the same JSON, read as they are.
+// The custody ledger (novox/hq ADR 0085, design 24), carried over from the TypeScript vault: for each pair
+// credential the controller delivers to this vault as the `secret` provider, who holds it, since when, its
+// fingerprint and every rotation — and never the value. The controller still mints and seals those until
+// the secrets move to the vault (design 57's V3, V4); the pass records what it delivered, so
+// `secret_holders` and `secret_verify` answer as before. Its files are the same JSON, read as they are.
+//
+// **A holder the mesh stops asking for is retired, never deleted by a pass** (novox/hq ADR 0230). The vault
+// is a mark-only provider: it has no access to switch off, so retiring writes the mark — when and why — into
+// the holder's own ledger file, which keeps its rotation history. A holder is retired only once the same set
+// has gone unasked in five consecutive passes that read the contributions file and for ten minutes; a set
+// over the bound (more than three, or more than half of those held) waits for a person; asked for again, a
+// retired holder is re-enabled at once; a holder is deleted only by `provisioner_delete`, a person's act
+// with a why. Because the mark is in the ledger file, a restart forgets nothing.
 
 import (
 	"crypto/sha256"
@@ -17,6 +25,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -33,6 +42,14 @@ type Held struct {
 	Changed     string    `json:"changed"`
 	Rotations   int       `json:"rotations"`
 	History     []Earlier `json:"history"`
+	// Retired is the mark a retirement writes; nil while the holder is active.
+	Retired *RetiredMark `json:"retired,omitempty"`
+}
+
+// RetiredMark is when and why a holder was retired.
+type RetiredMark struct {
+	At  string `json:"at"`
+	Why string `json:"why"`
 }
 
 // Earlier is a fingerprint a rotation replaced, and when.
@@ -80,7 +97,7 @@ func (l *Ledger) pathOf(as string) (string, error) {
 }
 
 // Record says the mesh delivered value for as: the same value again changes nothing, another is a rotation.
-// The value is fingerprinted here and goes no further.
+// The value is fingerprinted here and goes no further. A retirement mark is left as it is.
 func (l *Ledger) Record(as, consumer, value string, now time.Time) (Held, Outcome, error) {
 	path, err := l.pathOf(as)
 	if err != nil {
@@ -102,13 +119,31 @@ func (l *Ledger) Record(as, consumer, value string, now time.Time) (Held, Outcom
 	return h, Rotated, l.write(path, h)
 }
 
-// Withdraw forgets a holder, saying whether there was one.
-func (l *Ledger) Withdraw(as string) bool {
+// Mark writes or clears a holder's retirement mark.
+func (l *Ledger) Mark(as string, mark *RetiredMark) error {
+	path, err := l.pathOf(as)
+	if err != nil {
+		return err
+	}
+	h, ok := l.Get(as)
+	if !ok {
+		return fmt.Errorf("the ledger holds nothing for %s", as)
+	}
+	h.Retired = mark
+	return l.write(path, h)
+}
+
+// Delete forgets a holder, saying whether there was one. Only a person's `provisioner_delete` reaches it.
+func (l *Ledger) Delete(as string) bool {
 	path, err := l.pathOf(as)
 	if err != nil {
 		return false
 	}
-	return os.Remove(path) == nil
+	if os.Remove(path) != nil {
+		return false
+	}
+	_ = syncDir(l.dir)
+	return true
 }
 
 // Get is one holder.
@@ -163,19 +198,34 @@ type Contribution struct {
 }
 
 // Contributions is what the mesh asks this vault to serve: the entries with a login and a secret file.
+//
+// **A file that does not say who is asked for is not "nobody".** One that is unreadable, not JSON, without a
+// `given` list (missing or null), or written for another requirement than `secret` is refused, so a pass
+// over it retires nobody. An empty `given` list is an answer: nobody is asked for.
 func Contributions(receives string) ([]Contribution, error) {
 	raw, err := os.ReadFile(receives)
 	if err != nil {
 		return nil, err
 	}
 	var doc struct {
-		Given []Contribution `json:"given"`
+		Requirement *string         `json:"requirement"`
+		Given       json.RawMessage `json:"given"`
 	}
 	if err := json.Unmarshal(raw, &doc); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("the contributions file is not JSON: %w", err)
+	}
+	if doc.Requirement != nil && *doc.Requirement != "secret" {
+		return nil, fmt.Errorf("the contributions file is for %q, not for secret", *doc.Requirement)
+	}
+	if len(doc.Given) == 0 || string(doc.Given) == "null" {
+		return nil, errors.New("the contributions file has no `given` list, so it says nothing of who is asked for")
+	}
+	var given []Contribution
+	if err := json.Unmarshal(doc.Given, &given); err != nil {
+		return nil, fmt.Errorf("the contributions file's `given` is not a list of contributions: %w", err)
 	}
 	var out []Contribution
-	for _, g := range doc.Given {
+	for _, g := range given {
 		if g.As != "" && g.Secret != "" {
 			out = append(out, g)
 		}
@@ -201,23 +251,58 @@ func DeliveredFingerprint(c Contribution) (string, int, error) {
 	return Fingerprint(v), len(v), nil
 }
 
-// Custody is the pass over the contributions file that keeps the ledger.
+// EventRetirement is the one event a provider says about retirement (novox/hq ADR 0230 §35); the controller
+// derives the permission to emit it for every module that receives contributions.
+const EventRetirement = "provisioner.retirement"
+
+// The rules of ADR 0230 §15 and §18.
+const (
+	StablePasses   = 5
+	StableFor      = 10 * time.Minute
+	RetireAtOnce   = 3
+	RetireFraction = 0.5
+	SayAgainEvery  = time.Hour
+)
+
+// Custody is the pass over the contributions file that keeps the ledger, and what it remembers between
+// passes about the holders the mesh no longer asks for.
 type Custody struct {
 	Ledger   *Ledger
 	Receives string
 	Emit     func(event string, body map[string]any) error
 	Now      func() time.Time
 	Log      func(format string, args ...any)
+
+	mu        sync.Mutex
+	key       string
+	count     int
+	firstSeen time.Time
+	asked     map[string]bool
+	read      bool
+	waiting   *heldSet
+	rejected  *heldSet
 }
 
-// Pass records every delivered value and withdraws every holder no longer asked for — but only from a
-// contributions file it could read whole: an unreadable one withdraws nobody.
-func (c Custody) Pass() {
+type heldSet struct {
+	set     []string
+	since   time.Time
+	saidAt  time.Time
+	by, why string
+}
+
+// Pass records every delivered value, re-enables a retired holder asked for again, and counts the holders
+// no longer asked for toward their retirement.
+func (c *Custody) Pass() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
 	asked, err := Contributions(c.Receives)
 	if err != nil {
-		c.Log("the contributions file is not readable (%s): %v — nothing withdrawn", c.Receives, err)
+		c.Log("THE CONTRIBUTIONS FILE IS NOT USABLE (%s): %v — nothing is recorded or retired on this pass, and "+
+			"the count toward retiring anyone starts again", c.Receives, err)
+		c.key, c.count = "", 0
 		return
 	}
+	now := c.Now()
 	keep := map[string]bool{}
 	for _, a := range asked {
 		keep[a.As] = true
@@ -226,11 +311,19 @@ func (c Custody) Pass() {
 			c.Log("%s: %v", a.As, err)
 			continue
 		}
-		held, outcome, err := c.Ledger.Record(a.As, a.Node, value, c.Now())
+		held, outcome, err := c.Ledger.Record(a.As, a.Node, value, now)
 		if err != nil {
 			c.Log("%s: %v", a.As, err)
 			continue
 		}
+		if held.Retired != nil {
+			if err := c.Ledger.Mark(a.As, nil); err != nil {
+				c.Log("%s: asked for again, and its retirement mark could not be cleared: %v", a.As, err)
+			} else {
+				c.Log("%s: asked for again — re-enabled, its retirement mark cleared", a.As)
+				c.say(EventRetirement, map[string]any{"change": "reenabled", "consumers": []string{a.As}})
+			}
+		}
 		if outcome == Unchanged {
 			continue
 		}
@@ -241,15 +334,195 @@ func (c Custody) Pass() {
 		c.Log("%s: %s (%s…, rotations %d)", held.As, outcome, held.Fingerprint[:19], held.Rotations)
 		c.say(event, map[string]any{"consumer": held.Consumer, "as": held.As, "fingerprint": held.Fingerprint, "rotations": held.Rotations})
 	}
-	for _, h := range c.Ledger.List() {
-		if !keep[h.As] && c.Ledger.Withdraw(h.As) {
-			c.Log("%s: withdrawn", h.As)
-			c.say("deprovisioned", map[string]any{"as": h.As})
-		}
-	}
+	c.asked, c.read = keep, true
+	c.countUnasked(now)
 }
 
-func (c Custody) say(event string, body map[string]any) {
+func (c *Custody) active() []Held {
+	var out []Held
+	for _, h := range c.Ledger.List() {
+		if h.Retired == nil {
+			out = append(out, h)
+		}
+	}
+	return out
+}
+
+func (c *Custody) countUnasked(now time.Time) {
+	active := c.active()
+	var unasked []string
+	for _, h := range active {
+		if !c.asked[h.As] {
+			unasked = append(unasked, h.As)
+		}
+	}
+	key := strings.Join(unasked, ",")
+	switch {
+	case len(unasked) == 0:
+		c.settle("every holder is asked for again")
+		c.key, c.count = "", 0
+		return
+	case key != c.key:
+		if c.waiting != nil || c.rejected != nil {
+			c.settle("the set the mesh no longer asks for changed")
+		}
+		c.key, c.count, c.firstSeen = key, 1, now
+		c.Log("no longer asked for: %s; retired once the same set holds for %d passes and %s (novox/hq ADR 0230)",
+			key, StablePasses, StableFor)
+	default:
+		c.count++
+	}
+	if c.rejected != nil || c.count < StablePasses || now.Sub(c.firstSeen) < StableFor {
+		return
+	}
+	if overTheBound(len(unasked), len(active)) {
+		c.wait(unasked, len(active), now)
+		return
+	}
+	c.retire(unasked, fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s",
+		c.count, now.Sub(c.firstSeen).Round(time.Second)), now)
+}
+
+func overTheBound(n, held int) bool {
+	return n > RetireAtOnce || (held > 1 && float64(n) > RetireFraction*float64(held))
+}
+
+func (c *Custody) wait(set []string, held int, now time.Time) {
+	if c.waiting == nil {
+		c.waiting = &heldSet{set: set, since: now}
+		c.Log("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d holder(s) (%s), more than %d "+
+			"at once or more than half. Nothing is retired until provisioner_retire_approve or "+
+			"provisioner_retire_reject (novox/hq ADR 0230)", len(set), held, strings.Join(set, ", "), RetireAtOnce)
+	} else if now.Sub(c.waiting.saidAt) < SayAgainEvery {
+		return
+	}
+	c.waiting.saidAt = now
+	c.say(EventRetirement, map[string]any{"change": "waiting", "consumers": set, "held": held,
+		"since": c.waiting.since.UTC().Format(time.RFC3339)})
+}
+
+func (c *Custody) settle(why string) {
+	var set []string
+	switch {
+	case c.waiting != nil:
+		set = c.waiting.set
+	case c.rejected != nil:
+		set = c.rejected.set
+	default:
+		return
+	}
+	c.waiting, c.rejected = nil, nil
+	c.Log("the set waiting on retirement (%s) is settled: %s", strings.Join(set, ", "), why)
+	c.say(EventRetirement, map[string]any{"change": "settled", "consumers": set, "why": why})
+}
+
+// retire marks each holder in set retired: the ledger keeps it, and its history, until a person deletes it.
+func (c *Custody) retire(set []string, why string, now time.Time) {
+	at := now.UTC().Format(isoMillis)
+	var done []string
+	for _, as := range set {
+		if err := c.Ledger.Mark(as, &RetiredMark{At: at, Why: why}); err != nil {
+			c.Log("%s: could not be marked retired: %v", as, err)
+			continue
+		}
+		done = append(done, as)
+	}
+	c.key, c.count, c.waiting = "", 0, nil
+	if len(done) == 0 {
+		return
+	}
+	c.Log("retired, kept in the ledger until a person deletes it: %s — %s", strings.Join(done, ", "), why)
+	c.say(EventRetirement, map[string]any{"change": "retired", "consumers": done, "why": why})
+}
+
+// Approve retires exactly the set waiting (or rejected) for a person.
+func (c *Custody) Approve(by, why string) ([]string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if strings.TrimSpace(why) == "" {
+		return nil, errors.New("an approval says why")
+	}
+	set := c.waiting
+	if set == nil {
+		set = c.rejected
+	}
+	if set == nil {
+		return nil, errors.New("no set waits for retirement")
+	}
+	c.say(EventRetirement, map[string]any{"change": "approved", "consumers": set.set, "by": by, "why": why})
+	members := set.set
+	c.rejected = nil
+	c.retire(members, "approved by "+by+": "+why, c.Now())
+	return members, nil
+}
+
+// Reject keeps the waiting set active, and asks no more about it while it stays the same set.
+func (c *Custody) Reject(by, why string) ([]string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if strings.TrimSpace(why) == "" {
+		return nil, errors.New("a rejection says why")
+	}
+	if c.waiting == nil {
+		return nil, errors.New("no set waits for retirement")
+	}
+	c.rejected = &heldSet{set: c.waiting.set, since: c.Now(), by: by, why: why}
+	c.waiting = nil
+	c.Log("retirement of %s rejected by %s: %s", strings.Join(c.rejected.set, ", "), by, why)
+	c.say(EventRetirement, map[string]any{"change": "rejected", "consumers": c.rejected.set, "by": by, "why": why})
+	return c.rejected.set, nil
+}
+
+// Delete forgets one retired holder, on a person's word. Refused: an active holder, one the mesh asks for,
+// and any without a why.
+func (c *Custody) Delete(as, by, why string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if strings.TrimSpace(why) == "" {
+		return errors.New("a deletion says why")
+	}
+	h, ok := c.Ledger.Get(as)
+	if !ok {
+		return fmt.Errorf("the ledger holds nothing for %s", as)
+	}
+	if h.Retired == nil {
+		return fmt.Errorf("%s is active, and only a retired holder is deleted (novox/hq ADR 0230 §31)", as)
+	}
+	if c.asked[as] {
+		return fmt.Errorf("%s is asked for, and is not deleted", as)
+	}
+	if !c.Ledger.Delete(as) {
+		return fmt.Errorf("%s could not be deleted", as)
+	}
+	c.Log("%s deleted from the ledger by %s: %s", as, by, why)
+	c.say(EventRetirement, map[string]any{"change": "deleted", "consumers": []string{as}, "by": by, "why": why})
+	return nil
+}
+
+// Retirement is what is held, waiting, rejected and retired.
+func (c *Custody) Retirement() map[string]any {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	active, retired := []string{}, []map[string]any{}
+	for _, h := range c.Ledger.List() {
+		if h.Retired == nil {
+			active = append(active, h.As)
+		} else {
+			retired = append(retired, map[string]any{"consumer": h.As, "node": h.Consumer, "retired_at": h.Retired.At, "why": h.Retired.Why})
+		}
+	}
+	out := map[string]any{"active": active, "retired": retired, "stable_passes": StablePasses,
+		"stable_for_seconds": int(StableFor.Seconds()), "counting": c.key, "passes": c.count, "waiting": nil, "rejected": nil}
+	if c.waiting != nil {
+		out["waiting"] = map[string]any{"consumers": c.waiting.set, "since": c.waiting.since.UTC().Format(time.RFC3339)}
+	}
+	if c.rejected != nil {
+		out["rejected"] = map[string]any{"consumers": c.rejected.set, "by": c.rejected.by, "why": c.rejected.why}
+	}
+	return out
+}
+
+func (c *Custody) say(event string, body map[string]any) {
 	if c.Emit == nil {
 		return
 	}

modules/mesh-vault/cmd/mesh-vault/main.go

@@ -8,7 +8,6 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"time"
@@ -30,10 +29,18 @@ func build() *Env {
 	} else {
 		say("%v", err)
 	}
-	store, err1 := OpenStore(os.Getenv("MESH_VAULT_STORE"))
-	own, err2 := OwnKey(os.Getenv("MESH_VAULT_STATE"))
-	if err := errors.Join(err1, err2); err != nil {
-		env.Broken = fmt.Errorf("the vault could not open its store or its key: %w", err)
+	if env.Ledger != nil && env.Receives != "" {
+		env.Custody = &Custody{Ledger: env.Ledger, Receives: env.Receives, Emit: emit, Now: time.Now, Log: say}
+	}
+	store, err := OpenStore(os.Getenv("MESH_VAULT_STORE"))
+	if err != nil {
+		env.Broken = fmt.Errorf("the vault could not open its store: %w", err)
+		say("%v", env.Broken)
+		return env
+	}
+	own, err := OwnKey(os.Getenv("MESH_VAULT_STATE"), store)
+	if err != nil {
+		env.Broken = fmt.Errorf("the vault could not open its key: %w", err)
 		say("%v", env.Broken)
 		return env
 	}
@@ -43,30 +50,36 @@ func build() *Env {
 }
 
 // adoptOperatorKey takes the operator key from the export the mesh writes to the vault's root (what
-// `operator key set` recorded), when it differs from the one the store seals to.
+// `operator key set` recorded; the controller keeps only a public half there), and seals again any
+// operator copy not yet sealed to the current key. Run every minute, so a failure partway is finished.
 func adoptOperatorKey(env *Env) {
-	if env.Vault == nil || env.Root == "" {
+	if env.Vault == nil {
 		return
 	}
-	key := exportedOperatorKey(env.Root)
-	if key == "" {
+	if raw := exportedOperatorKey(env.Root); env.Root != "" && raw != "" {
+		key, err := publicPEM(raw)
+		if err == nil {
+			_, err = env.Vault.SetOperatorKey(key, "the mesh's export")
+		}
+		if err != nil {
+			say("the operator key in the mesh's export is not taken, or not every copy is sealed to it yet: %v", err)
+		}
 		return
 	}
-	if _, err := env.Vault.SetOperatorKey(key, "the mesh's export"); err != nil {
-		say("the operator key in the mesh's export is not taken: %v", err)
+	if _, err := env.Vault.Reseal(); err != nil {
+		say("%v", err)
 	}
 }
 
 func main() {
 	env := build()
 	go func() {
-		custody := Custody{Ledger: env.Ledger, Receives: env.Receives, Emit: emit, Now: time.Now, Log: say}
 		tick := time.NewTicker(time.Minute)
 		defer tick.Stop()
 		for {
 			adoptOperatorKey(env)
-			if env.Ledger != nil && env.Receives != "" {
-				custody.Pass()
+			if env.Custody != nil {
+				env.Custody.Pass()
 			}
 			<-tick.C
 		}