diff --git a/modules/mesh-vault/cmd/mesh-vault/ledger.go b/modules/mesh-vault/cmd/mesh-vault/ledger.go index 61988608..0abadb43 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ledger.go +++ b/modules/mesh-vault/cmd/mesh-vault/ledger.go @@ -1,10 +1,18 @@ package main -// The custody ledger (novox/hq ADR 0085, design 24), carried over from the TypeScript vault as it was: -// for each pair credential the controller delivers to this vault as the `secret` provider, who holds it, -// since when, its fingerprint and every rotation — and never the value. The controller still mints and -// seals those until the secrets move to the vault (design 57's V3, V4); this pass records what it delivered, -// so `secret_holders` and `secret_verify` answer as before. Its files are the same JSON, read as they are. +// The custody ledger (novox/hq ADR 0085, design 24), carried over from the TypeScript vault: for each pair +// credential the controller delivers to this vault as the `secret` provider, who holds it, since when, its +// fingerprint and every rotation — and never the value. The controller still mints and seals those until +// the secrets move to the vault (design 57's V3, V4); the pass records what it delivered, so +// `secret_holders` and `secret_verify` answer as before. Its files are the same JSON, read as they are. +// +// **A holder the mesh stops asking for is retired, never deleted by a pass** (novox/hq ADR 0230). The vault +// is a mark-only provider: it has no access to switch off, so retiring writes the mark — when and why — into +// the holder's own ledger file, which keeps its rotation history. A holder is retired only once the same set +// has gone unasked in five consecutive passes that read the contributions file and for ten minutes; a set +// over the bound (more than three, or more than half of those held) waits for a person; asked for again, a +// retired holder is re-enabled at once; a holder is deleted only by `provisioner_delete`, a person's act +// with a why. Because the mark is in the ledger file, a restart forgets nothing. import ( "crypto/sha256" @@ -17,6 +25,7 @@ import ( "regexp" "sort" "strings" + "sync" "time" ) @@ -33,6 +42,14 @@ type Held struct { Changed string `json:"changed"` Rotations int `json:"rotations"` History []Earlier `json:"history"` + // Retired is the mark a retirement writes; nil while the holder is active. + Retired *RetiredMark `json:"retired,omitempty"` +} + +// RetiredMark is when and why a holder was retired. +type RetiredMark struct { + At string `json:"at"` + Why string `json:"why"` } // Earlier is a fingerprint a rotation replaced, and when. @@ -80,7 +97,7 @@ func (l *Ledger) pathOf(as string) (string, error) { } // Record says the mesh delivered value for as: the same value again changes nothing, another is a rotation. -// The value is fingerprinted here and goes no further. +// The value is fingerprinted here and goes no further. A retirement mark is left as it is. func (l *Ledger) Record(as, consumer, value string, now time.Time) (Held, Outcome, error) { path, err := l.pathOf(as) if err != nil { @@ -102,13 +119,31 @@ func (l *Ledger) Record(as, consumer, value string, now time.Time) (Held, Outcom return h, Rotated, l.write(path, h) } -// Withdraw forgets a holder, saying whether there was one. -func (l *Ledger) Withdraw(as string) bool { +// Mark writes or clears a holder's retirement mark. +func (l *Ledger) Mark(as string, mark *RetiredMark) error { + path, err := l.pathOf(as) + if err != nil { + return err + } + h, ok := l.Get(as) + if !ok { + return fmt.Errorf("the ledger holds nothing for %s", as) + } + h.Retired = mark + return l.write(path, h) +} + +// Delete forgets a holder, saying whether there was one. Only a person's `provisioner_delete` reaches it. +func (l *Ledger) Delete(as string) bool { path, err := l.pathOf(as) if err != nil { return false } - return os.Remove(path) == nil + if os.Remove(path) != nil { + return false + } + _ = syncDir(l.dir) + return true } // Get is one holder. @@ -163,19 +198,34 @@ type Contribution struct { } // Contributions is what the mesh asks this vault to serve: the entries with a login and a secret file. +// +// **A file that does not say who is asked for is not "nobody".** One that is unreadable, not JSON, without a +// `given` list (missing or null), or written for another requirement than `secret` is refused, so a pass +// over it retires nobody. An empty `given` list is an answer: nobody is asked for. func Contributions(receives string) ([]Contribution, error) { raw, err := os.ReadFile(receives) if err != nil { return nil, err } var doc struct { - Given []Contribution `json:"given"` + Requirement *string `json:"requirement"` + Given json.RawMessage `json:"given"` } if err := json.Unmarshal(raw, &doc); err != nil { - return nil, err + return nil, fmt.Errorf("the contributions file is not JSON: %w", err) + } + if doc.Requirement != nil && *doc.Requirement != "secret" { + return nil, fmt.Errorf("the contributions file is for %q, not for secret", *doc.Requirement) + } + if len(doc.Given) == 0 || string(doc.Given) == "null" { + return nil, errors.New("the contributions file has no `given` list, so it says nothing of who is asked for") + } + var given []Contribution + if err := json.Unmarshal(doc.Given, &given); err != nil { + return nil, fmt.Errorf("the contributions file's `given` is not a list of contributions: %w", err) } var out []Contribution - for _, g := range doc.Given { + for _, g := range given { if g.As != "" && g.Secret != "" { out = append(out, g) } @@ -201,23 +251,58 @@ func DeliveredFingerprint(c Contribution) (string, int, error) { return Fingerprint(v), len(v), nil } -// Custody is the pass over the contributions file that keeps the ledger. +// EventRetirement is the one event a provider says about retirement (novox/hq ADR 0230 §35); the controller +// derives the permission to emit it for every module that receives contributions. +const EventRetirement = "provisioner.retirement" + +// The rules of ADR 0230 §15 and §18. +const ( + StablePasses = 5 + StableFor = 10 * time.Minute + RetireAtOnce = 3 + RetireFraction = 0.5 + SayAgainEvery = time.Hour +) + +// Custody is the pass over the contributions file that keeps the ledger, and what it remembers between +// passes about the holders the mesh no longer asks for. type Custody struct { Ledger *Ledger Receives string Emit func(event string, body map[string]any) error Now func() time.Time Log func(format string, args ...any) + + mu sync.Mutex + key string + count int + firstSeen time.Time + asked map[string]bool + read bool + waiting *heldSet + rejected *heldSet } -// Pass records every delivered value and withdraws every holder no longer asked for — but only from a -// contributions file it could read whole: an unreadable one withdraws nobody. -func (c Custody) Pass() { +type heldSet struct { + set []string + since time.Time + saidAt time.Time + by, why string +} + +// Pass records every delivered value, re-enables a retired holder asked for again, and counts the holders +// no longer asked for toward their retirement. +func (c *Custody) Pass() { + c.mu.Lock() + defer c.mu.Unlock() asked, err := Contributions(c.Receives) if err != nil { - c.Log("the contributions file is not readable (%s): %v — nothing withdrawn", c.Receives, err) + c.Log("THE CONTRIBUTIONS FILE IS NOT USABLE (%s): %v — nothing is recorded or retired on this pass, and "+ + "the count toward retiring anyone starts again", c.Receives, err) + c.key, c.count = "", 0 return } + now := c.Now() keep := map[string]bool{} for _, a := range asked { keep[a.As] = true @@ -226,11 +311,19 @@ func (c Custody) Pass() { c.Log("%s: %v", a.As, err) continue } - held, outcome, err := c.Ledger.Record(a.As, a.Node, value, c.Now()) + held, outcome, err := c.Ledger.Record(a.As, a.Node, value, now) if err != nil { c.Log("%s: %v", a.As, err) continue } + if held.Retired != nil { + if err := c.Ledger.Mark(a.As, nil); err != nil { + c.Log("%s: asked for again, and its retirement mark could not be cleared: %v", a.As, err) + } else { + c.Log("%s: asked for again — re-enabled, its retirement mark cleared", a.As) + c.say(EventRetirement, map[string]any{"change": "reenabled", "consumers": []string{a.As}}) + } + } if outcome == Unchanged { continue } @@ -241,15 +334,195 @@ func (c Custody) Pass() { c.Log("%s: %s (%s…, rotations %d)", held.As, outcome, held.Fingerprint[:19], held.Rotations) c.say(event, map[string]any{"consumer": held.Consumer, "as": held.As, "fingerprint": held.Fingerprint, "rotations": held.Rotations}) } - for _, h := range c.Ledger.List() { - if !keep[h.As] && c.Ledger.Withdraw(h.As) { - c.Log("%s: withdrawn", h.As) - c.say("deprovisioned", map[string]any{"as": h.As}) - } - } + c.asked, c.read = keep, true + c.countUnasked(now) } -func (c Custody) say(event string, body map[string]any) { +func (c *Custody) active() []Held { + var out []Held + for _, h := range c.Ledger.List() { + if h.Retired == nil { + out = append(out, h) + } + } + return out +} + +func (c *Custody) countUnasked(now time.Time) { + active := c.active() + var unasked []string + for _, h := range active { + if !c.asked[h.As] { + unasked = append(unasked, h.As) + } + } + key := strings.Join(unasked, ",") + switch { + case len(unasked) == 0: + c.settle("every holder is asked for again") + c.key, c.count = "", 0 + return + case key != c.key: + if c.waiting != nil || c.rejected != nil { + c.settle("the set the mesh no longer asks for changed") + } + c.key, c.count, c.firstSeen = key, 1, now + c.Log("no longer asked for: %s; retired once the same set holds for %d passes and %s (novox/hq ADR 0230)", + key, StablePasses, StableFor) + default: + c.count++ + } + if c.rejected != nil || c.count < StablePasses || now.Sub(c.firstSeen) < StableFor { + return + } + if overTheBound(len(unasked), len(active)) { + c.wait(unasked, len(active), now) + return + } + c.retire(unasked, fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s", + c.count, now.Sub(c.firstSeen).Round(time.Second)), now) +} + +func overTheBound(n, held int) bool { + return n > RetireAtOnce || (held > 1 && float64(n) > RetireFraction*float64(held)) +} + +func (c *Custody) wait(set []string, held int, now time.Time) { + if c.waiting == nil { + c.waiting = &heldSet{set: set, since: now} + c.Log("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d holder(s) (%s), more than %d "+ + "at once or more than half. Nothing is retired until provisioner_retire_approve or "+ + "provisioner_retire_reject (novox/hq ADR 0230)", len(set), held, strings.Join(set, ", "), RetireAtOnce) + } else if now.Sub(c.waiting.saidAt) < SayAgainEvery { + return + } + c.waiting.saidAt = now + c.say(EventRetirement, map[string]any{"change": "waiting", "consumers": set, "held": held, + "since": c.waiting.since.UTC().Format(time.RFC3339)}) +} + +func (c *Custody) settle(why string) { + var set []string + switch { + case c.waiting != nil: + set = c.waiting.set + case c.rejected != nil: + set = c.rejected.set + default: + return + } + c.waiting, c.rejected = nil, nil + c.Log("the set waiting on retirement (%s) is settled: %s", strings.Join(set, ", "), why) + c.say(EventRetirement, map[string]any{"change": "settled", "consumers": set, "why": why}) +} + +// retire marks each holder in set retired: the ledger keeps it, and its history, until a person deletes it. +func (c *Custody) retire(set []string, why string, now time.Time) { + at := now.UTC().Format(isoMillis) + var done []string + for _, as := range set { + if err := c.Ledger.Mark(as, &RetiredMark{At: at, Why: why}); err != nil { + c.Log("%s: could not be marked retired: %v", as, err) + continue + } + done = append(done, as) + } + c.key, c.count, c.waiting = "", 0, nil + if len(done) == 0 { + return + } + c.Log("retired, kept in the ledger until a person deletes it: %s — %s", strings.Join(done, ", "), why) + c.say(EventRetirement, map[string]any{"change": "retired", "consumers": done, "why": why}) +} + +// Approve retires exactly the set waiting (or rejected) for a person. +func (c *Custody) Approve(by, why string) ([]string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if strings.TrimSpace(why) == "" { + return nil, errors.New("an approval says why") + } + set := c.waiting + if set == nil { + set = c.rejected + } + if set == nil { + return nil, errors.New("no set waits for retirement") + } + c.say(EventRetirement, map[string]any{"change": "approved", "consumers": set.set, "by": by, "why": why}) + members := set.set + c.rejected = nil + c.retire(members, "approved by "+by+": "+why, c.Now()) + return members, nil +} + +// Reject keeps the waiting set active, and asks no more about it while it stays the same set. +func (c *Custody) Reject(by, why string) ([]string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if strings.TrimSpace(why) == "" { + return nil, errors.New("a rejection says why") + } + if c.waiting == nil { + return nil, errors.New("no set waits for retirement") + } + c.rejected = &heldSet{set: c.waiting.set, since: c.Now(), by: by, why: why} + c.waiting = nil + c.Log("retirement of %s rejected by %s: %s", strings.Join(c.rejected.set, ", "), by, why) + c.say(EventRetirement, map[string]any{"change": "rejected", "consumers": c.rejected.set, "by": by, "why": why}) + return c.rejected.set, nil +} + +// Delete forgets one retired holder, on a person's word. Refused: an active holder, one the mesh asks for, +// and any without a why. +func (c *Custody) Delete(as, by, why string) error { + c.mu.Lock() + defer c.mu.Unlock() + if strings.TrimSpace(why) == "" { + return errors.New("a deletion says why") + } + h, ok := c.Ledger.Get(as) + if !ok { + return fmt.Errorf("the ledger holds nothing for %s", as) + } + if h.Retired == nil { + return fmt.Errorf("%s is active, and only a retired holder is deleted (novox/hq ADR 0230 §31)", as) + } + if c.asked[as] { + return fmt.Errorf("%s is asked for, and is not deleted", as) + } + if !c.Ledger.Delete(as) { + return fmt.Errorf("%s could not be deleted", as) + } + c.Log("%s deleted from the ledger by %s: %s", as, by, why) + c.say(EventRetirement, map[string]any{"change": "deleted", "consumers": []string{as}, "by": by, "why": why}) + return nil +} + +// Retirement is what is held, waiting, rejected and retired. +func (c *Custody) Retirement() map[string]any { + c.mu.Lock() + defer c.mu.Unlock() + active, retired := []string{}, []map[string]any{} + for _, h := range c.Ledger.List() { + if h.Retired == nil { + active = append(active, h.As) + } else { + retired = append(retired, map[string]any{"consumer": h.As, "node": h.Consumer, "retired_at": h.Retired.At, "why": h.Retired.Why}) + } + } + out := map[string]any{"active": active, "retired": retired, "stable_passes": StablePasses, + "stable_for_seconds": int(StableFor.Seconds()), "counting": c.key, "passes": c.count, "waiting": nil, "rejected": nil} + if c.waiting != nil { + out["waiting"] = map[string]any{"consumers": c.waiting.set, "since": c.waiting.since.UTC().Format(time.RFC3339)} + } + if c.rejected != nil { + out["rejected"] = map[string]any{"consumers": c.rejected.set, "by": c.rejected.by, "why": c.rejected.why} + } + return out +} + +func (c *Custody) say(event string, body map[string]any) { if c.Emit == nil { return } diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index eb7d9458..35a66b7b 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -8,7 +8,6 @@ package main import ( - "errors" "fmt" "os" "time" @@ -30,10 +29,18 @@ func build() *Env { } else { say("%v", err) } - store, err1 := OpenStore(os.Getenv("MESH_VAULT_STORE")) - own, err2 := OwnKey(os.Getenv("MESH_VAULT_STATE")) - if err := errors.Join(err1, err2); err != nil { - env.Broken = fmt.Errorf("the vault could not open its store or its key: %w", err) + if env.Ledger != nil && env.Receives != "" { + env.Custody = &Custody{Ledger: env.Ledger, Receives: env.Receives, Emit: emit, Now: time.Now, Log: say} + } + store, err := OpenStore(os.Getenv("MESH_VAULT_STORE")) + if err != nil { + env.Broken = fmt.Errorf("the vault could not open its store: %w", err) + say("%v", env.Broken) + return env + } + own, err := OwnKey(os.Getenv("MESH_VAULT_STATE"), store) + if err != nil { + env.Broken = fmt.Errorf("the vault could not open its key: %w", err) say("%v", env.Broken) return env } @@ -43,30 +50,36 @@ func build() *Env { } // adoptOperatorKey takes the operator key from the export the mesh writes to the vault's root (what -// `operator key set` recorded), when it differs from the one the store seals to. +// `operator key set` recorded; the controller keeps only a public half there), and seals again any +// operator copy not yet sealed to the current key. Run every minute, so a failure partway is finished. func adoptOperatorKey(env *Env) { - if env.Vault == nil || env.Root == "" { + if env.Vault == nil { return } - key := exportedOperatorKey(env.Root) - if key == "" { + if raw := exportedOperatorKey(env.Root); env.Root != "" && raw != "" { + key, err := publicPEM(raw) + if err == nil { + _, err = env.Vault.SetOperatorKey(key, "the mesh's export") + } + if err != nil { + say("the operator key in the mesh's export is not taken, or not every copy is sealed to it yet: %v", err) + } return } - if _, err := env.Vault.SetOperatorKey(key, "the mesh's export"); err != nil { - say("the operator key in the mesh's export is not taken: %v", err) + if _, err := env.Vault.Reseal(); err != nil { + say("%v", err) } } func main() { env := build() go func() { - custody := Custody{Ledger: env.Ledger, Receives: env.Receives, Emit: emit, Now: time.Now, Log: say} tick := time.NewTicker(time.Minute) defer tick.Stop() for { adoptOperatorKey(env) - if env.Ledger != nil && env.Receives != "" { - custody.Pass() + if env.Custody != nil { + env.Custody.Pass() } <-tick.C }