mesh-vault: ADR 0230's rules pinned one at a time, and the words held to the controller's types (review 255)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

modules/mesh-vault/cmd/mesh-vault/retirement_test.go

@@ -0,0 +1,537 @@
+package main
+
+// ADR 0230 in the vault's custody ledger, and its words held to the controller's: testdata/retirement-*.json
+// are mesh-controller's link.Retirement and link.RetirementState marshalled at dc36536 (feat/57, with #317).
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+)
+
+// custodyAt is a custody pass over a contributions file in dir, on a clock the test moves.
+type custodyAt struct {
+	*Custody
+	bus      *published
+	receives string
+	dir      string
+	clock    *time.Time
+}
+
+func newCustody(t *testing.T) custodyAt {
+	dir := t.TempDir()
+	l, _ := NewLedger(filepath.Join(dir, "ledger"))
+	bus := &published{}
+	clock := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC)
+	c := custodyAt{bus: bus, receives: filepath.Join(dir, "mesh.json"), dir: dir, clock: &clock}
+	c.Custody = &Custody{Ledger: l, Receives: c.receives, Resource: "secret", Node: "novox", Emit: bus.emit,
+		Now: func() time.Time { return *c.clock }, Log: t.Logf}
+	return c
+}
+
+// ask writes a contributions file asking for these logins, each with its value.
+func (c custodyAt) ask(values map[string]string) {
+	given := []map[string]string{}
+	for as, v := range values {
+		f := filepath.Join(c.dir, as+".secret")
+		_ = os.WriteFile(f, []byte(v), 0o600)
+		given = append(given, map[string]string{"node": "anchor", "as": as, "secret": f})
+	}
+	doc, _ := json.Marshal(map[string]any{"requirement": "secret", "given": given})
+	_ = os.WriteFile(c.receives, doc, 0o600)
+}
+
+// apps is n holders named a-app, b-app, …
+func apps(n int) map[string]string {
+	out := map[string]string{}
+	for i := 0; i < n; i++ {
+		out[fmt.Sprintf("%c-app", 'a'+i)] = fmt.Sprint(i)
+	}
+	return out
+}
+
+func keepOnly(all map[string]string, n int) map[string]string {
+	names := make([]string, 0, len(all))
+	for k := range all {
+		names = append(names, k)
+	}
+	sort.Strings(names)
+	out := map[string]string{}
+	for _, k := range names[:n] {
+		out[k] = all[k]
+	}
+	return out
+}
+
+// passes runs n passes a minute apart.
+func (c custodyAt) passes(n int) {
+	for i := 0; i < n; i++ {
+		c.Pass()
+		*c.clock = c.clock.Add(time.Minute)
+	}
+}
+
+// stableFor runs StablePasses passes over an hour: past both of ADR 0230's conditions.
+func (c custodyAt) stable() {
+	for i := 0; i < StablePasses; i++ {
+		c.Pass()
+		*c.clock = c.clock.Add(15 * time.Minute)
+	}
+}
+
+func (c custodyAt) changes() []string {
+	var out []string
+	for _, e := range c.bus.events {
+		if e["event"] == EventRetirement {
+			out = append(out, e["body"].(map[string]any)["change"].(string))
+		} else {
+			out = append(out, e["event"].(string))
+		}
+	}
+	return out
+}
+
+func (c custodyAt) retired(as string) bool {
+	h, _ := c.Ledger.Get(as)
+	return h.Retired != nil
+}
+
+func waitingNames(st RetirementState) []string {
+	if st.Waiting == nil {
+		return nil
+	}
+	var out []string
+	for _, w := range st.Waiting.Consumers {
+		out = append(out, w.Consumer)
+	}
+	return out
+}
+
+func act(why string) Act { return Act{Why: why, By: "person.jochen", Via: "mesh-controller"} }
+
+// ---- the controller's words ----------------------------------------------------------------------------
+
+// paths is every key path in a JSON document, arrays read through their first element.
+func paths(t *testing.T, raw []byte) map[string]bool {
+	t.Helper()
+	var v any
+	if err := json.Unmarshal(raw, &v); err != nil {
+		t.Fatal(err)
+	}
+	out := map[string]bool{}
+	var walk func(prefix string, v any)
+	walk = func(prefix string, v any) {
+		switch x := v.(type) {
+		case map[string]any:
+			for k, e := range x {
+				out[prefix+k] = true
+				walk(prefix+k+".", e)
+			}
+		case []any:
+			if len(x) > 0 {
+				walk(prefix, x[0])
+			}
+		}
+	}
+	walk("", v)
+	return out
+}
+
+// sameWords says every key the vault says is a key the controller's type has, and every key the controller
+// needs is said.
+func sameWords(t *testing.T, what string, ours []byte, golden string, needed ...string) {
+	t.Helper()
+	theirs, err := os.ReadFile(filepath.Join("testdata", golden))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := paths(t, theirs)
+	got := paths(t, ours)
+	for p := range got {
+		if !want[p] {
+			t.Errorf("%s says %q, which the controller's type has no field for", what, p)
+		}
+	}
+	for _, p := range needed {
+		if !got[p] {
+			t.Errorf("%s does not say %q, which the controller reads", what, p)
+		}
+	}
+	// And decoding strictly into the shape of the controller's golden document's own Go type would succeed:
+	// the same kinds at the same keys.
+	var g, o any
+	_ = json.Unmarshal(theirs, &g)
+	_ = json.Unmarshal(ours, &o)
+	sameKinds(t, what, "", g, o)
+}
+
+func sameKinds(t *testing.T, what, at string, want, got any) {
+	switch w := want.(type) {
+	case map[string]any:
+		g, ok := got.(map[string]any)
+		if !ok {
+			if got != nil {
+				t.Errorf("%s: %q is %T, the controller's is an object", what, at, got)
+			}
+			return
+		}
+		for k, v := range g {
+			if wv, known := w[k]; known {
+				sameKinds(t, what, at+"."+k, wv, v)
+			}
+		}
+	case []any:
+		g, ok := got.([]any)
+		if !ok {
+			if got != nil {
+				t.Errorf("%s: %q is %T, the controller's is a list", what, at, got)
+			}
+			return
+		}
+		if len(w) > 0 {
+			for _, e := range g {
+				sameKinds(t, what, at+"[]", w[0], e)
+			}
+		}
+	default:
+		if got != nil && fmt.Sprintf("%T", want) != fmt.Sprintf("%T", got) {
+			t.Errorf("%s: %q is %T, the controller's is %T", what, at, got, want)
+		}
+	}
+}
+
+// Every retirement word the vault says is one the controller's link.ReadRetirement reads: its machine
+// named, its change known, its consumers objects.
+func TestTheRetirementWordsAreTheControllers(t *testing.T) {
+	c := newCustody(t)
+	all := apps(5)
+	c.ask(all)
+	c.passes(1)
+	c.ask(keepOnly(all, 1))
+	c.stable() // waiting: four of five
+	st := c.Retirement()
+	if _, err := c.Reject(waitingNames(st), act("kept for now")); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := c.Approve(waitingNames(st), act("they are gone")); err != nil {
+		t.Fatal(err)
+	}
+	c.passes(1)
+	if err := c.Delete("b-app", "b-app", act("gone for good")); err != nil {
+		t.Fatal(err)
+	}
+	c.ask(all)
+	c.passes(1) // re-enabled
+	seen := map[string]bool{}
+	for i, e := range c.bus.events {
+		if e["event"] != EventRetirement {
+			continue
+		}
+		raw := []byte(c.bus.raw[i])
+		var outer struct {
+			Body json.RawMessage `json:"body"`
+		}
+		_ = json.Unmarshal(raw, &outer)
+		var w struct {
+			Change       string `json:"change"`
+			ProviderNode string `json:"provider-node"`
+			Consumers    []struct {
+				Consumer string `json:"consumer"`
+			} `json:"consumers"`
+		}
+		dec := json.NewDecoder(bytes.NewReader(outer.Body))
+		if err := dec.Decode(&w); err != nil {
+			t.Fatalf("a retirement word the controller cannot read: %v: %s", err, outer.Body)
+		}
+		if w.ProviderNode != "novox" || len(w.Consumers) == 0 || w.Consumers[0].Consumer == "" {
+			t.Errorf("a %s word names no machine or no consumer: %s", w.Change, outer.Body)
+		}
+		seen[w.Change] = true
+		sameWords(t, "the "+w.Change+" word", outer.Body, "retirement-word.json",
+			"provider", "provider-node", "change", "consumers", "consumers.consumer", "at")
+	}
+	for _, change := range []string{"waiting", "rejected", "approved", "retired", "deleted", "reenabled"} {
+		if !seen[change] {
+			t.Errorf("no %s word was said", change)
+		}
+	}
+}
+
+// The answer to provisioner_retirement is the controller's RetirementState, so `retire approve`, `retire
+// reject` and `cleanup list` read the vault.
+func TestTheRetirementAnswerIsTheControllers(t *testing.T) {
+	c := newCustody(t)
+	all := apps(5)
+	c.ask(all)
+	c.passes(1)
+	c.ask(keepOnly(all, 1))
+	c.stable()
+	raw, _ := json.Marshal(c.Retirement())
+	sameWords(t, "the waiting answer", raw, "retirement-state.json", "resource", "node", "held", "stable_passes",
+		"retires_by", "bound", "waiting", "waiting.consumers", "waiting.consumers.consumer", "waiting.since", "retired")
+	if _, err := c.Approve(waitingNames(c.Retirement()), act("gone")); err != nil {
+		t.Fatal(err)
+	}
+	raw, _ = json.Marshal(c.Retirement())
+	sameWords(t, "the retired answer", raw, "retirement-state.json", "retired.consumer", "retired.retired_at",
+		"retired.why", "retired.access")
+	st := c.Retirement()
+	if st.RetiresBy != "mark-only" || st.Resource != "secret" || st.Node != "novox" || len(st.Retired) != 4 ||
+		st.Retired[0].Access != "kept" {
+		t.Errorf("the answer is %+v", st)
+	}
+}
+
+// ---- the rules, one at a time ---------------------------------------------------------------------------
+
+func TestTheCustodyPassRecordsAndRotates(t *testing.T) {
+	c := newCustody(t)
+	c.ask(map[string]string{"anchor-redis": "v1"})
+	c.passes(1)
+	c.ask(map[string]string{"anchor-redis": "v2"})
+	c.passes(1)
+	if got := strings.Join(c.changes(), ","); got != "provisioned,rotated" {
+		t.Errorf("announced %s", got)
+	}
+	for _, raw := range c.bus.raw {
+		if strings.Contains(raw, `"v1"`) || strings.Contains(raw, `"v2"`) {
+			t.Errorf("announced a value: %s", raw)
+		}
+	}
+}
+
+// A contributions file that does not say who is asked for is no answer: nobody is retired.
+func TestAFileThatSaysNothingRetiresNobody(t *testing.T) {
+	for _, doc := range []string{"{not json", `{"requirement":"secret"}`, `{"requirement":"secret","given":null}`,
+		`{"requirement":"redis-cache","given":[]}`} {
+		c := newCustody(t)
+		c.ask(map[string]string{"anchor-redis": "v1"})
+		c.passes(1)
+		_ = os.WriteFile(c.receives, []byte(doc), 0o600)
+		c.passes(30)
+		if h, ok := c.Ledger.Get("anchor-redis"); !ok || h.Retired != nil {
+			t.Errorf("%s: the holder was retired or lost: %+v %v", doc, h, ok)
+		}
+		if _, err := Contributions(c.receives); err == nil {
+			t.Errorf("%s was read as an answer", doc)
+		}
+	}
+	if _, err := Contributions(filepath.Join(t.TempDir(), "missing.json")); err == nil {
+		t.Error("a missing file was read as an answer")
+	}
+}
+
+// Five passes, not four, however much time has passed.
+func TestRetiringTakesFivePasses(t *testing.T) {
+	c := newCustody(t)
+	c.ask(apps(3))
+	c.passes(1)
+	c.ask(keepOnly(apps(3), 2))
+	for i := 0; i < StablePasses-1; i++ {
+		c.Pass()
+		*c.clock = c.clock.Add(time.Hour)
+	}
+	if c.retired("c-app") {
+		t.Fatal("retired after four passes")
+	}
+	c.Pass()
+	if !c.retired("c-app") {
+		t.Fatal("not retired after five passes and hours")
+	}
+}
+
+// Ten minutes, not less, however many passes.
+func TestRetiringTakesTenMinutes(t *testing.T) {
+	c := newCustody(t)
+	c.ask(apps(3))
+	c.passes(1)
+	c.ask(keepOnly(apps(3), 2))
+	for i := 0; i < 20; i++ {
+		c.Pass()
+		*c.clock = c.clock.Add(25 * time.Second)
+	}
+	if c.retired("c-app") {
+		t.Fatal("retired before ten minutes")
+	}
+	*c.clock = c.clock.Add(5 * time.Minute)
+	c.Pass()
+	if !c.retired("c-app") {
+		t.Fatal("not retired after twenty passes and ten minutes")
+	}
+}
+
+// More than three at once waits, though under half: 4 of 10.
+func TestMoreThanThreeWaits(t *testing.T) {
+	c := newCustody(t)
+	c.ask(apps(10))
+	c.passes(1)
+	c.ask(keepOnly(apps(10), 6))
+	c.stable()
+	if got := waitingNames(c.Retirement()); len(got) != 4 {
+		t.Fatalf("four of ten did not wait: %v", got)
+	}
+	if c.retired("j-app") {
+		t.Fatal("retired while waiting")
+	}
+}
+
+// More than half waits, though three or fewer: 2 of 3.
+func TestMoreThanHalfWaits(t *testing.T) {
+	c := newCustody(t)
+	c.ask(apps(3))
+	c.passes(1)
+	c.ask(keepOnly(apps(3), 1))
+	c.stable()
+	if got := waitingNames(c.Retirement()); len(got) != 2 {
+		t.Fatalf("two of three did not wait: %v", got)
+	}
+}
+
+// Within both bounds it retires without a person: 3 of 7.
+func TestWithinTheBoundRetires(t *testing.T) {
+	c := newCustody(t)
+	c.ask(apps(7))
+	c.passes(1)
+	c.ask(keepOnly(apps(7), 4))
+	c.stable()
+	if c.Retirement().Waiting != nil || !c.retired("g-app") {
+		t.Fatal("three of seven did not retire")
+	}
+}
+
+// A holder is retired, never deleted by a pass, keeps its history, and is re-enabled when asked again.
+func TestAHolderNoLongerAskedForIsRetiredNotDeleted(t *testing.T) {
+	c := newCustody(t)
+	c.ask(apps(3))
+	c.passes(1)
+	c.ask(keepOnly(apps(3), 2))
+	c.stable()
+	h, ok := c.Ledger.Get("c-app")
+	if !ok || h.Retired == nil || h.Retired.Why == "" || h.Fingerprint != Fingerprint("2") {
+		t.Fatalf("%+v %v", h, ok)
+	}
+	c.ask(apps(3))
+	c.Pass()
+	if c.retired("c-app") {
+		t.Error("asked for again, the holder stayed retired")
+	}
+	if got := strings.Join(c.changes(), ","); got != "provisioned,provisioned,provisioned,retired,reenabled" {
+		t.Errorf("announced %s", got)
+	}
+}
+
+// A file that says nothing starts the count again.
+func TestTheCountStartsAgain(t *testing.T) {
+	c := newCustody(t)
+	c.ask(apps(3))
+	c.passes(1)
+	c.ask(keepOnly(apps(3), 2))
+	c.passes(4)
+	_ = os.WriteFile(c.receives, []byte(`{"given":null}`), 0o600)
+	c.passes(1)
+	c.ask(keepOnly(apps(3), 2))
+	c.passes(4)
+	*c.clock = c.clock.Add(time.Hour)
+	if c.retired("c-app") {
+		t.Fatal("retired though the count was broken")
+	}
+	c.passes(1)
+	if !c.retired("c-app") {
+		t.Fatal("not retired after five unbroken passes and an hour")
+	}
+}
+
+// Approve and reject take exactly the set waiting, and refuse any other (ADR 0230 §20).
+func TestApproveAndRejectTakeExactlyTheSetWaiting(t *testing.T) {
+	for _, decide := range []string{"approve", "reject"} {
+		c := newCustody(t)
+		c.ask(apps(5))
+		c.passes(1)
+		c.ask(keepOnly(apps(5), 1))
+		c.stable()
+		set := waitingNames(c.Retirement())
+		for _, wrong := range [][]string{set[:3], append(append([]string{}, set...), "a-app"), {}} {
+			if _, err := c.Approve(wrong, act("x")); err == nil {
+				t.Errorf("approve took the set %v", wrong)
+			}
+			if _, err := c.Reject(wrong, act("x")); err == nil {
+				t.Errorf("reject took the set %v", wrong)
+			}
+		}
+		if _, err := c.Approve(set, Act{}); err == nil {
+			t.Error("an approval without a why was taken")
+		}
+		reversed := append([]string{}, set...)
+		sort.Sort(sort.Reverse(sort.StringSlice(reversed)))
+		if decide == "approve" {
+			done, err := c.Approve(reversed, act("the four apps are gone"))
+			if err != nil || len(done) != 4 || !c.retired("b-app") || c.retired("a-app") {
+				t.Fatalf("%v %v", done, err)
+			}
+		} else {
+			if _, err := c.Reject(reversed, act("they come back")); err != nil {
+				t.Fatal(err)
+			}
+			c.passes(10)
+			if c.retired("b-app") || c.Retirement().Waiting != nil {
+				t.Error("a rejected set was retired or asked about again")
+			}
+		}
+	}
+}
+
+// retiredC is a custody with c-app retired, d-app active.
+func retiredC(t *testing.T) custodyAt {
+	c := newCustody(t)
+	c.ask(apps(4))
+	c.passes(1)
+	c.ask(keepOnly(apps(4), 3))
+	c.stable()
+	if !c.retired("d-app") {
+		t.Fatal("d-app was not retired")
+	}
+	return c
+}
+
+// Each refusal of a delete, one at a time.
+func TestEachDeleteRefusal(t *testing.T) {
+	fresh := newCustody(t)
+	fresh.ask(apps(1))
+	if err := fresh.Delete("a-app", "a-app", act("x")); err == nil || !strings.Contains(err.Error(), "no pass") {
+		t.Errorf("a delete before any pass read the file: %v", err)
+	}
+	c := retiredC(t)
+	if err := c.Delete("a-app", "a-app", act("x")); err == nil || !strings.Contains(err.Error(), "active") {
+		t.Errorf("an active holder: %v", err)
+	}
+	if err := c.Delete("d-app", "d-app", Act{By: "person.jochen"}); err == nil || !strings.Contains(err.Error(), "why") {
+		t.Errorf("no why: %v", err)
+	}
+	if err := c.Delete("d-app", "c-app", act("x")); err == nil || !strings.Contains(err.Error(), "confirm") {
+		t.Errorf("a confirm naming another: %v", err)
+	}
+	if err := c.Delete("nobody", "nobody", act("x")); err == nil {
+		t.Error("a holder not held")
+	}
+	c.ask(apps(4)) // asked for again, no pass yet
+	if err := c.Delete("d-app", "d-app", act("x")); err == nil || !strings.Contains(err.Error(), "asks for") {
+		t.Errorf("a holder the file asks for again: %v", err)
+	}
+	c.ask(keepOnly(apps(4), 3))
+	if err := c.Delete("d-app", "d-app", act("gone for good")); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := c.Ledger.Get("d-app"); ok {
+		t.Error("deleted, but still in the ledger")
+	}
+	last := c.bus.events[len(c.bus.events)-1]["body"].(map[string]any)
+	if last["change"] != "deleted" || last["by"] != "person.jochen" || last["via"] != "mesh-controller" {
+		t.Errorf("the deletion said %v", last)
+	}
+}