package main // Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is // retired only after the same result in five consecutive passes, too many at once wait for a person, // asked for again it is re-enabled, and only a person deletes. The same file in every Go provider. import ( "context" "errors" "fmt" "os" "strings" "testing" "time" ) // recorder is a backend that remembers what it holds, active and retired, as a real one's mark does. type recorder struct { created []Provision removed []string // retired, in order deleted []string held bool failing error holdsErr error retErr error inv Inventory invErr error } func (r *recorder) Create(_ context.Context, p Provision) error { if r.failing != nil { return r.failing } r.created = append(r.created, p) r.inv.Retired = withoutRetired(r.inv.Retired, p.As) if !listHas(r.inv.Active, p.As) { r.inv.Active = append(r.inv.Active, p.As) } return nil } func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error { if r.retErr != nil { return r.retErr } r.removed = append(r.removed, as) r.inv.Active = without(r.inv.Active, as) r.inv.Retired = append(withoutRetired(r.inv.Retired, as), Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer}) return nil } func (r *recorder) Holds(context.Context, Provision) (bool, error) { if r.holdsErr != nil { return false, r.holdsErr } return r.held, nil } func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr } func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) { r.deleted = append(r.deleted, x.Consumer) r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer) return x.SizeBytes, nil } func listHas(list []string, s string) bool { for _, x := range list { if x == s { return true } } return false } func without(list []string, s string) []string { var out []string for _, x := range list { if x != s { out = append(out, x) } } return out } func withoutRetired(list []Retired, s string) []Retired { var out []Retired for _, x := range list { if x.Consumer != s { out = append(out, x) } } return out } // holding gives the provider n consumers c1…cn and applies them. func holding(w *world, n int) []map[string]any { var given []map[string]any for i := 1; i <= n; i++ { given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"}) } w.give(given...) w.h.Reconcile(ctx) return given } // pass is one reconcile five seconds after the last. func (w *world) pass() { w.now = w.now.Add(5 * time.Second) w.h.Reconcile(ctx) } func retirements(said []announced) []string { var out []string for _, a := range said { if a.event == EventRetirement { out = append(out, a.body["change"].(string)) } } return out } func lastRetirement(t *testing.T, said []announced) map[string]any { t.Helper() for i := len(said) - 1; i >= 0; i-- { if said[i].event == EventRetirement { return said[i].body } } t.Fatal("nothing about retirement was announced") return nil } func namesOf(body map[string]any) string { var out []string for _, c := range body["consumers"].([]map[string]any) { out = append(out, c["consumer"].(string)) } return strings.Join(out, ",") } func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) { w, said := standingWorld(t) given := holding(w, 1) w.give() for i := 0; i < 4; i++ { w.pass() } w.give(given...) w.pass() if len(w.a.removed) != 0 || len(retirements(*said)) != 0 { t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said)) } // Gone again: counted from one, not from where the transient left off. w.give() for i := 0; i < 4; i++ { w.pass() } if len(w.a.removed) != 0 { t.Fatalf("retired after four passes: %v", w.a.removed) } w.pass() if strings.Join(w.a.removed, ",") != "c1" { t.Fatalf("the fifth pass did not retire: %v", w.a.removed) } } func TestFiveStablePassesRetireAndAskedAgainReEnables(t *testing.T) { w, said := standingWorld(t) given := holding(w, 3) w.give(given[:2]...) for i := 0; i < 5; i++ { w.pass() } if strings.Join(w.a.removed, ",") != "c3" { t.Fatalf("retired %v", w.a.removed) } body := lastRetirement(t, *said) if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" || !strings.Contains(body["why"].(string), "5 consecutive passes") { t.Fatalf("%v", body) } if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" { t.Fatalf("the backend does not hold it retired: %+v", w.a.inv) } // Asked for again: the ordinary create, on the first pass, and said. created := len(w.a.created) w.give(given...) w.pass() if len(w.a.created) != created+1 || w.a.created[created].As != "c3" { t.Fatalf("not created again: %v", w.a.created) } if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" { t.Fatalf("%v", body) } if len(w.a.inv.Retired) != 0 { t.Fatalf("still marked retired: %+v", w.a.inv.Retired) } } func TestAnUnreadablePassStartsTheCountAgain(t *testing.T) { w := newWorld(t) holding(w, 1) w.give() for i := 0; i < 3; i++ { w.pass() } os.WriteFile(w.receives, []byte("{"), 0o600) w.pass() w.give() for i := 0; i < 4; i++ { w.pass() } if len(w.a.removed) != 0 { t.Fatalf("an unreadable pass counted: %v", w.a.removed) } w.pass() if len(w.a.removed) != 1 { t.Fatalf("not retired after five readable passes: %v", w.a.removed) } } func TestADifferentSetStartsTheCountAgain(t *testing.T) { w := newWorld(t) given := holding(w, 5) w.give(given[:4]...) for i := 0; i < 3; i++ { w.pass() } w.give(given[:3]...) for i := 0; i < 4; i++ { w.pass() } if len(w.a.removed) != 0 { t.Fatalf("a changed set kept its count: %v", w.a.removed) } w.pass() if strings.Join(w.a.removed, ",") != "c4,c5" { t.Fatalf("%v", w.a.removed) } } func TestAdditionsAndChangesAreNeverDelayed(t *testing.T) { w := newWorld(t) holding(w, 1) w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"}) w.pass() if len(w.a.created) != 2 || w.a.created[1].As != "c2" { t.Fatalf("an addition waited: %v", w.a.created) } w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"}) w.pass() if len(w.a.created) != 3 || w.a.created[2].As != "c1" { t.Fatalf("a change waited: %v", w.a.created) } } func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) { w, said := standingWorld(t) holding(w, 4) w.give() w.passes(10 * time.Minute) if len(w.a.removed) != 0 { t.Fatalf("four of four were retired without a person: %v", w.a.removed) } if got := strings.Join(retirements(*said), ","); got != ChangeWaiting { t.Fatalf("said %q, want one waiting", got) } body := lastRetirement(t, *said) if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" { t.Fatalf("%v", body) } if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") { t.Fatal("the wait was not said") } // Said again every quarter of an hour while it waits. w.passes(6 * time.Minute) if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" { t.Fatalf("%q", got) } if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil { t.Fatal("approved a set that is not the one waiting") } if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil { t.Fatal("approved without a why") } done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller") if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" { t.Fatal(done, err, w.a.removed) } changes := retirements(*said) if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" { t.Fatalf("%v", changes) } if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" || !strings.Contains(b["why"].(string), "the old machine is gone") { t.Fatalf("%v", b) } // Nothing more waits. w.passes(time.Minute) if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 { t.Fatalf("%v", r) } } func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) { w, said := standingWorld(t) given := holding(w, 4) w.give() w.passes(time.Minute) if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil { t.Fatal("rejected a set that is not the one waiting") } kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller") if err != nil || len(kept) != 4 { t.Fatal(kept, err) } w.passes(time.Hour) if len(w.a.removed) != 0 { t.Fatalf("a rejected set was retired: %v", w.a.removed) } if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" { t.Fatalf("asked again after a rejection: %q", got) } r, _ := w.h.Retirement(ctx) if r["rejected"] == nil || r["waiting"] != nil { t.Fatalf("%v", r) } // One of them asked for again: a different answer, so the rejection is settled and counting // starts over — three of four is over the bound again, and waits again. w.give(given[0]) w.passes(30 * time.Second) if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" { t.Fatalf("%q", got) } // A rejected set can still be approved later. w2, _ := standingWorld(t) holding(w2, 4) w2.give() w2.passes(time.Minute) w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "") if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 { t.Fatal(done, err) } } func TestAWaitingSetAskedForAgainSettles(t *testing.T) { w, said := standingWorld(t) given := holding(w, 4) w.give() w.passes(time.Minute) w.give(given...) w.pass() if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 { t.Fatalf("%q %v", got, w.a.removed) } if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil { t.Fatal("approved a set that no longer waits") } } func TestDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) { w, said := standingWorld(t) given := holding(w, 5) w.give(given[:3]...) w.passes(25 * time.Second) if strings.Join(w.a.removed, ",") != "c4,c5" { t.Fatalf("%v", w.a.removed) } if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil { t.Fatal("deleted an active consumer") } if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil { t.Fatal("deleted without a why") } if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil { t.Fatal("deleted something not retired") } freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller") if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" { t.Fatal(freed, err, w.a.deleted) } if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) { t.Fatalf("%v", b) } r, _ := w.h.Retirement(ctx) if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" { t.Fatalf("%v", r) } // Retired and asked for again before anybody deleted it: refused, it is the mesh's again. w.give(given[:4]...) w.pass() if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil { t.Fatal("deleted a consumer the mesh asks for") } } func TestTheBound(t *testing.T) { h := &Harness{} h.init() for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} { if h.overTheBound(c.n, c.held) { t.Errorf("%d of %d waits for a person", c.n, c.held) } } for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} { if !h.overTheBound(c.n, c.held) { t.Errorf("%d of %d is retired without a person", c.n, c.held) } } } func TestARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) { w, said := standingWorld(t) w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{ {Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer}, {Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"}, }} w.give(map[string]any{"as": "kept"}) w.h.Reconcile(ctx) if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" { t.Fatalf("%v", b) } if strings.Join(w.a.removed, ",") != "locked-before" { t.Fatalf("adopting did not mark it: %v", w.a.removed) } for i := 0; i < 5; i++ { w.pass() } if strings.Join(w.a.removed, ",") != "locked-before,orphan" { t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed) } } func TestABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) { w := newWorld(t) w.a.invErr = errors.New("connection refused") holding(w, 1) if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") { t.Fatal(w.said) } w.a.invErr = nil w.a.inv = Inventory{Active: []string{"c1", "orphan"}} w.pass() w.passes(25 * time.Second) if strings.Join(w.a.removed, ",") != "orphan" { t.Fatalf("%v", w.a.removed) } } func TestTheToolsAnswer(t *testing.T) { w, _ := standingWorld(t) holding(w, 4) w.give() w.passes(time.Minute) tools := map[string]func(map[string]any) (any, error){} for _, tool := range RetirementTools(w.h) { tools[tool.Name] = tool.Run } if len(tools) != 4 { t.Fatalf("%d tools", len(tools)) } got, err := tools["provisioner_retirement"](nil) if err != nil || got.(map[string]any)["waiting"] == nil { t.Fatal(got, err) } set := []any{"c1", "c2", "c3", "c4"} if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil { t.Fatal("approved without a why") } if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil { t.Fatal(err) } if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil { t.Fatal("deleted without confirm") } if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil { t.Fatal(err) } if strings.Join(w.a.deleted, ",") != "c1" { t.Fatal(w.a.deleted) } } func TestAFailingConsumerRetiredIsSaidRecovered(t *testing.T) { w, said := standingWorld(t) given := holding(w, 2) w.a.held = false w.a.failing = errors.New("boom") w.passes(7 * time.Minute) w.give(given[0]) w.passes(25 * time.Second) recovered := false for _, a := range *said { if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" { recovered = true } } if !recovered { t.Fatalf("%v", *said) } }