mesh-catalog/modules/postgres/cmd/postgres-provider/retirement_test.go

package main

// Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is
// retired only after the same result in five consecutive passes, too many at once wait for a person,
// asked for again it is re-enabled, and only a person deletes. The same file in every Go provider.

import (
	"context"
	"errors"
	"fmt"
	"os"
	"strings"
	"testing"
	"time"
)

// recorder is a backend that remembers what it holds, active and retired, as a real one's mark does.
type recorder struct {
	created  []Provision
	removed  []string // retired, in order
	deleted  []string
	held     bool
	failing  error
	holdsErr error
	retErr   error
	inv      Inventory
	invErr   error
}

func (r *recorder) Create(_ context.Context, p Provision) error {
	if r.failing != nil {
		return r.failing
	}
	r.created = append(r.created, p)
	r.inv.Retired = withoutRetired(r.inv.Retired, p.As)
	if !listHas(r.inv.Active, p.As) {
		r.inv.Active = append(r.inv.Active, p.As)
	}
	return nil
}

func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error {
	if r.retErr != nil {
		return r.retErr
	}
	r.removed = append(r.removed, as)
	r.inv.Active = without(r.inv.Active, as)
	r.inv.Retired = append(withoutRetired(r.inv.Retired, as),
		Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer})
	return nil
}

func (r *recorder) Holds(context.Context, Provision) (bool, error) {
	if r.holdsErr != nil {
		return false, r.holdsErr
	}
	return r.held, nil
}

func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr }

func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) {
	r.deleted = append(r.deleted, x.Consumer)
	r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer)
	return x.SizeBytes, nil
}

func listHas(list []string, s string) bool {
	for _, x := range list {
		if x == s {
			return true
		}
	}
	return false
}

func without(list []string, s string) []string {
	var out []string
	for _, x := range list {
		if x != s {
			out = append(out, x)
		}
	}
	return out
}

func withoutRetired(list []Retired, s string) []Retired {
	var out []Retired
	for _, x := range list {
		if x.Consumer != s {
			out = append(out, x)
		}
	}
	return out
}

// holding gives the provider n consumers c1…cn and applies them.
func holding(w *world, n int) []map[string]any {
	var given []map[string]any
	for i := 1; i <= n; i++ {
		given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"})
	}
	w.give(given...)
	w.h.Reconcile(ctx)
	return given
}

// pass is one reconcile five seconds after the last.
func (w *world) pass() {
	w.now = w.now.Add(5 * time.Second)
	w.h.Reconcile(ctx)
}

func retirements(said []announced) []string {
	var out []string
	for _, a := range said {
		if a.event == EventRetirement {
			out = append(out, a.body["change"].(string))
		}
	}
	return out
}

func lastRetirement(t *testing.T, said []announced) map[string]any {
	t.Helper()
	for i := len(said) - 1; i >= 0; i-- {
		if said[i].event == EventRetirement {
			return said[i].body
		}
	}
	t.Fatal("nothing about retirement was announced")
	return nil
}

func namesOf(body map[string]any) string {
	var out []string
	for _, c := range body["consumers"].([]map[string]any) {
		out = append(out, c["consumer"].(string))
	}
	return strings.Join(out, ",")
}

// settle passes every five seconds until the same answer has held for StableFor, and one pass more.
func (w *world) settle() { w.passes(StableFor + 5*time.Second) }

func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) {
	w, said := standingWorld(t)
	given := holding(w, 1)
	w.give()
	for i := 0; i < 4; i++ {
		w.pass()
	}
	w.give(given...)
	w.pass()
	if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
		t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said))
	}
}

// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold:
// five passes AND ten minutes of the same answer (novox/hq ADR 0230).
func TestFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) {
	w, said := standingWorld(t)
	holding(w, 1)
	w.give()
	for i := 0; i < 5; i++ {
		w.pass()
	}
	if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
		t.Fatalf("five passes in 25 s retired %v", w.a.removed)
	}
	w.passes(StableFor - 30*time.Second)
	if len(w.a.removed) != 0 {
		t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed)
	}
	w.passes(time.Minute)
	if strings.Join(w.a.removed, ",") != "c1" {
		t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed)
	}
	// Ten minutes in one slow pass are not five passes.
	w2 := newWorld(t)
	holding(w2, 1)
	w2.give()
	w2.pass()
	w2.now = w2.now.Add(StableFor)
	w2.pass()
	if len(w2.a.removed) != 0 {
		t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed)
	}
	w2.passes(15 * time.Second)
	if len(w2.a.removed) != 1 {
		t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed)
	}
}

func TestAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) {
	w, said := standingWorld(t)
	given := holding(w, 3)
	w.give(given[:2]...)
	w.settle()
	if strings.Join(w.a.removed, ",") != "c3" {
		t.Fatalf("retired %v", w.a.removed)
	}
	body := lastRetirement(t, *said)
	if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" ||
		!strings.Contains(body["why"].(string), "consecutive passes over 10m") {
		t.Fatalf("%v", body)
	}
	if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" {
		t.Fatalf("the backend does not hold it retired: %+v", w.a.inv)
	}
	// Asked for again: the ordinary create, on the first pass, and said.
	created := len(w.a.created)
	w.give(given...)
	w.pass()
	if len(w.a.created) != created+1 || w.a.created[created].As != "c3" {
		t.Fatalf("not created again: %v", w.a.created)
	}
	if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" {
		t.Fatalf("%v", body)
	}
	if len(w.a.inv.Retired) != 0 {
		t.Fatalf("still marked retired: %+v", w.a.inv.Retired)
	}
}

func TestAnUnreadablePassStartsTheCountAgain(t *testing.T) {
	w := newWorld(t)
	holding(w, 1)
	w.give()
	w.passes(StableFor - time.Minute)
	os.WriteFile(w.receives, []byte("{"), 0o600)
	w.pass()
	w.give()
	w.passes(StableFor - time.Minute)
	if len(w.a.removed) != 0 {
		t.Fatalf("an unreadable pass counted: %v", w.a.removed)
	}
	w.passes(2 * time.Minute)
	if len(w.a.removed) != 1 {
		t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed)
	}
}

func TestADifferentSetStartsTheCountAgain(t *testing.T) {
	w := newWorld(t)
	given := holding(w, 5)
	w.give(given[:4]...)
	w.passes(StableFor - time.Minute)
	w.give(given[:3]...)
	w.passes(StableFor - time.Minute)
	if len(w.a.removed) != 0 {
		t.Fatalf("a changed set kept its count: %v", w.a.removed)
	}
	w.passes(2 * time.Minute)
	if strings.Join(w.a.removed, ",") != "c4,c5" {
		t.Fatalf("%v", w.a.removed)
	}
}

func TestAdditionsAndChangesAreNeverDelayed(t *testing.T) {
	w := newWorld(t)
	holding(w, 1)
	w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"})
	w.pass()
	if len(w.a.created) != 2 || w.a.created[1].As != "c2" {
		t.Fatalf("an addition waited: %v", w.a.created)
	}
	w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"})
	w.pass()
	if len(w.a.created) != 3 || w.a.created[2].As != "c1" {
		t.Fatalf("a change waited: %v", w.a.created)
	}
}

func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) {
	w, said := standingWorld(t)
	holding(w, 4)
	w.give()
	w.passes(15 * time.Minute)
	if len(w.a.removed) != 0 {
		t.Fatalf("four of four were retired without a person: %v", w.a.removed)
	}
	if got := strings.Join(retirements(*said), ","); got != ChangeWaiting {
		t.Fatalf("said %q, want one waiting", got)
	}
	body := lastRetirement(t, *said)
	if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" {
		t.Fatalf("%v", body)
	}
	if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") {
		t.Fatal("the wait was not said")
	}
	// Said again every quarter of an hour while it waits.
	w.passes(11 * time.Minute)
	if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" {
		t.Fatalf("%q", got)
	}

	if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil {
		t.Fatal("approved a set that is not the one waiting")
	}
	if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil {
		t.Fatal("approved without a why")
	}
	done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller")
	if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" {
		t.Fatal(done, err, w.a.removed)
	}
	changes := retirements(*said)
	if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" {
		t.Fatalf("%v", changes)
	}
	if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" ||
		!strings.Contains(b["why"].(string), "the old machine is gone") {
		t.Fatalf("%v", b)
	}
	// Nothing more waits.
	w.passes(time.Minute)
	if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 {
		t.Fatalf("%v", r)
	}
}

func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) {
	w, said := standingWorld(t)
	given := holding(w, 4)
	w.give()
	w.settle()
	if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil {
		t.Fatal("rejected a set that is not the one waiting")
	}
	kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller")
	if err != nil || len(kept) != 4 {
		t.Fatal(kept, err)
	}
	w.passes(time.Hour)
	if len(w.a.removed) != 0 {
		t.Fatalf("a rejected set was retired: %v", w.a.removed)
	}
	if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" {
		t.Fatalf("asked again after a rejection: %q", got)
	}
	r, _ := w.h.Retirement(ctx)
	if r["rejected"] == nil || r["waiting"] != nil {
		t.Fatalf("%v", r)
	}
	// One of them asked for again: a different answer, so the rejection is settled and counting
	// starts over — three of four is over the bound again, and waits again.
	w.give(given[0])
	w.pass()
	if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" {
		t.Fatalf("%q", got)
	}
	w.settle()
	if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" {
		t.Fatalf("%q", got)
	}
	// A rejected set can still be approved later.
	w2, _ := standingWorld(t)
	holding(w2, 4)
	w2.give()
	w2.settle()
	w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "")
	if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 {
		t.Fatal(done, err)
	}
}

func TestAWaitingSetAskedForAgainSettles(t *testing.T) {
	w, said := standingWorld(t)
	given := holding(w, 4)
	w.give()
	w.settle()
	w.give(given...)
	w.pass()
	if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 {
		t.Fatalf("%q %v", got, w.a.removed)
	}
	if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil {
		t.Fatal("approved a set that no longer waits")
	}
}

func TestDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) {
	w, said := standingWorld(t)
	given := holding(w, 5)
	w.give(given[:3]...)
	w.settle()
	if strings.Join(w.a.removed, ",") != "c4,c5" {
		t.Fatalf("%v", w.a.removed)
	}
	if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil {
		t.Fatal("deleted an active consumer")
	}
	if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil {
		t.Fatal("deleted without a why")
	}
	if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil {
		t.Fatal("deleted something not retired")
	}
	freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller")
	if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" {
		t.Fatal(freed, err, w.a.deleted)
	}
	if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) {
		t.Fatalf("%v", b)
	}
	r, _ := w.h.Retirement(ctx)
	if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" {
		t.Fatalf("%v", r)
	}
	// Retired and asked for again before anybody deleted it: refused, it is the mesh's again.
	w.give(given[:4]...)
	w.pass()
	if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil {
		t.Fatal("deleted a consumer the mesh asks for")
	}
}

func TestTheBound(t *testing.T) {
	h := &Harness{}
	h.init()
	for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} {
		if h.overTheBound(c.n, c.held) {
			t.Errorf("%d of %d waits for a person", c.n, c.held)
		}
	}
	for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} {
		if !h.overTheBound(c.n, c.held) {
			t.Errorf("%d of %d is retired without a person", c.n, c.held)
		}
	}
}

func TestARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) {
	w, said := standingWorld(t)
	w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{
		{Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer},
		{Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"},
	}}
	w.give(map[string]any{"as": "kept"})
	w.h.Reconcile(ctx)
	if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" {
		t.Fatalf("%v", b)
	}
	if strings.Join(w.a.removed, ",") != "locked-before" {
		t.Fatalf("adopting did not mark it: %v", w.a.removed)
	}
	w.settle()
	if strings.Join(w.a.removed, ",") != "locked-before,orphan" {
		t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed)
	}
}

func TestABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) {
	w := newWorld(t)
	w.a.invErr = errors.New("connection refused")
	holding(w, 1)
	if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") {
		t.Fatal(w.said)
	}
	w.a.invErr = nil
	w.a.inv = Inventory{Active: []string{"c1", "orphan"}}
	w.pass()
	w.settle()
	if strings.Join(w.a.removed, ",") != "orphan" {
		t.Fatalf("%v", w.a.removed)
	}
}

func TestTheToolsAnswer(t *testing.T) {
	w, _ := standingWorld(t)
	holding(w, 4)
	w.give()
	w.settle()
	tools := map[string]func(map[string]any) (any, error){}
	for _, tool := range RetirementTools(w.h) {
		tools[tool.Name] = tool.Run
	}
	if len(tools) != 4 {
		t.Fatalf("%d tools", len(tools))
	}
	got, err := tools["provisioner_retirement"](nil)
	if err != nil || got.(map[string]any)["waiting"] == nil {
		t.Fatal(got, err)
	}
	set := []any{"c1", "c2", "c3", "c4"}
	if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil {
		t.Fatal("approved without a why")
	}
	if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil {
		t.Fatal(err)
	}
	if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil {
		t.Fatal("deleted without confirm")
	}
	if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil {
		t.Fatal(err)
	}
	if strings.Join(w.a.deleted, ",") != "c1" {
		t.Fatal(w.a.deleted)
	}
}

func TestAFailingConsumerRetiredIsSaidRecovered(t *testing.T) {
	w, said := standingWorld(t)
	given := holding(w, 2)
	w.a.held = false
	w.a.failing = errors.New("boom")
	w.passes(7 * time.Minute)
	w.give(given[0])
	w.settle()
	recovered := false
	for _, a := range *said {
		if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" {
			recovered = true
		}
	}
	if !recovered {
		t.Fatalf("%v", *said)
	}
}