mesh-controller/cmd/mesh-controller/retirement_test.go

package main

import (
	"context"
	"encoding/json"
	"flag"
	"os"
	"slices"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/link"
)

// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
// answer it ask the provider — never anything else — for exactly what it said.

func waitingWord(module, node string, names ...string) link.Retirement {
	r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
		Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
	for _, n := range names {
		r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
	}
	return r
}

func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
	t.Helper()
	all, err := k.Open(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	out := map[string]conditions.Condition{}
	for _, c := range all {
		out[c.Key] = c
	}
	return out
}

func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
	k, _ := withConditionsInMemory(t)
	var recorded []link.HandAct
	r := retirements{keeper: func() *conditions.Keeper { return k },
		record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
	ctx := t.Context()
	waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
	if err := r.Retired(ctx, waiting); err != nil {
		t.Fatal(err)
	}
	open := openKeys(t, k)
	c, ok := open["provider.postgres.anchor.retire"]
	if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
		t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
	}
	for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
		"retire reject anchor postgres", "more than 3"} {
		if !strings.Contains(c.Summary, want) {
			t.Errorf("the condition does not say %q: %s", want, c.Summary)
		}
	}

	// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
	// the verb recorded it before it asked.
	rejected := waiting
	rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
	if err := r.Retired(ctx, rejected); err != nil {
		t.Fatal(err)
	}
	open = openKeys(t, k)
	if _, still := open["provider.postgres.anchor.retire"]; still {
		t.Fatal("a rejection left the provider waiting")
	}
	if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
		c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
		t.Fatalf("a rejection is not a warning saying why: %+v", open)
	}
	if len(recorded) != 0 {
		t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
	}

	// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
	approved := waiting
	approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
	if err := r.Retired(ctx, approved); err != nil {
		t.Fatal(err)
	}
	if open := openKeys(t, k); len(open) != 0 {
		t.Fatalf("an approval left conditions open: %+v", open)
	}
	if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
		!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
		t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
	}

	// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
	for _, change := range []string{link.RetireSettled, link.RetireRetired} {
		if err := r.Retired(ctx, waiting); err != nil {
			t.Fatal(err)
		}
		done := waiting
		done.Change = change
		if err := r.Retired(ctx, done); err != nil {
			t.Fatal(err)
		}
		if open := openKeys(t, k); len(open) != 0 {
			t.Fatalf("%s left conditions open: %+v", change, open)
		}
	}

	// A deletion asked of the provider directly is recorded too.
	deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
		Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
	if err := r.Retired(ctx, deleted); err != nil {
		t.Fatal(err)
	}
	if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
		t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
	}
}

func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
	body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
		"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
	r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
	if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
		t.Fatalf("%+v %v", r, err)
	}
	if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
		[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
		t.Fatal("a change the mesh has no name for was read")
	}
	if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
		t.Fatal("a failing word was read as a retirement")
	}
	k, _ := withConditionsInMemory(t)
	if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
		t.Fatal(err)
	}
	if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
		t.Fatal("not kept under the emitter the subject names")
	}
}

func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
		Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
	if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
		t.Fatal(err)
	}
	r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
	for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
		if err := r.Retired(ctx, w); err != nil {
			t.Fatal(err)
		}
	}
	if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
		cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
			[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
	}); err != nil {
		t.Fatal(err)
	}
	all, _ := conditionsFrom.Open(ctx)
	if len(all) != 3 {
		t.Fatalf("%+v", all)
	}
	if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
		t.Fatal(err)
	}
	left := openKeys(t, conditionsFrom)
	if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
		t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
	}
}

// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
type fakeProvider struct {
	mu       sync.Mutex
	state    link.RetirementState
	asked    []map[string]any
	deleted  []string
	approved []string
	rejected []string
}

func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
	t.Helper()
	answer := func(m *nats.Msg, result any, refusal string) {
		body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
		_ = m.Respond(body)
	}
	names := func(args map[string]any) []string {
		var out []string
		for _, v := range args["consumers"].([]any) {
			out = append(out, v.(string))
		}
		sort.Strings(out)
		return out
	}
	set := func(cs []link.RetiredConsumer) []string {
		var out []string
		for _, c := range cs {
			out = append(out, c.Consumer)
		}
		sort.Strings(out)
		return out
	}
	for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
		tool := tool
		sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
			f.mu.Lock()
			defer f.mu.Unlock()
			var args map[string]any
			_ = json.Unmarshal(m.Data, &args)
			f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
			switch tool {
			case link.ToolRetirement:
				answer(m, f.state, "")
			case link.ToolRetireApprove:
				if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
					answer(m, nil, "not the set I wait with")
					return
				}
				f.approved = names(args)
				answer(m, map[string]any{"retired": f.approved}, "")
			case link.ToolRetireReject:
				if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
					answer(m, nil, "not the set I wait with")
					return
				}
				f.rejected = names(args)
				answer(m, map[string]any{"kept": f.rejected}, "")
			case link.ToolRetiredDelete:
				if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
					answer(m, nil, "confirm, why and via")
					return
				}
				f.deleted = append(f.deleted, args["consumer"].(string))
				answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
			}
		})
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { _ = sub.Unsubscribe() })
	}
	if err := conn.Flush(); err != nil {
		t.Fatal(err)
	}
}

func onATestBus(t *testing.T) *nats.Conn {
	t.Helper()
	url := os.Getenv("MESH_TEST_NATS")
	if url == "" {
		t.Skip("MESH_TEST_NATS unset")
	}
	js, err := broker.Dial(url)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(js.Close)
	if err := js.EnsureControllerBuckets(); err != nil {
		t.Fatal(err)
	}
	before := handActConn
	handActConn = js.Conn()
	t.Cleanup(func() { handActConn = before })
	return js.Conn()
}

func whyFlags(t *testing.T, why string) handActFlags {
	t.Helper()
	set := flag.NewFlagSet("t", flag.ContinueOnError)
	f := addHandActFlags(set)
	if err := set.Parse([]string{"--why", why}); err != nil {
		t.Fatal(err)
	}
	return f
}

func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
	t.Helper()
	acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
	if err != nil {
		t.Fatal(err)
	}
	var out []link.HandAct
	for _, a := range acts {
		if a.Verb == verb {
			out = append(out, a)
		}
	}
	return out
}

func retiredDaysAgo(name string, days int) link.RetiredConsumer {
	size := int64(4096)
	return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
		RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
}

func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
	conn := onATestBus(t)
	fake := &fakeProvider{}
	fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
	fake.serve(t, conn, "pg-approve", "anchor")
	p := providerInstance{Node: "anchor", Module: "pg-approve"}
	said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
	if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
		t.Fatalf("approved %v; said %s", fake.approved, said)
	}
	acts := handActsBy(t, conn, "retire approve")
	if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
		acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
		t.Fatalf("the approval is not in the hand-act log: %+v", acts)
	}

	// Reject, on another provider: the same set, and nothing approved.
	other := &fakeProvider{state: fake.state}
	other.serve(t, conn, "pg-reject", "anchor")
	printed(t, func() error {
		return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
			whyFlags(t, "still moving"))
	})
	if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
		t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
	}

	// Nothing waiting: refused, nothing asked but the question.
	idle := &fakeProvider{}
	idle.serve(t, conn, "pg-idle", "anchor")
	if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
		whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
		t.Fatalf("%v", err)
	}
	if len(idle.asked) != 1 {
		t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
	}
}

func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
	conn := onATestBus(t)
	fake := &fakeProvider{}
	fake.state.Held = []string{"active"}
	fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
	fake.serve(t, conn, "pg-clean", "anchor")
	p := providerInstance{Node: "anchor", Module: "pg-clean"}
	said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
	if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
		t.Fatalf("deleted %v; said %s", fake.deleted, said)
	}
	// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
	for _, name := range []string{"active", "nobody"} {
		if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
			!strings.Contains(err.Error(), "only a retired consumer is deleted") {
			t.Fatalf("%s: %v", name, err)
		}
	}
	if !slices.Equal(fake.deleted, []string{"old"}) {
		t.Fatalf("more was deleted: %v", fake.deleted)
	}
	if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
		t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
	}

	// Older than: listed, and nothing deleted without confirm; with it, only the old one.
	listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
	if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
		t.Fatalf("%+v", listing)
	}
	fake.deleted = nil
	said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
	if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
		strings.Contains(said, "young") {
		t.Fatalf("deleted %v; said %s", fake.deleted, said)
	}
	printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
	if !slices.Equal(fake.deleted, []string{"old"}) {
		t.Fatalf("confirmed, deleted %v", fake.deleted)
	}
}

func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
	conn := onATestBus(t)
	old := &fakeProvider{}
	old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
	old.serve(t, conn, "pg-d11-old", "anchor")
	young := &fakeProvider{}
	young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
	young.serve(t, conn, "pg-d11-young", "anchor")
	instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
		// Nothing serves this one: a provider older than the question is passed over, not a failure.
		{Node: "anchor", Module: "pg-d11-predates"}}
	found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
		found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
		strings.Contains(found[0].Summary, "fresh") {
		t.Fatalf("%+v", found)
	}
}

func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
	for _, c := range []struct {
		verb string
		args map[string]any
		want string
	}{
		{"retire", map[string]any{}, "retire --json"},
		{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
			"retire approve anchor postgres --why moved"},
		{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
			"retire reject anchor postgres --why no"},
		{"cleanup", map[string]any{}, "cleanup list --json"},
		{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
			"cleanup delete anchor postgres x --why gone"},
		{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
		{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
			"cleanup delete --older-than 30 --why tidy --confirm"},
	} {
		argv, err := argvFor(c.verb, c.args)
		if err != nil || strings.Join(argv, " ") != c.want {
			t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
		}
	}
	for _, args := range []map[string]any{
		{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
		{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
	} {
		if _, err := argvFor("retire", args); err == nil {
			t.Errorf("retire %v was composed", args)
		}
	}
	for _, args := range []map[string]any{
		{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
		{"older-than": "30"},
		{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
	} {
		if _, err := argvFor("cleanup", args); err == nil {
			t.Errorf("cleanup %v was composed", args)
		}
	}
	if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
		repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
		t.Error("the generic verb would let a retirement or a deletion through without a why")
	}
}

// A mark-only provider's retired consumer keeps its access; the listing and the approval say so
// rather than claiming it was disabled (ADR 0230).
func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) {
	conn := onATestBus(t)
	fake := &fakeProvider{}
	kept := retiredDaysAgo("ledger", 3)
	kept.Access = "kept"
	fake.state.Retired = []link.RetiredConsumer{kept}
	fake.state.RetiresBy = "mark-only"
	fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2}
	fake.serve(t, conn, "vault-mark", "anchor")
	p := providerInstance{Node: "anchor", Module: "vault-mark"}
	listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
	said := printed(t, func() error { return printRetired(listing, false) })
	if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" {
		t.Fatalf("%s %+v", said, listing)
	}
	said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) })
	if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") {
		t.Fatal(said)
	}
}