mesh-controller/cmd/mesh-controller/retirement_test.go

package main

import (
	"context"
	"encoding/json"
	"flag"
	"slices"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/link"
)

// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
// answer it ask the provider — never anything else — for exactly what it said.

func waitingWord(module, node string, names ...string) link.Retirement {
	r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
		Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
	for _, n := range names {
		r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
	}
	return r
}

func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
	t.Helper()
	all, err := k.Open(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	out := map[string]conditions.Condition{}
	for _, c := range all {
		out[c.Key] = c
	}
	return out
}

func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
	k, _ := withConditionsInMemory(t)
	var recorded []link.HandAct
	r := retirements{keeper: func() *conditions.Keeper { return k },
		record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
	ctx := t.Context()
	waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
	if err := r.Retired(ctx, waiting); err != nil {
		t.Fatal(err)
	}
	open := openKeys(t, k)
	c, ok := open["provider.postgres.anchor.retire"]
	if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
		t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
	}
	for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
		"retire reject anchor postgres", "more than 3"} {
		if !strings.Contains(c.Summary, want) {
			t.Errorf("the condition does not say %q: %s", want, c.Summary)
		}
	}

	// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
	// the verb recorded it before it asked.
	rejected := waiting
	rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
	if err := r.Retired(ctx, rejected); err != nil {
		t.Fatal(err)
	}
	open = openKeys(t, k)
	if _, still := open["provider.postgres.anchor.retire"]; still {
		t.Fatal("a rejection left the provider waiting")
	}
	if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
		c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
		t.Fatalf("a rejection is not a warning saying why: %+v", open)
	}
	if len(recorded) != 0 {
		t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
	}

	// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
	approved := waiting
	approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
	if err := r.Retired(ctx, approved); err != nil {
		t.Fatal(err)
	}
	if open := openKeys(t, k); len(open) != 0 {
		t.Fatalf("an approval left conditions open: %+v", open)
	}
	if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
		!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
		t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
	}

	// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
	for _, change := range []string{link.RetireSettled, link.RetireRetired} {
		if err := r.Retired(ctx, waiting); err != nil {
			t.Fatal(err)
		}
		done := waiting
		done.Change = change
		if err := r.Retired(ctx, done); err != nil {
			t.Fatal(err)
		}
		if open := openKeys(t, k); len(open) != 0 {
			t.Fatalf("%s left conditions open: %+v", change, open)
		}
	}

	// A deletion asked of the provider directly is recorded too.
	deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
		Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
	if err := r.Retired(ctx, deleted); err != nil {
		t.Fatal(err)
	}
	if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
		t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
	}
}

func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
	body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
		"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
	r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
	if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
		t.Fatalf("%+v %v", r, err)
	}
	if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
		[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
		t.Fatal("a change the mesh has no name for was read")
	}
	if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
		t.Fatal("a failing word was read as a retirement")
	}
	k, _ := withConditionsInMemory(t)
	if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
		t.Fatal(err)
	}
	if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
		t.Fatal("not kept under the emitter the subject names")
	}
}

// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
type fakeProvider struct {
	mu       sync.Mutex
	state    link.RetirementState
	asked    []map[string]any
	deleted  []string
	approved []string
	rejected []string
}

func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
	t.Helper()
	answer := func(m *nats.Msg, result any, refusal string) {
		body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
		_ = m.Respond(body)
	}
	names := func(args map[string]any) []string {
		var out []string
		for _, v := range args["consumers"].([]any) {
			out = append(out, v.(string))
		}
		sort.Strings(out)
		return out
	}
	set := func(cs []link.RetiredConsumer) []string {
		var out []string
		for _, c := range cs {
			out = append(out, c.Consumer)
		}
		sort.Strings(out)
		return out
	}
	for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
		tool := tool
		sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
			f.mu.Lock()
			defer f.mu.Unlock()
			var args map[string]any
			_ = json.Unmarshal(m.Data, &args)
			f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
			switch tool {
			case link.ToolRetirement:
				answer(m, f.state, "")
			case link.ToolRetireApprove:
				if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
					answer(m, nil, "not the set I wait with")
					return
				}
				f.approved = names(args)
				answer(m, map[string]any{"retired": f.approved}, "")
			case link.ToolRetireReject:
				if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
					answer(m, nil, "not the set I wait with")
					return
				}
				f.rejected = names(args)
				answer(m, map[string]any{"kept": f.rejected}, "")
			case link.ToolRetiredDelete:
				if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
					answer(m, nil, "confirm, why and via")
					return
				}
				f.deleted = append(f.deleted, args["consumer"].(string))
				answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
			}
		})
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { _ = sub.Unsubscribe() })
	}
	if err := conn.Flush(); err != nil {
		t.Fatal(err)
	}
}

func whyFlags(t *testing.T, why string) handActFlags {
	t.Helper()
	set := flag.NewFlagSet("t", flag.ContinueOnError)
	f := addHandActFlags(set)
	if err := set.Parse([]string{"--why", why}); err != nil {
		t.Fatal(err)
	}
	return f
}

func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
	t.Helper()
	acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
	if err != nil {
		t.Fatal(err)
	}
	var out []link.HandAct
	for _, a := range acts {
		if a.Verb == verb {
			out = append(out, a)
		}
	}
	return out
}

func retiredDaysAgo(name string, days int) link.RetiredConsumer {
	size := int64(4096)
	return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
		RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
}

func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
	for _, c := range []struct {
		verb string
		args map[string]any
		want string
	}{
		{"retire", map[string]any{}, "retire --json"},
		{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
			"retire approve anchor postgres --why moved"},
		{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
			"retire reject anchor postgres --why no"},
		{"cleanup", map[string]any{}, "cleanup list --json"},
		{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
			"cleanup delete anchor postgres x --why gone"},
		{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
		{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
			"cleanup delete --older-than 30 --why tidy --confirm"},
	} {
		argv, err := argvFor(c.verb, c.args)
		if err != nil || strings.Join(argv, " ") != c.want {
			t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
		}
	}
	for _, args := range []map[string]any{
		{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
		{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
	} {
		if _, err := argvFor("retire", args); err == nil {
			t.Errorf("retire %v was composed", args)
		}
	}
	for _, args := range []map[string]any{
		{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
		{"older-than": "30"},
		{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
	} {
		if _, err := argvFor("cleanup", args); err == nil {
			t.Errorf("cleanup %v was composed", args)
		}
	}
	if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
		repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
		t.Error("the generic verb would let a retirement or a deletion through without a why")
	}
}

// A mark-only provider's retired consumer keeps its access; the listing and the approval say so
// rather than claiming it was disabled (ADR 0230).