Files
mesh-controller/cmd/mesh-controller/retirement_test.go
T
jochen be92762969 Give every test a bus of its own, at the release the mesh runs (hq ADR 0237)
The live tests reached one shared bus and assert, read and remove the mesh's own objects by
their fixed names, so packages run in parallel deleted what each other read and the suite
passed only one package at a time; a red suite read as noise. internal/testbus starts a server
per test, linked in at the nats-server release go.mod pins, and a test holds that pin to the
catalogue's bus image and to the facts snapshot's bus when there is one, so the tests never run
a bus the mesh does not. The waiter test read a timing (the most connections held at one look)
and now reads the state it means (the fewest held across the wait). make check runs the packages
in parallel under the race detector, with a timeout.
2026-10-06 21:17:02 +02:00

482 lines
19 KiB
Go

package main
import (
"context"
"encoding/json"
"flag"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
// answer it ask the provider — never anything else — for exactly what it said.
func waitingWord(module, node string, names ...string) link.Retirement {
r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
for _, n := range names {
r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
}
return r
}
func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
t.Helper()
all, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range all {
out[c.Key] = c
}
return out
}
func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
k, _ := withConditionsInMemory(t)
var recorded []link.HandAct
r := retirements{keeper: func() *conditions.Keeper { return k },
record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
ctx := t.Context()
waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
open := openKeys(t, k)
c, ok := open["provider.postgres.anchor.retire"]
if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
}
for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
"retire reject anchor postgres", "more than 3"} {
if !strings.Contains(c.Summary, want) {
t.Errorf("the condition does not say %q: %s", want, c.Summary)
}
}
// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
// the verb recorded it before it asked.
rejected := waiting
rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
if err := r.Retired(ctx, rejected); err != nil {
t.Fatal(err)
}
open = openKeys(t, k)
if _, still := open["provider.postgres.anchor.retire"]; still {
t.Fatal("a rejection left the provider waiting")
}
if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
t.Fatalf("a rejection is not a warning saying why: %+v", open)
}
if len(recorded) != 0 {
t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
}
// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
approved := waiting
approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
if err := r.Retired(ctx, approved); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("an approval left conditions open: %+v", open)
}
if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
}
// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
for _, change := range []string{link.RetireSettled, link.RetireRetired} {
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
done := waiting
done.Change = change
if err := r.Retired(ctx, done); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("%s left conditions open: %+v", change, open)
}
}
// A deletion asked of the provider directly is recorded too.
deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
if err := r.Retired(ctx, deleted); err != nil {
t.Fatal(err)
}
if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
}
}
func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
t.Fatalf("%+v %v", r, err)
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
t.Fatal("a change the mesh has no name for was read")
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
t.Fatal("a failing word was read as a retirement")
}
k, _ := withConditionsInMemory(t)
if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
t.Fatal(err)
}
if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
t.Fatal("not kept under the emitter the subject names")
}
}
func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
t.Fatal(err)
}
r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
if err := r.Retired(ctx, w); err != nil {
t.Fatal(err)
}
}
if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
}); err != nil {
t.Fatal(err)
}
all, _ := conditionsFrom.Open(ctx)
if len(all) != 3 {
t.Fatalf("%+v", all)
}
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
t.Fatal(err)
}
left := openKeys(t, conditionsFrom)
if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
}
}
// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
type fakeProvider struct {
mu sync.Mutex
state link.RetirementState
asked []map[string]any
deleted []string
approved []string
rejected []string
}
func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
t.Helper()
answer := func(m *nats.Msg, result any, refusal string) {
body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
_ = m.Respond(body)
}
names := func(args map[string]any) []string {
var out []string
for _, v := range args["consumers"].([]any) {
out = append(out, v.(string))
}
sort.Strings(out)
return out
}
set := func(cs []link.RetiredConsumer) []string {
var out []string
for _, c := range cs {
out = append(out, c.Consumer)
}
sort.Strings(out)
return out
}
for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
tool := tool
sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
switch tool {
case link.ToolRetirement:
answer(m, f.state, "")
case link.ToolRetireApprove:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.approved = names(args)
answer(m, map[string]any{"retired": f.approved}, "")
case link.ToolRetireReject:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.rejected = names(args)
answer(m, map[string]any{"kept": f.rejected}, "")
case link.ToolRetiredDelete:
if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
answer(m, nil, "confirm, why and via")
return
}
f.deleted = append(f.deleted, args["consumer"].(string))
answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func onATestBus(t *testing.T) *nats.Conn {
t.Helper()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
return js.Conn()
}
func whyFlags(t *testing.T, why string) handActFlags {
t.Helper()
set := flag.NewFlagSet("t", flag.ContinueOnError)
f := addHandActFlags(set)
if err := set.Parse([]string{"--why", why}); err != nil {
t.Fatal(err)
}
return f
}
func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
t.Helper()
acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
if err != nil {
t.Fatal(err)
}
var out []link.HandAct
for _, a := range acts {
if a.Verb == verb {
out = append(out, a)
}
}
return out
}
func retiredDaysAgo(name string, days int) link.RetiredConsumer {
size := int64(4096)
return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
}
func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
fake.serve(t, conn, "pg-approve", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-approve"}
said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
t.Fatalf("approved %v; said %s", fake.approved, said)
}
acts := handActsBy(t, conn, "retire approve")
if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
t.Fatalf("the approval is not in the hand-act log: %+v", acts)
}
// Reject, on another provider: the same set, and nothing approved.
other := &fakeProvider{state: fake.state}
other.serve(t, conn, "pg-reject", "anchor")
printed(t, func() error {
return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
whyFlags(t, "still moving"))
})
if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
}
// Nothing waiting: refused, nothing asked but the question.
idle := &fakeProvider{}
idle.serve(t, conn, "pg-idle", "anchor")
if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
t.Fatalf("%v", err)
}
if len(idle.asked) != 1 {
t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
}
}
func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Held = []string{"active"}
fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
fake.serve(t, conn, "pg-clean", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-clean"}
said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
for _, name := range []string{"active", "nobody"} {
if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
!strings.Contains(err.Error(), "only a retired consumer is deleted") {
t.Fatalf("%s: %v", name, err)
}
}
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("more was deleted: %v", fake.deleted)
}
if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
}
// Older than: listed, and nothing deleted without confirm; with it, only the old one.
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
t.Fatalf("%+v", listing)
}
fake.deleted = nil
said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) })
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
strings.Contains(said, "young") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) })
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("confirmed, deleted %v", fake.deleted)
}
}
func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
conn := onATestBus(t)
old := &fakeProvider{}
old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
old.serve(t, conn, "pg-d11-old", "anchor")
young := &fakeProvider{}
young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
young.serve(t, conn, "pg-d11-young", "anchor")
instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
// Nothing serves this one: a provider older than the question is passed over, not a failure.
{Node: "anchor", Module: "pg-d11-predates"}}
found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
strings.Contains(found[0].Summary, "fresh") {
t.Fatalf("%+v", found)
}
}
func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want string
}{
{"retire", map[string]any{}, "retire --json"},
{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
"retire approve anchor postgres --why moved"},
{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
"retire reject anchor postgres --why no"},
{"cleanup", map[string]any{}, "cleanup list --json"},
{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
"cleanup delete anchor postgres x --why gone"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
"cleanup delete --older-than 30 --why tidy --confirm"},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
}
}
for _, args := range []map[string]any{
{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
} {
if _, err := argvFor("retire", args); err == nil {
t.Errorf("retire %v was composed", args)
}
}
for _, args := range []map[string]any{
{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
{"older-than": "30"},
{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
} {
if _, err := argvFor("cleanup", args); err == nil {
t.Errorf("cleanup %v was composed", args)
}
}
if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
t.Error("the generic verb would let a retirement or a deletion through without a why")
}
}
// A mark-only provider's retired consumer keeps its access; the listing and the approval say so
// rather than claiming it was disabled (ADR 0230).
func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
kept := retiredDaysAgo("ledger", 3)
kept.Access = "kept"
fake.state.Retired = []link.RetiredConsumer{kept}
fake.state.RetiresBy = "mark-only"
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2}
fake.serve(t, conn, "vault-mark", "anchor")
p := providerInstance{Node: "anchor", Module: "vault-mark"}
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
said := printed(t, func() error { return printRetired(listing, false) })
if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" {
t.Fatalf("%s %+v", said, listing)
}
said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) })
if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") {
t.Fatal(said)
}
}