package main import ( "context" "encoding/json" "flag" "os" "slices" "sort" "strings" "sync" "testing" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" ) // A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person // (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that // answer it ask the provider — never anything else — for exactly what it said. func waitingWord(module, node string, names ...string) link.Retirement { r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting, Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()} for _, n := range names { r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"}) } return r } func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition { t.Helper() all, err := k.Open(t.Context()) if err != nil { t.Fatal(err) } out := map[string]conditions.Condition{} for _, c := range all { out[c.Key] = c } return out } func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) { k, _ := withConditionsInMemory(t) var recorded []link.HandAct r := retirements{keeper: func() *conditions.Keeper { return k }, record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }} ctx := t.Context() waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d") if err := r.Retired(ctx, waiting); err != nil { t.Fatal(err) } open := openKeys(t, k) c, ok := open["provider.postgres.anchor.retire"] if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent { t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open) } for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres", "retire reject anchor postgres", "more than 3"} { if !strings.Contains(c.Summary, want) { t.Errorf("the condition does not say %q: %s", want, c.Summary) } } // Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here — // the verb recorded it before it asked. rejected := waiting rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController if err := r.Retired(ctx, rejected); err != nil { t.Fatal(err) } open = openKeys(t, k) if _, still := open["provider.postgres.anchor.retire"]; still { t.Fatal("a rejection left the provider waiting") } if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning || c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") { t.Fatalf("a rejection is not a warning saying why: %+v", open) } if len(recorded) != 0 { t.Fatalf("an act through the controller was recorded twice: %+v", recorded) } // Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here. approved := waiting approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", "" if err := r.Retired(ctx, approved); err != nil { t.Fatal(err) } if open := openKeys(t, k); len(open) != 0 { t.Fatalf("an approval left conditions open: %+v", open) } if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" || !strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") { t.Fatalf("an approval outside the controller was not recorded: %+v", recorded) } // Waiting again, then the set settles (asked for again): cleared. And retired clears too. for _, change := range []string{link.RetireSettled, link.RetireRetired} { if err := r.Retired(ctx, waiting); err != nil { t.Fatal(err) } done := waiting done.Change = change if err := r.Retired(ctx, done); err != nil { t.Fatal(err) } if open := openKeys(t, k); len(open) != 0 { t.Fatalf("%s left conditions open: %+v", change, open) } } // A deletion asked of the provider directly is recorded too. deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x", Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}} if err := r.Retired(ctx, deleted); err != nil { t.Fatal(err) } if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting { t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded) } } func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) { body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting", "held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"}) r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body) if err != nil || r.Module != "idp" || len(r.Consumers) != 2 { t.Fatalf("%+v %v", r, err) } if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", []byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil { t.Fatal("a change the mesh has no name for was read") } if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil { t.Fatal("a failing word was read as a retirement") } k, _ := withConditionsInMemory(t) if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil { t.Fatal(err) } if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok { t.Fatal("not kept under the emitter the subject names") } } func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) { open := aMesh(t) ctx := t.Context() register(t, open, catalogue.Manifest{Module: "pg", Version: "1", Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}}) if _, err := assign(ctx, open, "anchor", "pg"); err != nil { t.Fatal(err) } r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }} for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} { if err := r.Retired(ctx, w); err != nil { t.Fatal(err) } } if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{ cleanupObservation(providerInstance{Node: "anchor", Module: "gone"}, []link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()), }); err != nil { t.Fatal(err) } all, _ := conditionsFrom.Open(ctx) if len(all) != 3 { t.Fatalf("%+v", all) } if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil { t.Fatal(err) } left := openKeys(t, conditionsFrom) if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" { t.Fatalf("only the assigned provider's waiting should stay: %+v", left) } } // fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked. type fakeProvider struct { mu sync.Mutex state link.RetirementState asked []map[string]any deleted []string approved []string rejected []string } func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) { t.Helper() answer := func(m *nats.Msg, result any, refusal string) { body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node}) _ = m.Respond(body) } names := func(args map[string]any) []string { var out []string for _, v := range args["consumers"].([]any) { out = append(out, v.(string)) } sort.Strings(out) return out } set := func(cs []link.RetiredConsumer) []string { var out []string for _, c := range cs { out = append(out, c.Consumer) } sort.Strings(out) return out } for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} { tool := tool sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) { f.mu.Lock() defer f.mu.Unlock() var args map[string]any _ = json.Unmarshal(m.Data, &args) f.asked = append(f.asked, map[string]any{"tool": tool, "args": args}) switch tool { case link.ToolRetirement: answer(m, f.state, "") case link.ToolRetireApprove: if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) { answer(m, nil, "not the set I wait with") return } f.approved = names(args) answer(m, map[string]any{"retired": f.approved}, "") case link.ToolRetireReject: if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) { answer(m, nil, "not the set I wait with") return } f.rejected = names(args) answer(m, map[string]any{"kept": f.rejected}, "") case link.ToolRetiredDelete: if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController { answer(m, nil, "confirm, why and via") return } f.deleted = append(f.deleted, args["consumer"].(string)) answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "") } }) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) } if err := conn.Flush(); err != nil { t.Fatal(err) } } func onATestBus(t *testing.T) *nats.Conn { t.Helper() url := os.Getenv("MESH_TEST_NATS") if url == "" { t.Skip("MESH_TEST_NATS unset") } js, err := broker.Dial(url) if err != nil { t.Fatal(err) } t.Cleanup(js.Close) if err := js.EnsureControllerBuckets(); err != nil { t.Fatal(err) } before := handActConn handActConn = js.Conn() t.Cleanup(func() { handActConn = before }) return js.Conn() } func whyFlags(t *testing.T, why string) handActFlags { t.Helper() set := flag.NewFlagSet("t", flag.ContinueOnError) f := addHandActFlags(set) if err := set.Parse([]string{"--why", why}); err != nil { t.Fatal(err) } return f } func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct { t.Helper() acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute)) if err != nil { t.Fatal(err) } var out []link.HandAct for _, a := range acts { if a.Verb == verb { out = append(out, a) } } return out } func retiredDaysAgo(name string, days int) link.RetiredConsumer { size := int64(4096) return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it", RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size} } func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) { conn := onATestBus(t) fake := &fakeProvider{} fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6} fake.serve(t, conn, "pg-approve", "anchor") p := providerInstance{Node: "anchor", Module: "pg-approve"} said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) }) if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") { t.Fatalf("approved %v; said %s", fake.approved, said) } acts := handActsBy(t, conn, "retire approve") if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting || acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" { t.Fatalf("the approval is not in the hand-act log: %+v", acts) } // Reject, on another provider: the same set, and nothing approved. other := &fakeProvider{state: fake.state} other.serve(t, conn, "pg-reject", "anchor") printed(t, func() error { return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false, whyFlags(t, "still moving")) }) if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil { t.Fatalf("rejected %v approved %v", other.rejected, other.approved) } // Nothing waiting: refused, nothing asked but the question. idle := &fakeProvider{} idle.serve(t, conn, "pg-idle", "anchor") if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true, whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") { t.Fatalf("%v", err) } if len(idle.asked) != 1 { t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked) } } func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) { conn := onATestBus(t) fake := &fakeProvider{} fake.state.Held = []string{"active"} fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)} fake.serve(t, conn, "pg-clean", "anchor") p := providerInstance{Node: "anchor", Module: "pg-clean"} said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) }) if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") { t.Fatalf("deleted %v; said %s", fake.deleted, said) } // An active consumer, or one it does not hold, is refused before the provider is asked to delete. for _, name := range []string{"active", "nobody"} { if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "only a retired consumer is deleted") { t.Fatalf("%s: %v", name, err) } } if !slices.Equal(fake.deleted, []string{"old"}) { t.Fatalf("more was deleted: %v", fake.deleted) } if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" { t.Fatalf("the deletion is not in the hand-act log: %+v", acts) } // Older than: listed, and nothing deleted without confirm; with it, only the old one. listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 { t.Fatalf("%+v", listing) } fake.deleted = nil said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) }) if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || strings.Contains(said, "young") { t.Fatalf("deleted %v; said %s", fake.deleted, said) } printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) }) if !slices.Equal(fake.deleted, []string{"old"}) { t.Fatalf("confirmed, deleted %v", fake.deleted) } } func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) { conn := onATestBus(t) old := &fakeProvider{} old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)} old.serve(t, conn, "pg-d11-old", "anchor") young := &fakeProvider{} young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)} young.serve(t, conn, "pg-d11-young", "anchor") instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"}, // Nothing serves this one: a provider older than the question is passed over, not a failure. {Node: "anchor", Module: "pg-d11-predates"}} found, err := retiredTooLong(t.Context(), conn, instances, time.Now()) if err != nil { t.Fatal(err) } if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting || found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") || strings.Contains(found[0].Summary, "fresh") { t.Fatalf("%+v", found) } } func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) { for _, c := range []struct { verb string args map[string]any want string }{ {"retire", map[string]any{}, "retire --json"}, {"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"}, "retire approve anchor postgres --why moved"}, {"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"}, "retire reject anchor postgres --why no"}, {"cleanup", map[string]any{}, "cleanup list --json"}, {"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"}, "cleanup delete anchor postgres x --why gone"}, {"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"}, {"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"}, "cleanup delete --older-than 30 --why tidy --confirm"}, } { argv, err := argvFor(c.verb, c.args) if err != nil || strings.Join(argv, " ") != c.want { t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want) } } for _, args := range []map[string]any{ {"answer": "approve", "node": "anchor", "module": "postgres"}, // no why {"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"}, } { if _, err := argvFor("retire", args); err == nil { t.Errorf("retire %v was composed", args) } } for _, args := range []map[string]any{ {"node": "anchor", "module": "postgres", "consumer": "x"}, // no why {"older-than": "30"}, {"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"}, } { if _, err := argvFor("cleanup", args); err == nil { t.Errorf("cleanup %v was composed", args) } } if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" || repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" { t.Error("the generic verb would let a retirement or a deletion through without a why") } } // A mark-only provider's retired consumer keeps its access; the listing and the approval say so // rather than claiming it was disabled (ADR 0230). func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) { conn := onATestBus(t) fake := &fakeProvider{} kept := retiredDaysAgo("ledger", 3) kept.Access = "kept" fake.state.Retired = []link.RetiredConsumer{kept} fake.state.RetiresBy = "mark-only" fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2} fake.serve(t, conn, "vault-mark", "anchor") p := providerInstance{Node: "anchor", Module: "vault-mark"} listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) said := printed(t, func() error { return printRetired(listing, false) }) if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" { t.Fatalf("%s %+v", said, listing) } said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) }) if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") { t.Fatal(said) } }