Hear what providers retire, and let a person approve, reject and delete (hq ADR 0230)

A provider now waits for a person before retiring more than three consumers
or half of what it holds, and deletes only when asked. The controller is that
person's way in: it keeps waiting and rejected sets as conditions, answers them
with retire approve|reject, lists and deletes retired consumers through the
provider's own tools on its machine, records each act in the hand-act log, and
probes for anything retired longer than thirty days (D11).
This commit is contained in:
jochen
2026-10-06 14:41:15 +02:00
parent 298daa6fbe
commit 68009b16fe
23 changed files with 1678 additions and 27 deletions
+460
View File
@@ -0,0 +1,460 @@
package main
import (
"context"
"encoding/json"
"flag"
"os"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
// (novox/hq ADR 0230): what a provider says becomes a condition a person answers, and the verbs that
// answer it ask the provider — never anything else — for exactly what it said.
func waitingWord(module, node string, names ...string) link.Retirement {
r := link.Retirement{Module: module, Provider: "postgres-database", ProviderNode: node, Change: link.RetireWaiting,
Held: 7, Bound: "more than 3, or more than half of the 7 held", At: time.Now(), Since: time.Now()}
for _, n := range names {
r.Consumers = append(r.Consumers, link.RetiredConsumer{Consumer: n, Node: "laptop"})
}
return r
}
func openKeys(t *testing.T, k *conditions.Keeper) map[string]conditions.Condition {
t.Helper()
all, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range all {
out[c.Key] = c
}
return out
}
func TestARetirementWaitingIsUrgentARejectionAWarningAndARetirementClears(t *testing.T) {
k, _ := withConditionsInMemory(t)
var recorded []link.HandAct
r := retirements{keeper: func() *conditions.Keeper { return k },
record: func(_ context.Context, a link.HandAct) error { recorded = append(recorded, a); return nil }}
ctx := t.Context()
waiting := waitingWord("postgres", "anchor", "a", "b", "c", "d")
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
open := openKeys(t, k)
c, ok := open["provider.postgres.anchor.retire"]
if !ok || c.Kind != kindRetireWaiting || c.Severity != conditions.Urgent {
t.Fatalf("waiting is not an urgent retire-waiting condition: %+v", open)
}
for _, want := range []string{"a (laptop), b (laptop), c (laptop), d (laptop)", "retire approve anchor postgres",
"retire reject anchor postgres", "more than 3"} {
if !strings.Contains(c.Summary, want) {
t.Errorf("the condition does not say %q: %s", want, c.Summary)
}
}
// Rejected, through the controller: the urgent one becomes a warning, and nothing is recorded here —
// the verb recorded it before it asked.
rejected := waiting
rejected.Change, rejected.By, rejected.Why, rejected.Via = link.RetireRejected, "operator", "moving them", link.ViaController
if err := r.Retired(ctx, rejected); err != nil {
t.Fatal(err)
}
open = openKeys(t, k)
if _, still := open["provider.postgres.anchor.retire"]; still {
t.Fatal("a rejection left the provider waiting")
}
if c, ok := open["provider.postgres.anchor.retire-rejected"]; !ok || c.Severity != conditions.Warning ||
c.Kind != kindRetireRejected || !strings.Contains(c.Summary, "moving them") {
t.Fatalf("a rejection is not a warning saying why: %+v", open)
}
if len(recorded) != 0 {
t.Fatalf("an act through the controller was recorded twice: %+v", recorded)
}
// Approved afterwards, asked of the provider directly: both cleared, and recorded by hand here.
approved := waiting
approved.Change, approved.By, approved.Why, approved.Via = link.RetireApproved, "someone", "done moving", ""
if err := r.Retired(ctx, approved); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("an approval left conditions open: %+v", open)
}
if len(recorded) != 1 || recorded[0].Verb != "retire approve" || recorded[0].By != "someone" ||
!strings.Contains(recorded[0].Why, "directly") || !slices.Contains(recorded[0].Args, "d") {
t.Fatalf("an approval outside the controller was not recorded: %+v", recorded)
}
// Waiting again, then the set settles (asked for again): cleared. And retired clears too.
for _, change := range []string{link.RetireSettled, link.RetireRetired} {
if err := r.Retired(ctx, waiting); err != nil {
t.Fatal(err)
}
done := waiting
done.Change = change
if err := r.Retired(ctx, done); err != nil {
t.Fatal(err)
}
if open := openKeys(t, k); len(open) != 0 {
t.Fatalf("%s left conditions open: %+v", change, open)
}
}
// A deletion asked of the provider directly is recorded too.
deleted := link.Retirement{Module: "postgres", ProviderNode: "anchor", Change: link.RetireDeleted, By: "x",
Why: "gone for good", At: time.Now(), Consumers: []link.RetiredConsumer{{Consumer: "a"}}}
if err := r.Retired(ctx, deleted); err != nil {
t.Fatal(err)
}
if len(recorded) != 2 || recorded[1].Verb != "cleanup delete" || recorded[1].Cause != kindCleanupWaiting {
t.Fatalf("a deletion outside the controller was not recorded: %+v", recorded)
}
}
func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *testing.T) {
body, _ := json.Marshal(map[string]any{"provider": "oidc-client", "provider-node": "anchor", "change": "waiting",
"held": 2, "consumers": []map[string]any{{"consumer": "x"}, {"consumer": "y"}}, "module": "liar"})
r, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement", body)
if err != nil || r.Module != "idp" || len(r.Consumers) != 2 {
t.Fatalf("%+v %v", r, err)
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.retirement",
[]byte(`{"provider-node":"anchor","change":"vanished"}`)); err == nil {
t.Fatal("a change the mesh has no name for was read")
}
if _, err := link.ReadRetirement("mesh.mod.idp.event.provisioner.failing", body); err == nil {
t.Fatal("a failing word was read as a retirement")
}
k, _ := withConditionsInMemory(t)
if err := (retirements{keeper: func() *conditions.Keeper { return k }}).Retired(t.Context(), r); err != nil {
t.Fatal(err)
}
if _, ok := openKeys(t, k)["provider.idp.anchor.retire"]; !ok {
t.Fatal("not kept under the emitter the subject names")
}
}
func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "pg", Version: "1",
Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}})
if _, err := assign(ctx, open, "anchor", "pg"); err != nil {
t.Fatal(err)
}
r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }}
for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} {
if err := r.Retired(ctx, w); err != nil {
t.Fatal(err)
}
}
if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{
cleanupObservation(providerInstance{Node: "anchor", Module: "gone"},
[]link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()),
}); err != nil {
t.Fatal(err)
}
all, _ := conditionsFrom.Open(ctx)
if len(all) != 3 {
t.Fatalf("%+v", all)
}
if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil {
t.Fatal(err)
}
left := openKeys(t, conditionsFrom)
if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" {
t.Fatalf("only the assigned provider's waiting should stay: %+v", left)
}
}
// fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked.
type fakeProvider struct {
mu sync.Mutex
state link.RetirementState
asked []map[string]any
deleted []string
approved []string
rejected []string
}
func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) {
t.Helper()
answer := func(m *nats.Msg, result any, refusal string) {
body, _ := json.Marshal(map[string]any{"result": result, "error": refusal, "node": node})
_ = m.Respond(body)
}
names := func(args map[string]any) []string {
var out []string
for _, v := range args["consumers"].([]any) {
out = append(out, v.(string))
}
sort.Strings(out)
return out
}
set := func(cs []link.RetiredConsumer) []string {
var out []string
for _, c := range cs {
out = append(out, c.Consumer)
}
sort.Strings(out)
return out
}
for _, tool := range []string{link.ToolRetirement, link.ToolRetireApprove, link.ToolRetireReject, link.ToolRetiredDelete} {
tool := tool
sub, err := conn.Subscribe(link.ModuleToolOn(module, tool, node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
f.asked = append(f.asked, map[string]any{"tool": tool, "args": args})
switch tool {
case link.ToolRetirement:
answer(m, f.state, "")
case link.ToolRetireApprove:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.approved = names(args)
answer(m, map[string]any{"retired": f.approved}, "")
case link.ToolRetireReject:
if f.state.Waiting == nil || !slices.Equal(names(args), set(f.state.Waiting.Consumers)) {
answer(m, nil, "not the set I wait with")
return
}
f.rejected = names(args)
answer(m, map[string]any{"kept": f.rejected}, "")
case link.ToolRetiredDelete:
if args["confirm"] != args["consumer"] || args["why"] == "" || args["via"] != link.ViaController {
answer(m, nil, "confirm, why and via")
return
}
f.deleted = append(f.deleted, args["consumer"].(string))
answer(m, map[string]any{"deleted": args["consumer"], "freed_bytes": 1024}, "")
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func onATestBus(t *testing.T) *nats.Conn {
t.Helper()
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
return js.Conn()
}
func whyFlags(t *testing.T, why string) handActFlags {
t.Helper()
set := flag.NewFlagSet("t", flag.ContinueOnError)
f := addHandActFlags(set)
if err := set.Parse([]string{"--why", why}); err != nil {
t.Fatal(err)
}
return f
}
func handActsBy(t *testing.T, conn *nats.Conn, verb string) []link.HandAct {
t.Helper()
acts, err := link.HandActs(t.Context(), conn, time.Now().Add(-time.Minute))
if err != nil {
t.Fatal(err)
}
var out []link.HandAct
for _, a := range acts {
if a.Verb == verb {
out = append(out, a)
}
}
return out
}
func retiredDaysAgo(name string, days int) link.RetiredConsumer {
size := int64(4096)
return link.RetiredConsumer{Consumer: name, Kind: "consumer", Why: "the mesh stopped asking for it",
RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size}
}
func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6}
fake.serve(t, conn, "pg-approve", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-approve"}
said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) })
if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") {
t.Fatalf("approved %v; said %s", fake.approved, said)
}
acts := handActsBy(t, conn, "retire approve")
if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting ||
acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" {
t.Fatalf("the approval is not in the hand-act log: %+v", acts)
}
// Reject, on another provider: the same set, and nothing approved.
other := &fakeProvider{state: fake.state}
other.serve(t, conn, "pg-reject", "anchor")
printed(t, func() error {
return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false,
whyFlags(t, "still moving"))
})
if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil {
t.Fatalf("rejected %v approved %v", other.rejected, other.approved)
}
// Nothing waiting: refused, nothing asked but the question.
idle := &fakeProvider{}
idle.serve(t, conn, "pg-idle", "anchor")
if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true,
whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") {
t.Fatalf("%v", err)
}
if len(idle.asked) != 1 {
t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked)
}
}
func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) {
conn := onATestBus(t)
fake := &fakeProvider{}
fake.state.Held = []string{"active"}
fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)}
fake.serve(t, conn, "pg-clean", "anchor")
p := providerInstance{Node: "anchor", Module: "pg-clean"}
said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) })
if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
// An active consumer, or one it does not hold, is refused before the provider is asked to delete.
for _, name := range []string{"active", "nobody"} {
if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil ||
!strings.Contains(err.Error(), "only a retired consumer is deleted") {
t.Fatalf("%s: %v", name, err)
}
}
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("more was deleted: %v", fake.deleted)
}
if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" {
t.Fatalf("the deletion is not in the hand-act log: %+v", acts)
}
// Older than: listed, and nothing deleted without confirm; with it, only the old one.
listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now())
if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 {
t.Fatalf("%+v", listing)
}
fake.deleted = nil
said = printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, false, whyFlags(t, "tidy")) })
if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") ||
strings.Contains(said, "young") {
t.Fatalf("deleted %v; said %s", fake.deleted, said)
}
printed(t, func() error { return deleteFrom(t.Context(), conn, listing, 30, true, whyFlags(t, "tidy")) })
if !slices.Equal(fake.deleted, []string{"old"}) {
t.Fatalf("confirmed, deleted %v", fake.deleted)
}
}
func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) {
conn := onATestBus(t)
old := &fakeProvider{}
old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)}
old.serve(t, conn, "pg-d11-old", "anchor")
young := &fakeProvider{}
young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)}
young.serve(t, conn, "pg-d11-young", "anchor")
instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"},
// Nothing serves this one: a provider older than the question is passed over, not a failure.
{Node: "anchor", Module: "pg-d11-predates"}}
found, err := retiredTooLong(t.Context(), conn, instances, time.Now())
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting ||
found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") ||
strings.Contains(found[0].Summary, "fresh") {
t.Fatalf("%+v", found)
}
}
func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want string
}{
{"retire", map[string]any{}, "retire --json"},
{"retire", map[string]any{"answer": "approve", "node": "anchor", "module": "postgres", "why": "moved"},
"retire approve anchor postgres --why moved"},
{"retire", map[string]any{"answer": "reject", "node": "anchor", "module": "postgres", "why": "no"},
"retire reject anchor postgres --why no"},
{"cleanup", map[string]any{}, "cleanup list --json"},
{"cleanup", map[string]any{"node": "anchor", "module": "postgres", "consumer": "x", "why": "gone"},
"cleanup delete anchor postgres x --why gone"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy"}, "cleanup delete --older-than 30 --why tidy"},
{"cleanup", map[string]any{"older-than": "30", "why": "tidy", "confirm": "true"},
"cleanup delete --older-than 30 --why tidy --confirm"},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%s %v: %q %v, want %q", c.verb, c.args, argv, err, c.want)
}
}
for _, args := range []map[string]any{
{"answer": "approve", "node": "anchor", "module": "postgres"}, // no why
{"answer": "maybe", "node": "anchor", "module": "postgres", "why": "x"},
} {
if _, err := argvFor("retire", args); err == nil {
t.Errorf("retire %v was composed", args)
}
}
for _, args := range []map[string]any{
{"node": "anchor", "module": "postgres", "consumer": "x"}, // no why
{"older-than": "30"},
{"consumer": "x", "older-than": "30", "node": "a", "module": "b", "why": "y"},
} {
if _, err := argvFor("cleanup", args); err == nil {
t.Errorf("cleanup %v was composed", args)
}
}
if repairingCommand([]string{"cleanup", "delete", "a", "b", "c"}) == "" ||
repairingCommand([]string{"retire", "approve", "a", "b"}) == "" || repairingCommand([]string{"retire"}) != "" {
t.Error("the generic verb would let a retirement or a deletion through without a why")
}
}