Merge pull request 'postgres, keycloak: retire a consumer, delete only on a person's word (hq ADR 0230)' (#91) from feat/retired-consumers into main
mesh/delivery held for a person: merged without a passing check: only a person decides that it goes on
mesh/delivery held for a person: merged without a passing check: only a person decides that it goes on
This commit was merged in pull request #91.
This commit is contained in:
@@ -44,6 +44,16 @@ check or secret keeps failing for 5 minutes with no success in between is announ
|
||||
every 15 minutes while the failure lasts. `provisioner.recovered` follows the first success
|
||||
(ADR 0224), and the controller shows the latest one in `status`.
|
||||
|
||||
A consumer the mesh no longer asks for is **retired**, not deleted (novox/hq ADR 0230): its client is
|
||||
disabled — Keycloak refuses its authorization and token requests — and marked with `mesh.retired` and
|
||||
`mesh.retired-why`; its secret, redirects and mappers are kept, and asked for again it is enabled as it
|
||||
was. Retiring waits five passes and ten minutes, and for a person's `retire approve` when it is more
|
||||
than three clients or more than half of those held. Only `cleanup delete` removes a client, and only a disabled one the
|
||||
mesh made. The tools `provisioner_retirement`, `provisioner_retire_approve`, `provisioner_retire_reject`
|
||||
and `provisioner_delete` are what the controller's `retire` and `cleanup` verbs ask.
|
||||
`MESH_KEYCLOAK_LIVE_URL` and `MESH_KEYCLOAK_LIVE_PASSWORD` run `live_retire_test.go` against a throwaway
|
||||
server.
|
||||
|
||||
## Tools
|
||||
|
||||
The realm, user, client, group and role tools (`keycloak_list_realms`, `keycloak_create_user`,
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package main
|
||||
|
||||
// The reconcile loop every provider shares, as the TypeScript SDK's runProvisioner runs it
|
||||
// (@novox/mesh-sdk/provisioner, 0.1.11). The Go SDK has no provisioner yet, so this module carries
|
||||
// (@novox/mesh-sdk/provisioner, 0.1.12). The Go SDK has no provisioner yet, so this module carries
|
||||
// the loop itself, line for line in behaviour; when the Go SDK grows one, this file is what moves
|
||||
// there (novox/hq ADR 0039: the loop is the SDK's, the adapter is the module's).
|
||||
//
|
||||
// Read the contributions the mesh delivered; bring each consumer's resource into being through the
|
||||
// adapter, under the login and password the mesh minted; withdraw what the mesh no longer asks for.
|
||||
// adapter, under the login and password the mesh minted; retire what the mesh no longer asks for.
|
||||
// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).**
|
||||
//
|
||||
// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer
|
||||
@@ -17,18 +17,16 @@ package main
|
||||
// identity provider failed every consumer 31,000 times in a day and said so only in its journal
|
||||
// (novox/hq issue 179).
|
||||
//
|
||||
// **A reconcile that would withdraw more than its bound stops, and says so (novox/hq to-be 45 Phase 2,
|
||||
// ADR 0227 rule 4).** Withdrawing more than WithdrawAtOnce consumers in one pass — or more than
|
||||
// WithdrawFraction of those this process holds — is the shape of issue 241, where one misread file
|
||||
// withdrew seven at once. Such a pass withdraws nothing: each consumer it would have withdrawn is kept,
|
||||
// announced `provisioner.failing` with the class `withdrawal-braked` (the controller raises it as a
|
||||
// condition), and said. While the same consumers stay unasked for, one is released every ReleaseEvery,
|
||||
// said and announced as it goes, so an intended unassignment of many completes without a hand and a
|
||||
// mistaken one costs at most one consumer an hour while the operator is told. Withdrawal never destroys
|
||||
// data (issue 241's second half), so a release is a login locked, not a database dropped.
|
||||
// **A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
// (novox/hq ADR 0230, the operator's model of 2026-10-06).** Retiring disables its access — reversibly —
|
||||
// and marks its login and data "to delete" with when and why; nothing is deleted. Asked for again, the
|
||||
// ordinary create re-enables it as it was. It is retired only once the same set has gone unasked in
|
||||
// StablePasses consecutive passes and for StableFor, and a set larger than the bound waits for a
|
||||
// person. retirement.go.
|
||||
//
|
||||
// Carried, identical, by every Go provider until the Go SDK has the loop: postgres and keycloak.
|
||||
// Each module's `harness_same_test.go` fails when its copy and the other's differ.
|
||||
// Each module's `harness_same_test.go` fails when its copy and the other's differ (this file and
|
||||
// retirement.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -37,8 +35,8 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -59,11 +57,21 @@ type Provision struct {
|
||||
|
||||
// Adapter is the per-service half.
|
||||
type Adapter interface {
|
||||
// Create brings the consumer into being under the mesh's login and password — and, for one that
|
||||
// was retired, re-enables it as it was and clears its mark to delete.
|
||||
Create(ctx context.Context, p Provision) error
|
||||
// Remove withdraws what Create made; derived is what the mesh last derived, remembered here.
|
||||
Remove(ctx context.Context, as string, derived map[string]any) error
|
||||
// Retire disables the consumer's access, reversibly, and marks its login and data to delete with
|
||||
// when and why. It deletes nothing (novox/hq ADR 0230). derived is what the mesh last derived,
|
||||
// remembered here; nil for a consumer this process did not make.
|
||||
Retire(ctx context.Context, as string, derived map[string]any, why string, at time.Time) error
|
||||
// Holds says whether the backend still holds the consumer exactly as p says. Read-only.
|
||||
Holds(ctx context.Context, p Provision) (bool, error)
|
||||
// Inventory is what the backend holds that the mesh made: consumers active and retired, read from
|
||||
// the backend itself so a restart forgets nothing. Read-only.
|
||||
Inventory(ctx context.Context) (Inventory, error)
|
||||
// Delete removes one retired consumer — its login and its data — for good. Only ever asked by a
|
||||
// person, through `cleanup delete`; the harness has checked it is retired and not asked for.
|
||||
Delete(ctx context.Context, r Retired) (freedBytes int64, err error)
|
||||
}
|
||||
|
||||
// Harness is the loop's settings and memory.
|
||||
@@ -85,19 +93,19 @@ type Harness struct {
|
||||
// missed the first hears the next, and a standing nobody repeats can be told from one that holds.
|
||||
FailingAfter time.Duration
|
||||
SayAgainEvery time.Duration
|
||||
// WithdrawAtOnce (1) and WithdrawFraction (0.5) bound what one pass may withdraw: more consumers
|
||||
// than WithdrawAtOnce, or a larger share of those held than WithdrawFraction, brakes the pass.
|
||||
// ReleaseEvery (1h) is how often a braked withdrawal lets one consumer go.
|
||||
WithdrawAtOnce int
|
||||
WithdrawFraction float64
|
||||
ReleaseEvery time.Duration
|
||||
// StablePasses (5) is how many consecutive passes must see the same set no longer asked for, and
|
||||
// StableFor (10m) how long it must have held since the first of them, before it is retired. RetireAtOnce (3) and RetireFraction (0.5) bound what is retired without a person:
|
||||
// more consumers than RetireAtOnce, or — where more than one is held — a larger share of those held
|
||||
// than RetireFraction, waits for `retire approve` (novox/hq ADR 0230).
|
||||
StablePasses int
|
||||
StableFor time.Duration
|
||||
RetireAtOnce int
|
||||
RetireFraction float64
|
||||
|
||||
verifiedAt time.Time
|
||||
// braked is every consumer a braked pass kept, by when it was first kept; releasedAt is when the
|
||||
// brake last let one go, and brakeSaid the set it last said, so a pass repeats nothing.
|
||||
braked map[string]time.Time
|
||||
releasedAt time.Time
|
||||
brakeSaid string
|
||||
// mu is held by a pass and by every tool a person asks, so the two never interleave.
|
||||
mu sync.Mutex
|
||||
verifiedAt time.Time
|
||||
r retirement
|
||||
applied map[string]appliedEntry
|
||||
lost map[string]brake
|
||||
waiting map[string]int
|
||||
@@ -131,9 +139,6 @@ const (
|
||||
ClassUnreachable = "unreachable"
|
||||
ClassSecret = "secret-unreadable"
|
||||
ClassRefused = "refused"
|
||||
// ClassWithdrawalBraked is a consumer the mesh no longer asks for, kept because the pass that would
|
||||
// withdraw it would withdraw more than its bound (ADR 0227 rule 4).
|
||||
ClassWithdrawalBraked = "withdrawal-braked"
|
||||
)
|
||||
|
||||
// Classifier is an adapter that can say what class an error of its own is.
|
||||
@@ -196,14 +201,17 @@ func (h *Harness) init() {
|
||||
if h.SayAgainEvery == 0 {
|
||||
h.SayAgainEvery = 15 * time.Minute
|
||||
}
|
||||
if h.WithdrawAtOnce == 0 {
|
||||
h.WithdrawAtOnce = 1
|
||||
if h.StablePasses == 0 {
|
||||
h.StablePasses = 5
|
||||
}
|
||||
if h.WithdrawFraction == 0 {
|
||||
h.WithdrawFraction = 0.5
|
||||
if h.StableFor == 0 {
|
||||
h.StableFor = StableFor
|
||||
}
|
||||
if h.ReleaseEvery == 0 {
|
||||
h.ReleaseEvery = time.Hour
|
||||
if h.RetireAtOnce == 0 {
|
||||
h.RetireAtOnce = 3
|
||||
}
|
||||
if h.RetireFraction == 0 {
|
||||
h.RetireFraction = 0.5
|
||||
}
|
||||
if h.Log == nil {
|
||||
h.Log = func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }
|
||||
@@ -215,7 +223,7 @@ func (h *Harness) init() {
|
||||
h.failing = map[string]failure{}
|
||||
h.trouble = map[string]*standing{}
|
||||
h.cleared = map[string]bool{}
|
||||
h.braked = map[string]time.Time{}
|
||||
h.r.retired = map[string]retiredEntry{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,7 +257,7 @@ func (h *Harness) warn(why string) {
|
||||
}
|
||||
|
||||
// readContributions answers the consumers asked for, or nil when the file says nothing usable.
|
||||
// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can withdraw.
|
||||
// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can retire anything.
|
||||
func (h *Harness) readContributions() []contribution {
|
||||
raw, err := os.ReadFile(h.Receives)
|
||||
if err != nil {
|
||||
@@ -299,15 +307,20 @@ func orEmpty(m map[string]any) map[string]any {
|
||||
|
||||
// Reconcile is one pass.
|
||||
func (h *Harness) Reconcile(ctx context.Context) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
given := h.readContributions()
|
||||
if given == nil {
|
||||
// Not a result: the passes that must agree before anything is retired start again.
|
||||
h.r.key, h.r.count = "", 0
|
||||
return
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, g := range given {
|
||||
want[g.As] = true
|
||||
}
|
||||
h.seed(ctx, want)
|
||||
verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery
|
||||
if verifying {
|
||||
h.verifiedAt = h.Now()
|
||||
@@ -396,6 +409,7 @@ func (h *Harness) Reconcile(ctx context.Context) {
|
||||
}
|
||||
h.succeeded(g.As)
|
||||
h.applied[g.As] = appliedEntry{hash: hash, derived: p.Derived, node: g.Node}
|
||||
h.reenabled(g.As, g.Node)
|
||||
if reapplying == 0 {
|
||||
delete(h.lost, g.As)
|
||||
} else {
|
||||
@@ -410,97 +424,24 @@ func (h *Harness) Reconcile(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Withdraw every login this process made that the mesh no longer asks for — within the bound.
|
||||
var withdrawing []string
|
||||
for as := range h.applied {
|
||||
if !want[as] {
|
||||
withdrawing = append(withdrawing, as)
|
||||
}
|
||||
}
|
||||
sort.Strings(withdrawing)
|
||||
for as := range h.braked {
|
||||
if want[as] {
|
||||
// Asked for again: the brake held what the mesh still wanted. Its standing was ended by this
|
||||
// pass's success above, as any consumer's is.
|
||||
delete(h.braked, as)
|
||||
}
|
||||
}
|
||||
if h.overTheBound(len(withdrawing), len(h.applied)) {
|
||||
withdrawing = h.brakeWithdrawal(withdrawing)
|
||||
} else if len(h.braked) > 0 {
|
||||
h.say("the withdrawal is within its bound again: %s withdrawn as asked", strings.Join(withdrawing, ", "))
|
||||
h.braked, h.brakeSaid = map[string]time.Time{}, ""
|
||||
}
|
||||
for _, as := range withdrawing {
|
||||
was := h.applied[as]
|
||||
h.say("%s: no longer in %s; withdrawing it from the backend", as, h.Receives)
|
||||
if err := h.Adapter.Remove(ctx, as, was.derived); err != nil {
|
||||
h.say("%s: remove failed, will retry: %v", as, err)
|
||||
continue
|
||||
}
|
||||
delete(h.applied, as)
|
||||
delete(h.lost, as)
|
||||
delete(h.braked, as)
|
||||
}
|
||||
// Retire what the mesh has stably stopped asking for — within the bound, or with a person.
|
||||
h.retireUnasked(ctx, want)
|
||||
h.r.lastWant = want
|
||||
for as := range h.failing {
|
||||
if !want[as] {
|
||||
delete(h.failing, as)
|
||||
}
|
||||
}
|
||||
// A consumer the mesh stopped asking for is no longer failed by anyone: said, so a standing
|
||||
// the controller keeps for it is cleared rather than left naming a consumer that is gone. One the
|
||||
// brake holds is still kept, and its standing stays.
|
||||
// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by
|
||||
// anyone: said, so a standing the controller keeps for it is cleared rather than left naming a
|
||||
// consumer that is gone. One still held is said recovered when it is retired.
|
||||
for as := range h.trouble {
|
||||
if _, held := h.braked[as]; !want[as] && !held {
|
||||
h.recovered(as, "withdrawn")
|
||||
if _, held := h.applied[as]; !want[as] && !held {
|
||||
h.recovered(as, "no longer asked for")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// overTheBound says a pass withdrawing n of the held consumers would withdraw more than it may.
|
||||
func (h *Harness) overTheBound(n, held int) bool {
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
return n > h.WithdrawAtOnce || (held > 1 && float64(n) > h.WithdrawFraction*float64(held))
|
||||
}
|
||||
|
||||
// brakeWithdrawal keeps every consumer a pass over its bound would withdraw, announces each as failing
|
||||
// with the class withdrawal-braked, and answers the one it releases now, if one is due.
|
||||
func (h *Harness) brakeWithdrawal(withdrawing []string) []string {
|
||||
now := h.Now()
|
||||
set := strings.Join(withdrawing, ", ")
|
||||
if set != h.brakeSaid {
|
||||
h.say("WITHDRAWAL BRAKED: this pass would withdraw %d of the %d consumer(s) this provider holds (%s), "+
|
||||
"more than %d at once or %.0f%% of them. Nothing is withdrawn; each is announced as %s (%s), and one "+
|
||||
"is let go every %s while the mesh goes on not asking for them (novox/hq ADR 0227 rule 4)",
|
||||
len(withdrawing), len(h.applied), set, h.WithdrawAtOnce, h.WithdrawFraction*100, EventFailing,
|
||||
ClassWithdrawalBraked, h.ReleaseEvery)
|
||||
h.brakeSaid = set
|
||||
}
|
||||
if len(h.braked) == 0 {
|
||||
// The release clock starts with the brake, not at the last release of an earlier one.
|
||||
h.releasedAt = now
|
||||
}
|
||||
for _, as := range withdrawing {
|
||||
if _, kept := h.braked[as]; !kept {
|
||||
h.braked[as] = now
|
||||
}
|
||||
text := fmt.Sprintf("the mesh no longer asks for it, and the pass that would withdraw it would withdraw %d "+
|
||||
"consumers at once: kept until released (%s)", len(withdrawing), set)
|
||||
h.failed(as, h.applied[as].node, ClassWithdrawalBraked, text)
|
||||
}
|
||||
if now.Sub(h.releasedAt) < h.ReleaseEvery {
|
||||
return nil
|
||||
}
|
||||
h.releasedAt = now
|
||||
release := withdrawing[0]
|
||||
h.say("%s: released by the withdrawal brake after %s; %d more kept", release,
|
||||
now.Sub(h.braked[release]).Round(time.Second), len(withdrawing)-1)
|
||||
h.recovered(release, "withdrawn")
|
||||
return []string{release}
|
||||
}
|
||||
|
||||
// failed counts one more failure in a consumer's unbroken run, and announces the run once it has
|
||||
// lasted FailingAfter — then again every SayAgainEvery while it lasts.
|
||||
func (h *Harness) failed(as, node, class, text string) {
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
package main
|
||||
|
||||
// The provisioner loop is carried, identical, by every Go provider until the Go SDK has it
|
||||
// (harness.go). Two copies drift the moment one is fixed and the other is not — and the one left
|
||||
// behind is the provider that fails a consumer without saying so (novox/hq ADR 0224). This holds
|
||||
// them to one text. Skipped where postgres is not beside this module, as in a build of this one alone.
|
||||
// (harness.go, retirement.go, and retirement_test.go which tests it). Two copies drift the moment one
|
||||
// is fixed and the other is not — and the one left behind is the provider that fails a consumer
|
||||
// without saying so (novox/hq ADR 0224), or retires one it should not (ADR 0230). This holds them to
|
||||
// one text. Skipped where postgres is not beside this module, as in a build of this one alone.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -14,18 +15,20 @@ import (
|
||||
)
|
||||
|
||||
func TestTheHarnessIsTheSameAsPostgress(t *testing.T) {
|
||||
theirs, err := os.ReadFile("../../../postgres/cmd/postgres-provider/harness.go")
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
t.Skip("postgres is not beside this module")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours, err := os.ReadFile("harness.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(ours, theirs) {
|
||||
t.Fatal("harness.go differs from postgres/cmd/postgres-provider/harness.go: change both, identically")
|
||||
for _, file := range []string{"harness.go", "retirement.go", "retirement_test.go"} {
|
||||
theirs, err := os.ReadFile("../../../postgres/cmd/postgres-provider/" + file)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
t.Skip("postgres is not beside this module")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(ours, theirs) {
|
||||
t.Fatalf("%s differs from postgres/cmd/postgres-provider/%s: change both, identically", file, file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
// The shared harness, as postgres tests it (harness.go is the same file in both modules).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -12,34 +11,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type recorder struct {
|
||||
created []Provision
|
||||
removed []string
|
||||
held bool
|
||||
failing error
|
||||
holdsErr error
|
||||
}
|
||||
|
||||
func (r *recorder) Create(_ context.Context, p Provision) error {
|
||||
if r.failing != nil {
|
||||
return r.failing
|
||||
}
|
||||
r.created = append(r.created, p)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Remove(_ context.Context, as string, _ map[string]any) error {
|
||||
r.removed = append(r.removed, as)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Holds(context.Context, Provision) (bool, error) {
|
||||
if r.holdsErr != nil {
|
||||
return false, r.holdsErr
|
||||
}
|
||||
return r.held, nil
|
||||
}
|
||||
|
||||
type world struct {
|
||||
t *testing.T
|
||||
dir string
|
||||
@@ -90,18 +61,18 @@ func TestAConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConsumerNoLongerAskedForIsWithdrawn(t *testing.T) {
|
||||
func TestAConsumerNoLongerAskedForIsRetiredOnceStable(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.settle() // five passes and ten minutes
|
||||
if strings.Join(w.a.removed, ",") != "b" {
|
||||
t.Fatal(w.a.removed)
|
||||
}
|
||||
// Only a file that says nobody asks withdraws everybody.
|
||||
// Only a file that says nobody asks retires the last one.
|
||||
w.give()
|
||||
w.h.Reconcile(ctx)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "b,a" {
|
||||
t.Fatal(w.a.removed)
|
||||
}
|
||||
@@ -125,7 +96,7 @@ func TestNothingReadIsNotNobodyAsking(t *testing.T) {
|
||||
}
|
||||
w.h.Reconcile(ctx)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("withdrew %v on a file it could not use", w.a.removed)
|
||||
t.Fatalf("retired %v on a file it could not use", w.a.removed)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
// Retirement against a real Keycloak (novox/hq ADR 0230). Skipped unless MESH_KEYCLOAK_LIVE_URL reaches
|
||||
// a throwaway Keycloak whose master admin's password is MESH_KEYCLOAK_LIVE_PASSWORD, e.g.:
|
||||
//
|
||||
// docker run -d --rm --name kc-retire -p 127.0.0.1:18081:8080 -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
|
||||
// -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin quay.io/keycloak/keycloak:26.0.8 start-dev
|
||||
// MESH_KEYCLOAK_LIVE_URL=http://127.0.0.1:18081 MESH_KEYCLOAK_LIVE_PASSWORD=admin go test -run LiveRetire ./...
|
||||
//
|
||||
// It proves what the fake cannot: a retired client is refused by Keycloak itself at the authorization
|
||||
// endpoint, keeps its secret and redirects, is served again once enabled, and is deleted only once
|
||||
// retired.
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLiveRetirement(t *testing.T) {
|
||||
base, pw := os.Getenv("MESH_KEYCLOAK_LIVE_URL"), os.Getenv("MESH_KEYCLOAK_LIVE_PASSWORD")
|
||||
if base == "" || pw == "" {
|
||||
t.Skip("no MESH_KEYCLOAK_LIVE_URL / MESH_KEYCLOAK_LIVE_PASSWORD")
|
||||
}
|
||||
kc := NewClient(base, "admin", func() (string, error) { return pw, nil }, "master")
|
||||
o := OidcClients{KC: kc, Realm: "master"}
|
||||
p := grafana("live-secret", nil)
|
||||
p.As = "mesh_live_retire"
|
||||
t.Cleanup(func() {
|
||||
if found, _ := kc.FindClient(ctx, "master", p.As); found != nil {
|
||||
id, _ := found["id"].(string)
|
||||
kc.DeleteClientByID(ctx, "master", id)
|
||||
}
|
||||
})
|
||||
if _, err := o.Ensure(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The authorization endpoint is where a consumer's users arrive: 200 with a login page while the
|
||||
// client is enabled, refused while it is not.
|
||||
authorize := func() int {
|
||||
q := url.Values{"client_id": {p.As}, "response_type": {"code"}, "scope": {"openid"},
|
||||
"redirect_uri": {"https://grafana.example.org/login/generic_oauth"}}
|
||||
resp, err := (&http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}).
|
||||
Get(base + "/realms/master/protocol/openid-connect/auth?" + q.Encode())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
if code := authorize(); code != 200 {
|
||||
t.Fatalf("an enabled client is refused: %d", code)
|
||||
}
|
||||
mustHold(t, o, p, true)
|
||||
|
||||
at := time.Now().UTC().Truncate(time.Second)
|
||||
if done, err := o.Retire(ctx, p.As, "the mesh stopped asking for it", at); done != "retired" || err != nil {
|
||||
t.Fatal(done, err)
|
||||
}
|
||||
if code := authorize(); code == 200 {
|
||||
t.Fatal("a retired client is still served")
|
||||
}
|
||||
// Retired, so a person may delete it.
|
||||
if err := o.Delete(ctx, p.As); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Made again, retired, listed, and enabled again as it was.
|
||||
if _, err := o.Ensure(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o.Retire(ctx, p.As, "again", at)
|
||||
inv, err := o.Inventory(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range inv.Retired {
|
||||
found = found || (r.Consumer == p.As && r.RetiredAt.Equal(at) && r.Why == "again")
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("not listed retired: %+v", inv)
|
||||
}
|
||||
secret, err := kc.ClientSecretByID(ctx, "master", clientID(t, kc, p.As))
|
||||
if err != nil || secret != "live-secret" {
|
||||
t.Fatal("the secret was not kept:", secret, err)
|
||||
}
|
||||
if _, err := o.Ensure(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code := authorize(); code != 200 {
|
||||
t.Fatalf("re-enabled and still refused: %d", code)
|
||||
}
|
||||
mustHold(t, o, p, true)
|
||||
if err := o.Delete(ctx, p.As); err == nil {
|
||||
t.Fatal("deleted an enabled client")
|
||||
}
|
||||
}
|
||||
|
||||
func clientID(t *testing.T, kc *Client, clientId string) string {
|
||||
found, err := kc.FindClient(ctx, "master", clientId)
|
||||
if err != nil || found == nil {
|
||||
t.Fatal(found, err)
|
||||
}
|
||||
id, _ := found["id"].(string)
|
||||
return id
|
||||
}
|
||||
@@ -42,6 +42,7 @@ func main() {
|
||||
kc.onRejected = guard.Nudge
|
||||
go guard.Run(context.Background())
|
||||
|
||||
var h *Harness
|
||||
if receives := os.Getenv("MESH_RECEIVES"); receives == "" {
|
||||
say("MESH_RECEIVES is not set — the provisioner cannot run without it")
|
||||
} else if issuer, err := Issuer(os.Getenv); err != nil {
|
||||
@@ -49,7 +50,7 @@ func main() {
|
||||
} else if realm, err := RealmOf(issuer); err != nil {
|
||||
say("%v — the provisioner cannot run without it", err)
|
||||
} else {
|
||||
h := &Harness{
|
||||
h = &Harness{
|
||||
Resource: "oidc-client",
|
||||
Receives: receives,
|
||||
Adapter: provisioner{clients: OidcClients{KC: kc, Realm: realm}, guard: guard, announce: announce, log: logStderr},
|
||||
@@ -61,7 +62,8 @@ func main() {
|
||||
}
|
||||
go h.Run(context.Background())
|
||||
}
|
||||
if err := stdio.Serve("", Tools(kc, guard)); err != nil {
|
||||
// The retirement tools beside keycloak's own (novox/hq ADR 0230).
|
||||
if err := stdio.Serve("", append(Tools(kc, guard), RetirementTools(h)...)); err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,12 @@ package main
|
||||
// **Only what the mesh made is touched.** A client this module creates carries the attribute
|
||||
// `mesh.provisioned=true`. A client with the same id that lacks the mark is somebody else's: it is
|
||||
// refused, never adopted, never updated, never deleted.
|
||||
//
|
||||
// **Retired is the client disabled and marked** (novox/hq ADR 0230). `enabled: false` — Keycloak then
|
||||
// refuses the client's authorization and token requests, so nobody signs in through it — and the
|
||||
// attributes `mesh.retired` (when) and `mesh.retired-why` (why). Its secret, redirects, mappers and
|
||||
// every other setting are kept, so asked for again it is enabled as it was: Ensure sets `enabled` and
|
||||
// removes the two attributes. Deleted — only through `cleanup delete` — the client is removed.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -25,11 +31,18 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Mark is the attribute marking a client as the mesh's own work.
|
||||
const Mark = "mesh.provisioned"
|
||||
|
||||
// MarkRetired and MarkRetiredWhy mark a retired client: when and why (novox/hq ADR 0230).
|
||||
const (
|
||||
MarkRetired = "mesh.retired"
|
||||
MarkRetiredWhy = "mesh.retired-why"
|
||||
)
|
||||
|
||||
// RolesMapper is the mapper every mesh client carries: realm roles as a flat `roles` claim in the id
|
||||
// token, the access token and userinfo — what a consumer maps its own roles from.
|
||||
func RolesMapper() Rep {
|
||||
@@ -214,6 +227,9 @@ func (o OidcClients) Ensure(ctx context.Context, p Provision) (string, error) {
|
||||
attrs[k] = v
|
||||
}
|
||||
attrs[Mark] = "true"
|
||||
// Asked for again: no longer marked to delete (novox/hq ADR 0230).
|
||||
delete(attrs, MarkRetired)
|
||||
delete(attrs, MarkRetiredWhy)
|
||||
merged["attributes"] = attrs
|
||||
id, _ := found["id"].(string)
|
||||
if err := o.KC.UpdateClient(ctx, o.Realm, id, merged); err != nil {
|
||||
@@ -290,8 +306,9 @@ func (o OidcClients) Holds(ctx context.Context, p Provision) (bool, error) {
|
||||
return secret == p.Password, nil
|
||||
}
|
||||
|
||||
// Remove withdraws a consumer's client — only one the mesh made. Answers what happened.
|
||||
func (o OidcClients) Remove(ctx context.Context, as string) (string, error) {
|
||||
// Retire disables a consumer's client — only one the mesh made — and marks it with when and why.
|
||||
// Answers what happened: "retired", "absent" or "not ours".
|
||||
func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) (string, error) {
|
||||
found, err := o.KC.FindClient(ctx, o.Realm, as)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -302,6 +319,63 @@ func (o OidcClients) Remove(ctx context.Context, as string) (string, error) {
|
||||
if !marked(found) {
|
||||
return "not ours", nil
|
||||
}
|
||||
merged := Rep{}
|
||||
for k, v := range found {
|
||||
merged[k] = v
|
||||
}
|
||||
attrs := map[string]any{}
|
||||
for k, v := range attributes(found) {
|
||||
attrs[k] = v
|
||||
}
|
||||
attrs[MarkRetired] = at.UTC().Format(time.RFC3339)
|
||||
attrs[MarkRetiredWhy] = why
|
||||
merged["attributes"] = attrs
|
||||
merged["enabled"] = false
|
||||
id, _ := found["id"].(string)
|
||||
return "removed", o.KC.DeleteClientByID(ctx, o.Realm, id)
|
||||
return "retired", o.KC.UpdateClient(ctx, o.Realm, id, merged)
|
||||
}
|
||||
|
||||
// Inventory is every client in the realm the mesh made: enabled ones active, disabled ones retired —
|
||||
// with when and why from the mark, or none for one disabled before the mark existed.
|
||||
func (o OidcClients) Inventory(ctx context.Context) (Inventory, error) {
|
||||
inv := Inventory{Active: []string{}, Retired: []Retired{}}
|
||||
clients, err := o.KC.ListClients(ctx, o.Realm)
|
||||
if err != nil {
|
||||
return inv, err
|
||||
}
|
||||
for _, c := range clients {
|
||||
if !marked(c) {
|
||||
continue
|
||||
}
|
||||
id, _ := c["clientId"].(string)
|
||||
a := attributes(c)
|
||||
when, _ := a[MarkRetired].(string)
|
||||
if c["enabled"] != false && when == "" {
|
||||
inv.Active = append(inv.Active, id)
|
||||
continue
|
||||
}
|
||||
at, _ := time.Parse(time.RFC3339, when)
|
||||
why, _ := a[MarkRetiredWhy].(string)
|
||||
inv.Retired = append(inv.Retired, Retired{Consumer: id, RetiredAt: at, Why: why, SizeBytes: -1, Kind: KindConsumer})
|
||||
}
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
// Delete removes a retired client — only one the mesh made, and only while it is disabled.
|
||||
func (o OidcClients) Delete(ctx context.Context, as string) error {
|
||||
found, err := o.KC.FindClient(ctx, o.Realm, as)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found == nil {
|
||||
return nil
|
||||
}
|
||||
if !marked(found) {
|
||||
return fmt.Errorf("realm %s's client %s was not made by the mesh — left alone", o.Realm, as)
|
||||
}
|
||||
if found["enabled"] != false {
|
||||
return fmt.Errorf("client %s is enabled — it is active, not retired, and is not deleted", as)
|
||||
}
|
||||
id, _ := found["id"].(string)
|
||||
return o.KC.DeleteClientByID(ctx, o.Realm, id)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func oidcWorld(t *testing.T) (*fakeKeycloak, OidcClients) {
|
||||
@@ -173,22 +174,87 @@ func TestAClientTheMeshDidNotMakeIsRefusedAndLeftAlone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
mustHold(t, o, grafana("s", nil), false)
|
||||
if done, _ := o.Remove(ctx, "mesh_home_grafana"); done != "not ours" || f.clients["theirs"] == nil {
|
||||
if done, _ := o.Retire(ctx, "mesh_home_grafana", "x", time.Now()); done != "not ours" || f.clients["theirs"]["enabled"] == false {
|
||||
t.Fatal(done)
|
||||
}
|
||||
if err := o.Delete(ctx, "mesh_home_grafana"); err == nil || f.clients["theirs"] == nil {
|
||||
t.Fatal("deleted a client the mesh did not make")
|
||||
}
|
||||
if inv, _ := o.Inventory(ctx); len(inv.Active)+len(inv.Retired) != 0 {
|
||||
t.Fatalf("listed a client the mesh did not make: %+v", inv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWithdrawnClientIsRemovedAndAnAbsentOneIsNoError(t *testing.T) {
|
||||
// Retired is disabled and marked, everything else kept; asked for again it is enabled as it was; only
|
||||
// a deletion removes it, and never while enabled (novox/hq ADR 0230).
|
||||
func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) {
|
||||
f, o := oidcWorld(t)
|
||||
o.Ensure(ctx, grafana("s", nil))
|
||||
if done, err := o.Remove(ctx, "mesh_home_grafana"); done != "removed" || err != nil || len(f.clients) != 0 {
|
||||
p := grafana("s", nil)
|
||||
if _, err := o.Ensure(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var id string
|
||||
for k := range f.clients {
|
||||
id = k
|
||||
}
|
||||
f.clients[id]["description2"] = "an operator's own field"
|
||||
at := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
if done, err := o.Retire(ctx, "mesh_home_grafana", "the mesh stopped asking for it", at); done != "retired" || err != nil {
|
||||
t.Fatal(done, err)
|
||||
}
|
||||
if done, err := o.Remove(ctx, "mesh_home_grafana"); done != "absent" || err != nil {
|
||||
c := f.clients[id]
|
||||
if c["enabled"] != false || c["secret"] != "s" || attributes(c)[MarkRetired] != "2026-10-06T12:00:00Z" ||
|
||||
attributes(c)[MarkRetiredWhy] != "the mesh stopped asking for it" || c["description2"] == nil {
|
||||
t.Fatalf("%v", c)
|
||||
}
|
||||
mustHold(t, o, p, false)
|
||||
inv, err := o.Inventory(ctx)
|
||||
if err != nil || len(inv.Active) != 0 || len(inv.Retired) != 1 || !inv.Retired[0].RetiredAt.Equal(at) ||
|
||||
inv.Retired[0].Consumer != "mesh_home_grafana" {
|
||||
t.Fatalf("%+v %v", inv, err)
|
||||
}
|
||||
// Asked for again: enabled, unmarked, the same client.
|
||||
if _, err := o.Ensure(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c = f.clients[id]
|
||||
if c["enabled"] != true || attributes(c)[MarkRetired] != nil || attributes(c)[MarkRetiredWhy] != nil || c["description2"] == nil {
|
||||
t.Fatalf("%v", c)
|
||||
}
|
||||
mustHold(t, o, p, true)
|
||||
if inv, _ := o.Inventory(ctx); len(inv.Active) != 1 || len(inv.Retired) != 0 {
|
||||
t.Fatalf("%+v", inv)
|
||||
}
|
||||
// Never deleted while enabled; deleted once retired; absent is no error.
|
||||
if err := o.Delete(ctx, "mesh_home_grafana"); err == nil || len(f.clients) != 1 {
|
||||
t.Fatal("deleted an enabled client")
|
||||
}
|
||||
o.Retire(ctx, "mesh_home_grafana", "again", at)
|
||||
if err := o.Delete(ctx, "mesh_home_grafana"); err != nil || len(f.clients) != 0 {
|
||||
t.Fatal(err, f.clients)
|
||||
}
|
||||
if err := o.Delete(ctx, "mesh_home_grafana"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done, err := o.Retire(ctx, "mesh_home_grafana", "x", at); done != "absent" || err != nil {
|
||||
t.Fatal(done, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A client the mesh made and found disabled without the mark — before ADR 0230 — is listed retired
|
||||
// with no date, which the loop adopts.
|
||||
func TestAClientFoundDisabledIsListedRetiredWithoutADate(t *testing.T) {
|
||||
f, o := oidcWorld(t)
|
||||
o.Ensure(ctx, grafana("s", nil))
|
||||
for _, c := range f.clients {
|
||||
c["enabled"] = false
|
||||
}
|
||||
inv, err := o.Inventory(ctx)
|
||||
if err != nil || len(inv.Retired) != 1 || !inv.Retired[0].RetiredAt.IsZero() {
|
||||
t.Fatalf("%+v %v", inv, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAContributionWithNoCallbackMakesNoClient(t *testing.T) {
|
||||
f, o := oidcWorld(t)
|
||||
p := grafana("s", nil)
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issuer is the issuer this assignment serves, from the settings-merged config the mesh delivers.
|
||||
@@ -60,23 +61,46 @@ func (a provisioner) Create(ctx context.Context, p Provision) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a provisioner) Remove(ctx context.Context, as string, _ map[string]any) error {
|
||||
// Retire disables the consumer's client and marks it, never deletes it (novox/hq ADR 0230).
|
||||
func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, why string, at time.Time) error {
|
||||
if err := a.refused(); err != nil {
|
||||
return err
|
||||
}
|
||||
done, err := a.clients.Remove(ctx, as)
|
||||
done, err := a.clients.Retire(ctx, as, why, at)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch done {
|
||||
case "not ours":
|
||||
a.log("[provisioner:oidc-client] %s: a client of that id exists that the mesh did not make — left alone", as)
|
||||
case "removed":
|
||||
a.log("[provisioner:oidc-client] removed client %s from realm %s", as, a.clients.Realm)
|
||||
case "retired":
|
||||
a.log("[provisioner:oidc-client] retired client %s in realm %s: disabled, kept, marked to delete", as, a.clients.Realm)
|
||||
a.announce("client.retired", map[string]any{"realm": a.clients.Realm, "clientId": as})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Inventory is every client in the realm the mesh made, active and retired.
|
||||
func (a provisioner) Inventory(ctx context.Context) (Inventory, error) {
|
||||
if err := a.refused(); err != nil {
|
||||
return Inventory{}, err
|
||||
}
|
||||
return a.clients.Inventory(ctx)
|
||||
}
|
||||
|
||||
// Delete removes a retired client, a person's act through `cleanup delete`. Nothing is freed that
|
||||
// Keycloak counts in bytes.
|
||||
func (a provisioner) Delete(ctx context.Context, r Retired) (int64, error) {
|
||||
if err := a.refused(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := a.clients.Delete(ctx, r.Consumer); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
a.log("[provisioner:oidc-client] deleted retired client %s from realm %s", r.Consumer, a.clients.Realm)
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
// Holds is asked every minute by the harness: whether Keycloak still holds this consumer's client
|
||||
// exactly as the mesh gave it, so one deleted or edited behind the mesh's back is made again
|
||||
// (novox/hq issue 120).
|
||||
|
||||
@@ -0,0 +1,603 @@
|
||||
package main
|
||||
|
||||
// What becomes of a consumer the mesh no longer asks for (novox/hq ADR 0230 — the operator's model,
|
||||
// decided 2026-10-06; it replaces ADR 0229's withdrawal brake, which let one consumer go every hour).
|
||||
//
|
||||
// A consumer — a login, a client, a key, and the data behind it — is in one of three states:
|
||||
//
|
||||
// 1. ACTIVE.
|
||||
// 2. RETIRED. The mesh stopped asking for it: its access is disabled, reversibly, and its login and
|
||||
// data are marked "to delete" with when and why. Nothing is deleted. Asked for again, the ordinary
|
||||
// create re-enables it at once, as it was.
|
||||
// 3. DELETED, only through `cleanup delete`, a person's act, which this provider executes because it
|
||||
// owns its backend.
|
||||
//
|
||||
// **Stable removals.** A consumer is retired only once the same set of consumers has gone unasked BOTH
|
||||
// in StablePasses (5) consecutive passes that read the file AND for at least StableFor (10 minutes)
|
||||
// since the first pass that saw it. Five passes alone are twenty-five seconds — shorter than a
|
||||
// controller restart, a store reconnecting or a file half written. A pass that could not read the file
|
||||
// is not a result and starts both again; so does a different set. Additions and changes are never
|
||||
// delayed.
|
||||
//
|
||||
// **Too many is a person.** A stable set of more than RetireAtOnce (3), or — where more than one is
|
||||
// held — of more than RetireFraction (half) of those held, retires nothing: it WAITS, said in the
|
||||
// journal and announced `provisioner.retirement` `waiting` (again every SayAgainEvery), which the
|
||||
// controller raises as an urgent condition, until `retire approve <node> <module>` or `retire reject`.
|
||||
// An unassignment the controller made itself is no exception: many changes at once means a person is
|
||||
// at work, and a person confirms once. On 2026-10-04 one misread file withdrew seven consumers at once
|
||||
// (issue 241); under this rule that is a question, not an act.
|
||||
//
|
||||
// **The backend remembers, not this process.** What is retired, since when and why is read from the
|
||||
// backend's own mark (Adapter.Inventory), so a restart forgets nothing; a consumer the backend holds
|
||||
// active that the mesh no longer asks for is retired by the same rules after a restart as before one.
|
||||
// A consumer found disabled without the mark — withdrawn before this record — is ADOPTED as retired:
|
||||
// marked, said, announced `adopted`, and its clock starts then.
|
||||
//
|
||||
// **A rejection lives as long as this process.** Rejected, the set is kept active and not asked about
|
||||
// again while it stays the same set; a restart asks again — loudly, never silently.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// EventRetirement is the one event a provider says about retirement, its `change` saying what
|
||||
// happened. The controller derives the permission to emit it for every module that receives
|
||||
// contributions, as it does the standing events (novox/hq ADR 0224, 0230).
|
||||
const EventRetirement = "provisioner.retirement"
|
||||
|
||||
// The changes EventRetirement carries.
|
||||
const (
|
||||
ChangeWaiting = "waiting" // a stable set over the bound waits for a person
|
||||
ChangeSettled = "settled" // a waiting or rejected set ended without being retired
|
||||
ChangeApproved = "approved" // a person approved the waiting (or rejected) set
|
||||
ChangeRejected = "rejected" // a person kept the waiting set active
|
||||
ChangeRetired = "retired" // consumers disabled and marked to delete
|
||||
ChangeReenabled = "reenabled" // a retired consumer asked for again, enabled as it was
|
||||
ChangeDeleted = "deleted" // a retired consumer deleted, by a person
|
||||
ChangeAdopted = "adopted" // found disabled without the mark, now retired on record
|
||||
)
|
||||
|
||||
// StableFor is the least time the same unasked set must hold before it is retired (novox/hq ADR 0230):
|
||||
// longer than a controller restart, a store reconnecting or a file half written.
|
||||
const StableFor = 10 * time.Minute
|
||||
|
||||
// KindConsumer is a retired consumer. A backend may list other things set aside for deletion under a
|
||||
// word of its own (postgres: a database renamed aside).
|
||||
const KindConsumer = "consumer"
|
||||
|
||||
// Retired is one thing a provider holds retired, as its backend's mark says.
|
||||
type Retired struct {
|
||||
Consumer string
|
||||
// Node is the consumer's machine, where the mark records it.
|
||||
Node string
|
||||
// RetiredAt is when it was retired; zero for one found disabled without the mesh's mark.
|
||||
RetiredAt time.Time
|
||||
Why string
|
||||
// SizeBytes is what deleting it would free; -1 when the backend cannot say.
|
||||
SizeBytes int64
|
||||
Kind string
|
||||
}
|
||||
|
||||
// Inventory is what a backend holds that the mesh made.
|
||||
type Inventory struct {
|
||||
Active []string
|
||||
Retired []Retired
|
||||
}
|
||||
|
||||
// retirement is the loop's memory of the sets it is counting, waiting on and was told to keep.
|
||||
type retirement struct {
|
||||
key string // the unasked set, joined
|
||||
count int // consecutive passes that saw it
|
||||
firstSeen time.Time
|
||||
waiting *waitingSet
|
||||
rejected *rejectedSet
|
||||
retired map[string]retiredEntry
|
||||
lastWant map[string]bool
|
||||
seeded bool
|
||||
seedSaid string
|
||||
}
|
||||
|
||||
type waitingSet struct {
|
||||
set []string
|
||||
since time.Time
|
||||
saidAt time.Time
|
||||
held int
|
||||
}
|
||||
|
||||
type rejectedSet struct {
|
||||
set []string
|
||||
by, why string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
type retiredEntry struct {
|
||||
node string
|
||||
at time.Time
|
||||
why string
|
||||
}
|
||||
|
||||
// seed reads what the backend holds, once per process: an active consumer the mesh no longer asks
|
||||
// for is held, so it is retired by the same rules as one this process made; one found disabled
|
||||
// without the mark is adopted as retired.
|
||||
func (h *Harness) seed(ctx context.Context, want map[string]bool) {
|
||||
if h.r.seeded {
|
||||
return
|
||||
}
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
if said := err.Error(); said != h.r.seedSaid {
|
||||
h.say("cannot list what the backend holds (%v); a consumer the mesh stopped asking for while this "+
|
||||
"provider was down is not retired until it can", err)
|
||||
h.r.seedSaid = said
|
||||
}
|
||||
return
|
||||
}
|
||||
h.r.seeded = true
|
||||
for _, as := range inv.Active {
|
||||
if _, held := h.applied[as]; !held && !want[as] {
|
||||
// No hash: asked for again, it is applied again, which is harmless and marks it.
|
||||
h.applied[as] = appliedEntry{}
|
||||
h.say("%s: held by the backend and no longer asked for; retired once that holds for %d passes "+
|
||||
"and %s (novox/hq ADR 0230)", as, h.StablePasses, h.StableFor)
|
||||
}
|
||||
}
|
||||
now := h.Now()
|
||||
for _, r := range inv.Retired {
|
||||
if r.Kind != "" && r.Kind != KindConsumer {
|
||||
continue
|
||||
}
|
||||
if !r.RetiredAt.IsZero() {
|
||||
h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: r.RetiredAt, why: r.Why}
|
||||
continue
|
||||
}
|
||||
if want[r.Consumer] {
|
||||
continue // asked for: this pass's create enables it
|
||||
}
|
||||
why := "found disabled without the mesh's mark — withdrawn before retirement was recorded " +
|
||||
"(novox/hq ADR 0230); retired as found"
|
||||
if err := h.Adapter.Retire(ctx, r.Consumer, nil, why, now); err != nil {
|
||||
h.say("%s: found disabled and could not be marked retired, will try at the next start: %v", r.Consumer, err)
|
||||
continue
|
||||
}
|
||||
h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: now, why: why}
|
||||
h.say("%s: ADOPTED as retired — %s", r.Consumer, why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeAdopted, []map[string]any{
|
||||
{"consumer": r.Consumer, "node": r.Node, "retired_at": stamp(now), "why": why, "size_bytes": r.SizeBytes},
|
||||
}, map[string]any{"why": why}))
|
||||
}
|
||||
}
|
||||
|
||||
// retireUnasked counts the passes that saw the same set no longer asked for and, once it is stable,
|
||||
// retires it — or, past the bound, waits for a person.
|
||||
func (h *Harness) retireUnasked(ctx context.Context, want map[string]bool) {
|
||||
var unasked []string
|
||||
for as := range h.applied {
|
||||
if !want[as] {
|
||||
unasked = append(unasked, as)
|
||||
}
|
||||
}
|
||||
sort.Strings(unasked)
|
||||
key := strings.Join(unasked, "\x00")
|
||||
now := h.Now()
|
||||
switch {
|
||||
case len(unasked) == 0:
|
||||
h.settle("every consumer held is asked for again")
|
||||
h.r.key, h.r.count = "", 0
|
||||
return
|
||||
case key != h.r.key:
|
||||
if h.r.waiting != nil || h.r.rejected != nil {
|
||||
h.settle("the set the mesh no longer asks for changed")
|
||||
}
|
||||
h.r.key, h.r.count, h.r.firstSeen = key, 1, now
|
||||
h.say("no longer asked for: %s; retired once the same set holds for %d passes and %s",
|
||||
strings.Join(unasked, ", "), h.StablePasses, h.StableFor)
|
||||
default:
|
||||
h.r.count++
|
||||
}
|
||||
if h.r.rejected != nil || h.r.count < h.StablePasses || now.Sub(h.r.firstSeen) < h.StableFor {
|
||||
return
|
||||
}
|
||||
if h.overTheBound(len(unasked), len(h.applied)) {
|
||||
h.wait(unasked)
|
||||
return
|
||||
}
|
||||
why := fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s, from %s",
|
||||
h.r.count, now.Sub(h.r.firstSeen).Round(time.Second), stamp(h.r.firstSeen))
|
||||
h.retire(ctx, unasked, why, nil)
|
||||
}
|
||||
|
||||
// overTheBound says retiring n of the held consumers at once needs a person.
|
||||
func (h *Harness) overTheBound(n, held int) bool {
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
return n > h.RetireAtOnce || (held > 1 && float64(n) > h.RetireFraction*float64(held))
|
||||
}
|
||||
|
||||
func (h *Harness) bound(held int) string {
|
||||
return fmt.Sprintf("more than %d at once, or more than %.0f%% of the %d held", h.RetireAtOnce,
|
||||
h.RetireFraction*100, held)
|
||||
}
|
||||
|
||||
// wait holds a stable set over the bound for a person, said once and announced again every
|
||||
// SayAgainEvery so a controller that missed the first hears the next.
|
||||
func (h *Harness) wait(set []string) {
|
||||
now := h.Now()
|
||||
w := h.r.waiting
|
||||
if w == nil {
|
||||
w = &waitingSet{set: set, since: now, held: len(h.applied)}
|
||||
h.r.waiting = w
|
||||
h.say("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d consumer(s) this provider "+
|
||||
"holds (%s), %s. Nothing is retired; each stays active until `retire approve %s <module>` or "+
|
||||
"`retire reject` (novox/hq ADR 0230)", len(set), w.held, strings.Join(set, ", "), h.bound(w.held), h.Node)
|
||||
} else if now.Sub(w.saidAt) < h.SayAgainEvery {
|
||||
return
|
||||
}
|
||||
w.saidAt = now
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeWaiting, h.named(set), map[string]any{
|
||||
"held": w.held, "bound": h.bound(w.held), "since": stamp(w.since),
|
||||
}))
|
||||
}
|
||||
|
||||
// settle ends a waiting or rejected set that the mesh's own answer made moot.
|
||||
func (h *Harness) settle(why string) {
|
||||
var set []string
|
||||
switch {
|
||||
case h.r.waiting != nil:
|
||||
set = h.r.waiting.set
|
||||
case h.r.rejected != nil:
|
||||
set = h.r.rejected.set
|
||||
default:
|
||||
return
|
||||
}
|
||||
h.r.waiting, h.r.rejected = nil, nil
|
||||
h.say("retirement of %s settled without retiring: %s", strings.Join(set, ", "), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeSettled, h.named(set), map[string]any{"why": why}))
|
||||
}
|
||||
|
||||
// retire disables and marks each consumer of set; one that fails stays held and is tried again once
|
||||
// its set is stable again.
|
||||
func (h *Harness) retire(ctx context.Context, set []string, why string, extra map[string]any) []string {
|
||||
now := h.Now()
|
||||
held := len(h.applied)
|
||||
var done []string
|
||||
var said []map[string]any
|
||||
for _, as := range set {
|
||||
was, ok := h.applied[as]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := h.Adapter.Retire(ctx, as, was.derived, why, now); err != nil {
|
||||
h.say("%s: retiring failed, will try again: %v", as, err)
|
||||
continue
|
||||
}
|
||||
delete(h.applied, as)
|
||||
delete(h.lost, as)
|
||||
delete(h.failing, as)
|
||||
h.r.retired[as] = retiredEntry{node: was.node, at: now, why: why}
|
||||
h.recovered(as, "retired")
|
||||
h.say("%s: RETIRED — its access disabled and its data kept, marked to delete (%s). Asked for again "+
|
||||
"it is re-enabled as it was; it is deleted only by `cleanup delete` (novox/hq ADR 0230)", as, why)
|
||||
done = append(done, as)
|
||||
said = append(said, map[string]any{"consumer": as, "node": was.node, "retired_at": stamp(now), "why": why})
|
||||
}
|
||||
h.r.key, h.r.count, h.r.waiting, h.r.rejected = "", 0, nil, nil
|
||||
if len(done) > 0 {
|
||||
body := map[string]any{"held": held, "why": why}
|
||||
for k, v := range extra {
|
||||
body[k] = v
|
||||
}
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeRetired, said, body))
|
||||
}
|
||||
return done
|
||||
}
|
||||
|
||||
// reenabled says a retired consumer was asked for again and its create enabled it.
|
||||
func (h *Harness) reenabled(as, node string) {
|
||||
e, was := h.r.retired[as]
|
||||
if !was {
|
||||
return
|
||||
}
|
||||
delete(h.r.retired, as)
|
||||
h.say("%s: asked for again — re-enabled as it was, its mark to delete cleared (retired %s: %s)", as,
|
||||
stamp(e.at), e.why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeReenabled, []map[string]any{
|
||||
{"consumer": as, "node": node, "retired_at": stamp(e.at), "why": e.why},
|
||||
}, nil))
|
||||
}
|
||||
|
||||
// Approve retires exactly the set waiting (or the set rejected) — a person's confirmation.
|
||||
func (h *Harness) Approve(ctx context.Context, consumers []string, why, by, via string) ([]string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("approve: say why")
|
||||
}
|
||||
set := sorted(consumers)
|
||||
var held []string
|
||||
switch {
|
||||
case h.r.waiting != nil && same(set, h.r.waiting.set):
|
||||
held = h.r.waiting.set
|
||||
case h.r.rejected != nil && same(set, h.r.rejected.set):
|
||||
held = h.r.rejected.set
|
||||
default:
|
||||
return nil, fmt.Errorf("approve: %s is not the set waiting here — %s", orNone(set), h.waitingWords())
|
||||
}
|
||||
h.say("retirement of %s APPROVED by %s: %s", strings.Join(held, ", "), orSomebody(by), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeApproved, h.named(held),
|
||||
map[string]any{"why": why, "by": by, "via": via}))
|
||||
reason := fmt.Sprintf("the mesh stopped asking for it; approved by %s: %s", orSomebody(by), why)
|
||||
return h.retire(ctx, held, reason, map[string]any{"by": by, "via": via}), nil
|
||||
}
|
||||
|
||||
// Reject keeps the waiting set active; it is not asked about again while it stays the same set.
|
||||
func (h *Harness) Reject(consumers []string, why, by, via string) ([]string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("reject: say why")
|
||||
}
|
||||
set := sorted(consumers)
|
||||
if h.r.waiting == nil || !same(set, h.r.waiting.set) {
|
||||
return nil, fmt.Errorf("reject: %s is not the set waiting here — %s", orNone(set), h.waitingWords())
|
||||
}
|
||||
h.r.rejected = &rejectedSet{set: h.r.waiting.set, by: by, why: why, at: h.Now()}
|
||||
h.r.waiting = nil
|
||||
h.say("retirement of %s REJECTED by %s: %s. Kept active, and not asked about again while the mesh goes on "+
|
||||
"not asking for exactly these (until this provider restarts)", strings.Join(set, ", "), orSomebody(by), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeRejected, h.named(set),
|
||||
map[string]any{"why": why, "by": by, "via": via}))
|
||||
return set, nil
|
||||
}
|
||||
|
||||
// DeleteRetired deletes one retired consumer, by a person's word: never an active one, never one the
|
||||
// mesh asks for.
|
||||
func (h *Harness) DeleteRetired(ctx context.Context, consumer, why, by, via string) (int64, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return 0, errors.New("delete: say why")
|
||||
}
|
||||
if h.r.lastWant[consumer] {
|
||||
return 0, fmt.Errorf("delete: the mesh asks for %s — it is not retired", consumer)
|
||||
}
|
||||
if _, held := h.applied[consumer]; held {
|
||||
return 0, fmt.Errorf("delete: %s is active here — only a retired consumer is deleted", consumer)
|
||||
}
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete: what the backend holds cannot be read, so nothing is deleted: %w", err)
|
||||
}
|
||||
var found *Retired
|
||||
for i := range inv.Retired {
|
||||
if inv.Retired[i].Consumer == consumer {
|
||||
found = &inv.Retired[i]
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
return 0, fmt.Errorf("delete: %s is not retired here — only a retired consumer is deleted", consumer)
|
||||
}
|
||||
freed, err := h.Adapter.Delete(ctx, *found)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
delete(h.r.retired, consumer)
|
||||
h.say("%s: DELETED by %s: %s (retired %s: %s; %d bytes freed)", consumer, orSomebody(by), why,
|
||||
stampOr(found.RetiredAt), found.Why, freed)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeDeleted, []map[string]any{{
|
||||
"consumer": consumer, "node": found.Node, "retired_at": stampOr(found.RetiredAt), "why": found.Why,
|
||||
"size_bytes": found.SizeBytes, "kind": kindOf(*found),
|
||||
}}, map[string]any{"why": why, "by": by, "via": via, "freed_bytes": freed}))
|
||||
return freed, nil
|
||||
}
|
||||
|
||||
// Retirement is what a person (and the controller's `cleanup list` and its probe) asks: what is
|
||||
// held, waiting, rejected and retired here.
|
||||
func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var held []string
|
||||
for as := range h.applied {
|
||||
held = append(held, as)
|
||||
}
|
||||
sort.Strings(held)
|
||||
retired := []map[string]any{}
|
||||
for _, r := range inv.Retired {
|
||||
retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node,
|
||||
"retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)})
|
||||
}
|
||||
out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held),
|
||||
"stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil,
|
||||
"retired": retired}
|
||||
if w := h.r.waiting; w != nil {
|
||||
out["waiting"] = map[string]any{"consumers": h.named(w.set), "since": stamp(w.since), "held": w.held}
|
||||
}
|
||||
if r := h.r.rejected; r != nil {
|
||||
out["rejected"] = map[string]any{"consumers": h.named(r.set), "by": r.by, "why": r.why, "at": stamp(r.at)}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *Harness) waitingWords() string {
|
||||
switch {
|
||||
case h.r.waiting != nil:
|
||||
return "waiting: " + strings.Join(h.r.waiting.set, ", ")
|
||||
case h.r.rejected != nil:
|
||||
return "nothing waits; rejected and kept: " + strings.Join(h.r.rejected.set, ", ")
|
||||
}
|
||||
return "nothing waits for a person here"
|
||||
}
|
||||
|
||||
// named is a set with each consumer's machine, as the events and the tools say it.
|
||||
func (h *Harness) named(set []string) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(set))
|
||||
for _, as := range set {
|
||||
out = append(out, map[string]any{"consumer": as, "node": h.applied[as].node})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *Harness) retirementBody(change string, consumers []map[string]any, extra map[string]any) map[string]any {
|
||||
body := map[string]any{"provider": h.Resource, "provider-node": h.Node, "change": change,
|
||||
"consumers": consumers, "at": stamp(h.Now())}
|
||||
for k, v := range extra {
|
||||
body[k] = v
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// RetirementTools are the four tools every provider serves for retirement, the same names in every
|
||||
// module: the controller's `retire` and `cleanup` verbs ask them on the provider's machine.
|
||||
func RetirementTools(h *Harness) []stdio.Tool {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
ask := func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
}
|
||||
who := map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why — required, kept in the hand-act log"},
|
||||
"by": map[string]any{"type": "string", "description": "who decided"},
|
||||
"via": map[string]any{"type": "string", "description": "mesh-controller when asked through its verbs"},
|
||||
}
|
||||
withSet := map[string]any{"consumers": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the set, exactly as provisioner_retirement names it"}}
|
||||
for k, v := range who {
|
||||
withSet[k] = v
|
||||
}
|
||||
withOne := map[string]any{
|
||||
"consumer": map[string]any{"type": "string", "description": "the retired consumer to delete"},
|
||||
"confirm": map[string]any{"type": "string", "description": "the consumer's name again, to say this is meant"},
|
||||
}
|
||||
for k, v := range who {
|
||||
withOne[k] = v
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: "provisioner_retirement",
|
||||
Description: "What this provider holds, what waits for a person to approve its retirement, what was rejected, " +
|
||||
"and every retired consumer with when, why and its size (novox/hq ADR 0230).",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
return h.Retirement(ctx)
|
||||
}},
|
||||
{Name: "provisioner_retire_approve",
|
||||
Description: "Retire exactly the set waiting for a person (or the set rejected earlier): access disabled, data " +
|
||||
"kept and marked to delete. Use the controller's `retire approve`, which records the hand act.",
|
||||
Input: withSet,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
done, err := h.Approve(ctx, strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"retired": orEmptyList(done)}, nil
|
||||
}},
|
||||
{Name: "provisioner_retire_reject",
|
||||
Description: "Keep the set waiting for a person active. Use the controller's `retire reject`.",
|
||||
Input: withSet,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
kept, err := h.Reject(strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"kept": kept}, nil
|
||||
}},
|
||||
{Name: "provisioner_delete",
|
||||
Description: "Delete one RETIRED consumer — its login and its data — for good. Refused for anything active or " +
|
||||
"asked for. Use the controller's `cleanup delete`, which records the hand act.",
|
||||
Input: withOne,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
consumer := argString(args, "consumer")
|
||||
if consumer == "" || argString(args, "confirm") != consumer {
|
||||
return nil, errors.New("delete: name the consumer, and repeat it in confirm")
|
||||
}
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
freed, err := h.DeleteRetired(ctx, consumer, argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"deleted": consumer, "freed_bytes": freed}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func strs(v any) []string {
|
||||
list, _ := v.([]any)
|
||||
out := []string{}
|
||||
for _, x := range list {
|
||||
if s, ok := x.(string); ok && s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sorted(list []string) []string {
|
||||
out := append([]string(nil), list...)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func same(a, b []string) bool {
|
||||
return strings.Join(sorted(a), "\x00") == strings.Join(sorted(b), "\x00")
|
||||
}
|
||||
|
||||
func orNone(set []string) string {
|
||||
if len(set) == 0 {
|
||||
return "an empty set"
|
||||
}
|
||||
return strings.Join(set, ", ")
|
||||
}
|
||||
|
||||
func orSomebody(by string) string {
|
||||
if by == "" {
|
||||
return "a person"
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
func orEmptyList(list []string) []string {
|
||||
if list == nil {
|
||||
return []string{}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func kindOf(r Retired) string {
|
||||
if r.Kind == "" {
|
||||
return KindConsumer
|
||||
}
|
||||
return r.Kind
|
||||
}
|
||||
|
||||
func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339) }
|
||||
|
||||
func stampOr(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return stamp(t)
|
||||
}
|
||||
|
||||
func argString(args map[string]any, key string) string {
|
||||
s, _ := args[key].(string)
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
package main
|
||||
|
||||
// Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is
|
||||
// retired only after the same result in five consecutive passes, too many at once wait for a person,
|
||||
// asked for again it is re-enabled, and only a person deletes. The same file in every Go provider.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// recorder is a backend that remembers what it holds, active and retired, as a real one's mark does.
|
||||
type recorder struct {
|
||||
created []Provision
|
||||
removed []string // retired, in order
|
||||
deleted []string
|
||||
held bool
|
||||
failing error
|
||||
holdsErr error
|
||||
retErr error
|
||||
inv Inventory
|
||||
invErr error
|
||||
}
|
||||
|
||||
func (r *recorder) Create(_ context.Context, p Provision) error {
|
||||
if r.failing != nil {
|
||||
return r.failing
|
||||
}
|
||||
r.created = append(r.created, p)
|
||||
r.inv.Retired = withoutRetired(r.inv.Retired, p.As)
|
||||
if !listHas(r.inv.Active, p.As) {
|
||||
r.inv.Active = append(r.inv.Active, p.As)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error {
|
||||
if r.retErr != nil {
|
||||
return r.retErr
|
||||
}
|
||||
r.removed = append(r.removed, as)
|
||||
r.inv.Active = without(r.inv.Active, as)
|
||||
r.inv.Retired = append(withoutRetired(r.inv.Retired, as),
|
||||
Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Holds(context.Context, Provision) (bool, error) {
|
||||
if r.holdsErr != nil {
|
||||
return false, r.holdsErr
|
||||
}
|
||||
return r.held, nil
|
||||
}
|
||||
|
||||
func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr }
|
||||
|
||||
func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) {
|
||||
r.deleted = append(r.deleted, x.Consumer)
|
||||
r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer)
|
||||
return x.SizeBytes, nil
|
||||
}
|
||||
|
||||
func listHas(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func without(list []string, s string) []string {
|
||||
var out []string
|
||||
for _, x := range list {
|
||||
if x != s {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func withoutRetired(list []Retired, s string) []Retired {
|
||||
var out []Retired
|
||||
for _, x := range list {
|
||||
if x.Consumer != s {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// holding gives the provider n consumers c1…cn and applies them.
|
||||
func holding(w *world, n int) []map[string]any {
|
||||
var given []map[string]any
|
||||
for i := 1; i <= n; i++ {
|
||||
given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"})
|
||||
}
|
||||
w.give(given...)
|
||||
w.h.Reconcile(ctx)
|
||||
return given
|
||||
}
|
||||
|
||||
// pass is one reconcile five seconds after the last.
|
||||
func (w *world) pass() {
|
||||
w.now = w.now.Add(5 * time.Second)
|
||||
w.h.Reconcile(ctx)
|
||||
}
|
||||
|
||||
func retirements(said []announced) []string {
|
||||
var out []string
|
||||
for _, a := range said {
|
||||
if a.event == EventRetirement {
|
||||
out = append(out, a.body["change"].(string))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func lastRetirement(t *testing.T, said []announced) map[string]any {
|
||||
t.Helper()
|
||||
for i := len(said) - 1; i >= 0; i-- {
|
||||
if said[i].event == EventRetirement {
|
||||
return said[i].body
|
||||
}
|
||||
}
|
||||
t.Fatal("nothing about retirement was announced")
|
||||
return nil
|
||||
}
|
||||
|
||||
func namesOf(body map[string]any) string {
|
||||
var out []string
|
||||
for _, c := range body["consumers"].([]map[string]any) {
|
||||
out = append(out, c["consumer"].(string))
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
// settle passes every five seconds until the same answer has held for StableFor, and one pass more.
|
||||
func (w *world) settle() { w.passes(StableFor + 5*time.Second) }
|
||||
|
||||
func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 1)
|
||||
w.give()
|
||||
for i := 0; i < 4; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
|
||||
t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said))
|
||||
}
|
||||
}
|
||||
|
||||
// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold:
|
||||
// five passes AND ten minutes of the same answer (novox/hq ADR 0230).
|
||||
func TestFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
holding(w, 1)
|
||||
w.give()
|
||||
for i := 0; i < 5; i++ {
|
||||
w.pass()
|
||||
}
|
||||
if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
|
||||
t.Fatalf("five passes in 25 s retired %v", w.a.removed)
|
||||
}
|
||||
w.passes(StableFor - 30*time.Second)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed)
|
||||
}
|
||||
w.passes(time.Minute)
|
||||
if strings.Join(w.a.removed, ",") != "c1" {
|
||||
t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed)
|
||||
}
|
||||
// Ten minutes in one slow pass are not five passes.
|
||||
w2 := newWorld(t)
|
||||
holding(w2, 1)
|
||||
w2.give()
|
||||
w2.pass()
|
||||
w2.now = w2.now.Add(StableFor)
|
||||
w2.pass()
|
||||
if len(w2.a.removed) != 0 {
|
||||
t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed)
|
||||
}
|
||||
w2.passes(15 * time.Second)
|
||||
if len(w2.a.removed) != 1 {
|
||||
t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 3)
|
||||
w.give(given[:2]...)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "c3" {
|
||||
t.Fatalf("retired %v", w.a.removed)
|
||||
}
|
||||
body := lastRetirement(t, *said)
|
||||
if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" ||
|
||||
!strings.Contains(body["why"].(string), "consecutive passes over 10m") {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" {
|
||||
t.Fatalf("the backend does not hold it retired: %+v", w.a.inv)
|
||||
}
|
||||
// Asked for again: the ordinary create, on the first pass, and said.
|
||||
created := len(w.a.created)
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if len(w.a.created) != created+1 || w.a.created[created].As != "c3" {
|
||||
t.Fatalf("not created again: %v", w.a.created)
|
||||
}
|
||||
if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if len(w.a.inv.Retired) != 0 {
|
||||
t.Fatalf("still marked retired: %+v", w.a.inv.Retired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadablePassStartsTheCountAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
holding(w, 1)
|
||||
w.give()
|
||||
w.passes(StableFor - time.Minute)
|
||||
os.WriteFile(w.receives, []byte("{"), 0o600)
|
||||
w.pass()
|
||||
w.give()
|
||||
w.passes(StableFor - time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("an unreadable pass counted: %v", w.a.removed)
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if len(w.a.removed) != 1 {
|
||||
t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADifferentSetStartsTheCountAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
given := holding(w, 5)
|
||||
w.give(given[:4]...)
|
||||
w.passes(StableFor - time.Minute)
|
||||
w.give(given[:3]...)
|
||||
w.passes(StableFor - time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("a changed set kept its count: %v", w.a.removed)
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if strings.Join(w.a.removed, ",") != "c4,c5" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdditionsAndChangesAreNeverDelayed(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
holding(w, 1)
|
||||
w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"})
|
||||
w.pass()
|
||||
if len(w.a.created) != 2 || w.a.created[1].As != "c2" {
|
||||
t.Fatalf("an addition waited: %v", w.a.created)
|
||||
}
|
||||
w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"})
|
||||
w.pass()
|
||||
if len(w.a.created) != 3 || w.a.created[2].As != "c1" {
|
||||
t.Fatalf("a change waited: %v", w.a.created)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
holding(w, 4)
|
||||
w.give()
|
||||
w.passes(15 * time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("four of four were retired without a person: %v", w.a.removed)
|
||||
}
|
||||
if got := strings.Join(retirements(*said), ","); got != ChangeWaiting {
|
||||
t.Fatalf("said %q, want one waiting", got)
|
||||
}
|
||||
body := lastRetirement(t, *said)
|
||||
if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") {
|
||||
t.Fatal("the wait was not said")
|
||||
}
|
||||
// Said again every quarter of an hour while it waits.
|
||||
w.passes(11 * time.Minute)
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
|
||||
if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil {
|
||||
t.Fatal("approved a set that is not the one waiting")
|
||||
}
|
||||
if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil {
|
||||
t.Fatal("approved without a why")
|
||||
}
|
||||
done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller")
|
||||
if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" {
|
||||
t.Fatal(done, err, w.a.removed)
|
||||
}
|
||||
changes := retirements(*said)
|
||||
if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" {
|
||||
t.Fatalf("%v", changes)
|
||||
}
|
||||
if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" ||
|
||||
!strings.Contains(b["why"].(string), "the old machine is gone") {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
// Nothing more waits.
|
||||
w.passes(time.Minute)
|
||||
if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil {
|
||||
t.Fatal("rejected a set that is not the one waiting")
|
||||
}
|
||||
kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller")
|
||||
if err != nil || len(kept) != 4 {
|
||||
t.Fatal(kept, err)
|
||||
}
|
||||
w.passes(time.Hour)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("a rejected set was retired: %v", w.a.removed)
|
||||
}
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" {
|
||||
t.Fatalf("asked again after a rejection: %q", got)
|
||||
}
|
||||
r, _ := w.h.Retirement(ctx)
|
||||
if r["rejected"] == nil || r["waiting"] != nil {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
// One of them asked for again: a different answer, so the rejection is settled and counting
|
||||
// starts over — three of four is over the bound again, and waits again.
|
||||
w.give(given[0])
|
||||
w.pass()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
w.settle()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
// A rejected set can still be approved later.
|
||||
w2, _ := standingWorld(t)
|
||||
holding(w2, 4)
|
||||
w2.give()
|
||||
w2.settle()
|
||||
w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "")
|
||||
if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 {
|
||||
t.Fatal(done, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingSetAskedForAgainSettles(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 {
|
||||
t.Fatalf("%q %v", got, w.a.removed)
|
||||
}
|
||||
if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil {
|
||||
t.Fatal("approved a set that no longer waits")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 5)
|
||||
w.give(given[:3]...)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "c4,c5" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted an active consumer")
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil {
|
||||
t.Fatal("deleted without a why")
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted something not retired")
|
||||
}
|
||||
freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller")
|
||||
if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" {
|
||||
t.Fatal(freed, err, w.a.deleted)
|
||||
}
|
||||
if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
r, _ := w.h.Retirement(ctx)
|
||||
if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
// Retired and asked for again before anybody deleted it: refused, it is the mesh's again.
|
||||
w.give(given[:4]...)
|
||||
w.pass()
|
||||
if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted a consumer the mesh asks for")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBound(t *testing.T) {
|
||||
h := &Harness{}
|
||||
h.init()
|
||||
for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} {
|
||||
if h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d waits for a person", c.n, c.held)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} {
|
||||
if !h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d is retired without a person", c.n, c.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{
|
||||
{Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer},
|
||||
{Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"},
|
||||
}}
|
||||
w.give(map[string]any{"as": "kept"})
|
||||
w.h.Reconcile(ctx)
|
||||
if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
if strings.Join(w.a.removed, ",") != "locked-before" {
|
||||
t.Fatalf("adopting did not mark it: %v", w.a.removed)
|
||||
}
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "locked-before,orphan" {
|
||||
t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.a.invErr = errors.New("connection refused")
|
||||
holding(w, 1)
|
||||
if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") {
|
||||
t.Fatal(w.said)
|
||||
}
|
||||
w.a.invErr = nil
|
||||
w.a.inv = Inventory{Active: []string{"c1", "orphan"}}
|
||||
w.pass()
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "orphan" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsAnswer(t *testing.T) {
|
||||
w, _ := standingWorld(t)
|
||||
holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tool := range RetirementTools(w.h) {
|
||||
tools[tool.Name] = tool.Run
|
||||
}
|
||||
if len(tools) != 4 {
|
||||
t.Fatalf("%d tools", len(tools))
|
||||
}
|
||||
got, err := tools["provisioner_retirement"](nil)
|
||||
if err != nil || got.(map[string]any)["waiting"] == nil {
|
||||
t.Fatal(got, err)
|
||||
}
|
||||
set := []any{"c1", "c2", "c3", "c4"}
|
||||
if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil {
|
||||
t.Fatal("approved without a why")
|
||||
}
|
||||
if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil {
|
||||
t.Fatal("deleted without confirm")
|
||||
}
|
||||
if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(w.a.deleted, ",") != "c1" {
|
||||
t.Fatal(w.a.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailingConsumerRetiredIsSaidRecovered(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 2)
|
||||
w.a.held = false
|
||||
w.a.failing = errors.New("boom")
|
||||
w.passes(7 * time.Minute)
|
||||
w.give(given[0])
|
||||
w.settle()
|
||||
recovered := false
|
||||
for _, a := range *said {
|
||||
if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" {
|
||||
recovered = true
|
||||
}
|
||||
}
|
||||
if !recovered {
|
||||
t.Fatalf("%v", *said)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func TestAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWithdrawnConsumerIsNoLongerFailing(t *testing.T) {
|
||||
func TestAConsumerNoLongerAskedForIsNoLongerFailing(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.failing = errors.New("boom")
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
||||
@@ -139,7 +139,7 @@ func TestAWithdrawnConsumerIsNoLongerFailing(t *testing.T) {
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.passes(5 * time.Second)
|
||||
last := (*said)[len(*said)-1]
|
||||
if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "withdrawn" {
|
||||
if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "no longer asked for" {
|
||||
t.Fatalf("%v", *said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@
|
||||
"user.deleted",
|
||||
"password.reset",
|
||||
"client.created",
|
||||
"client.retired",
|
||||
"group.created",
|
||||
"role.created",
|
||||
"admin.repaired",
|
||||
|
||||
@@ -38,9 +38,12 @@ for, so one removed by hand is installed again.
|
||||
|
||||
## What is never done
|
||||
|
||||
- **No database is ever dropped.** A consumer the mesh no longer asks for is *withdrawn*: its login is
|
||||
set `NOLOGIN` and its sessions are ended, and its database stays as it was under its own name
|
||||
(issue 241). A consumer that comes back is given the same database.
|
||||
- **No database is dropped by the mesh on its own.** A consumer the mesh no longer asks for is
|
||||
*retired* (novox/hq ADR 0230), once the same result holds for five passes and ten minutes and, if
|
||||
it is more than three consumers or more than half of those held, once a person approved: its login is set `NOLOGIN`,
|
||||
its sessions are ended, its role's comment marks it retired with when and why, and its database stays
|
||||
exactly as it was under its own name — still backed up. A consumer asked for again is enabled at once
|
||||
with the same database. Only `cleanup delete`, a person's act through the controller, drops it.
|
||||
- **`postgres_retire_database` renames, it does not drop**: the database becomes
|
||||
`<name>_deleted_<yyyymmdd>` and its owner is locked. Removing the data is a person's act, by hand.
|
||||
- **A caller's statement never runs as the superuser.** `postgres_query` and the seat's `query` verb
|
||||
@@ -56,16 +59,21 @@ for, so one removed by hand is installed again.
|
||||
| `postgres_query` `{database, sql}` | one read-only statement, as the reader; rows keyed by column, `NULL` as null |
|
||||
| `postgres_retire_database` `{database, confirm}` | renames a database aside and locks its owner; `confirm` repeats the name |
|
||||
| `mesh-store.databases`, `mesh-store.query` | the store seat's verbs: the same listing and read-only query |
|
||||
| `provisioner_retirement` | what is held, what waits for a person, what was rejected, every retired consumer and set-aside database with when, why and size |
|
||||
| `provisioner_retire_approve`, `provisioner_retire_reject` `{consumers, why, by}` | a person's answer to a set waiting; through the controller's `retire approve|reject` |
|
||||
| `provisioner_delete` `{consumer, confirm, why, by}` | drops one retired consumer's database and role, or one set-aside database; through the controller's `cleanup delete` |
|
||||
|
||||
## Where the code lives
|
||||
|
||||
One Go bundle, `cmd/postgres-provider`, launched by the node's runtime and speaking MCP over stdio
|
||||
through the Go SDK (ADR 0193). Beside the tools it runs the provisioner: every five seconds it reads the
|
||||
contributions file the mesh writes (`MESH_RECEIVES`), applies each consumer whose login, password or
|
||||
contribution changed, and withdraws each one no longer listed; a file it cannot read withdraws nobody.
|
||||
It is the TypeScript SDK's `runProvisioner` loop, carried in the module until the Go SDK has one.
|
||||
contribution changed, and retires what is no longer listed (`retirement.go`); a file it cannot read
|
||||
retires nobody. It is the TypeScript SDK's `runProvisioner` loop, carried in the module until the Go SDK
|
||||
has one, byte for byte the same as keycloak's.
|
||||
|
||||
`go test ./...` runs against a fake server. `MESH_POSTGRES_LIVE=postgres://postgres:…@host:port/postgres`
|
||||
also runs `live_test.go` against a real, throwaway one (see the file for a `pgvector/pgvector`
|
||||
container): the extension installed and a second pass a no-op, the reader unable to write, a
|
||||
withdrawn login locked out with its data kept.
|
||||
retired login locked out with its data kept and listed retired, enabled again as it was, and a deletion
|
||||
dropping only its own database and role.
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
package main
|
||||
|
||||
// The withdrawal brake (novox/hq to-be 45 Phase 2, ADR 0227 rule 4; replay R6 of issue 241): a pass that
|
||||
// would withdraw more than its bound withdraws nothing, says so, and announces every consumer it kept as
|
||||
// failing with the class withdrawal-braked, which the controller raises as a condition; one is let go
|
||||
// every hour while the mesh goes on not asking; a consumer asked for again is kept and said recovered.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// sevenConsumers is the control node's postgres on 2026-10-04: seven databases, all in use.
|
||||
func sevenConsumers(w *world) {
|
||||
var given []map[string]any
|
||||
for i := 1; i <= 7; i++ {
|
||||
given = append(given, map[string]any{"as": fmt.Sprintf("consumer%d", i), "node": "anchor"})
|
||||
}
|
||||
w.give(given...)
|
||||
w.h.Reconcile(ctx)
|
||||
}
|
||||
|
||||
func TestAWithdrawalOfEveryConsumerIsBrakedAndSaid(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
sevenConsumers(w)
|
||||
// A file read whole that names nobody: the shape of 241 that its first fix does not catch.
|
||||
w.give()
|
||||
w.passes(6 * time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("a pass over its bound withdrew %v", w.a.removed)
|
||||
}
|
||||
braked := 0
|
||||
for _, a := range *said {
|
||||
if a.event == EventFailing && a.body["class"] == ClassWithdrawalBraked && a.body["node"] == "anchor" {
|
||||
braked++
|
||||
}
|
||||
}
|
||||
if braked != 7 {
|
||||
t.Fatalf("%d of the 7 consumers kept were announced as braked: %v", braked, *said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(w.said, "\n"), "WITHDRAWAL BRAKED") {
|
||||
t.Fatal("the brake was not said")
|
||||
}
|
||||
|
||||
// One is let go once the brake has held an hour, and then one an hour.
|
||||
w.passes(55 * time.Minute)
|
||||
if len(w.a.removed) != 1 {
|
||||
t.Fatalf("after an hour of the mesh not asking, %d were withdrawn, want 1: %v", len(w.a.removed), w.a.removed)
|
||||
}
|
||||
w.passes(59 * time.Minute)
|
||||
if len(w.a.removed) != 1 {
|
||||
t.Fatalf("a second was let go within the hour: %v", w.a.removed)
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if len(w.a.removed) != 2 {
|
||||
t.Fatalf("the second was not let go after another hour: %v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConsumerAskedForAgainIsKeptAndNotWithdrawn(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
sevenConsumers(w)
|
||||
w.give()
|
||||
w.passes(6 * time.Minute)
|
||||
// The file is right again: every consumer is asked for, nothing was withdrawn, each is recovered.
|
||||
sevenConsumers(w)
|
||||
w.passes(5 * time.Second)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("withdrew %v", w.a.removed)
|
||||
}
|
||||
recovered := 0
|
||||
for _, a := range *said {
|
||||
if a.event == EventRecovered && a.body["why"] == nil {
|
||||
recovered++
|
||||
}
|
||||
}
|
||||
if recovered != 7 {
|
||||
t.Fatalf("%d of the 7 kept consumers were said recovered: %v", recovered, *said)
|
||||
}
|
||||
if len(w.h.braked) != 0 {
|
||||
t.Fatalf("the brake still holds %v", w.h.braked)
|
||||
}
|
||||
}
|
||||
|
||||
// Within the bound — one consumer of several, or the last one held — a withdrawal goes as asked.
|
||||
func TestAWithdrawalWithinItsBoundIsNotBraked(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}, map[string]any{"as": "c"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
||||
w.h.Reconcile(ctx)
|
||||
if strings.Join(w.a.removed, ",") != "c" {
|
||||
t.Fatalf("one of three was not withdrawn: %v", w.a.removed)
|
||||
}
|
||||
// Two of the remaining two at once is over the bound.
|
||||
w.give()
|
||||
w.h.Reconcile(ctx)
|
||||
if strings.Join(w.a.removed, ",") != "c" {
|
||||
t.Fatalf("two at once were withdrawn: %v", w.a.removed)
|
||||
}
|
||||
for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}} {
|
||||
if w.h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d is over the bound", c.n, c.held)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct{ n, held int }{{2, 2}, {2, 7}, {7, 7}} {
|
||||
if !w.h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d is within the bound", c.n, c.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,12 @@
|
||||
package main
|
||||
|
||||
// The reconcile loop every provider shares, as the TypeScript SDK's runProvisioner runs it
|
||||
// (@novox/mesh-sdk/provisioner, 0.1.11). The Go SDK has no provisioner yet, so this module carries
|
||||
// (@novox/mesh-sdk/provisioner, 0.1.12). The Go SDK has no provisioner yet, so this module carries
|
||||
// the loop itself, line for line in behaviour; when the Go SDK grows one, this file is what moves
|
||||
// there (novox/hq ADR 0039: the loop is the SDK's, the adapter is the module's).
|
||||
//
|
||||
// Read the contributions the mesh delivered; bring each consumer's resource into being through the
|
||||
// adapter, under the login and password the mesh minted; withdraw what the mesh no longer asks for.
|
||||
// adapter, under the login and password the mesh minted; retire what the mesh no longer asks for.
|
||||
// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).**
|
||||
//
|
||||
// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer
|
||||
@@ -17,18 +17,16 @@ package main
|
||||
// identity provider failed every consumer 31,000 times in a day and said so only in its journal
|
||||
// (novox/hq issue 179).
|
||||
//
|
||||
// **A reconcile that would withdraw more than its bound stops, and says so (novox/hq to-be 45 Phase 2,
|
||||
// ADR 0227 rule 4).** Withdrawing more than WithdrawAtOnce consumers in one pass — or more than
|
||||
// WithdrawFraction of those this process holds — is the shape of issue 241, where one misread file
|
||||
// withdrew seven at once. Such a pass withdraws nothing: each consumer it would have withdrawn is kept,
|
||||
// announced `provisioner.failing` with the class `withdrawal-braked` (the controller raises it as a
|
||||
// condition), and said. While the same consumers stay unasked for, one is released every ReleaseEvery,
|
||||
// said and announced as it goes, so an intended unassignment of many completes without a hand and a
|
||||
// mistaken one costs at most one consumer an hour while the operator is told. Withdrawal never destroys
|
||||
// data (issue 241's second half), so a release is a login locked, not a database dropped.
|
||||
// **A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
// (novox/hq ADR 0230, the operator's model of 2026-10-06).** Retiring disables its access — reversibly —
|
||||
// and marks its login and data "to delete" with when and why; nothing is deleted. Asked for again, the
|
||||
// ordinary create re-enables it as it was. It is retired only once the same set has gone unasked in
|
||||
// StablePasses consecutive passes and for StableFor, and a set larger than the bound waits for a
|
||||
// person. retirement.go.
|
||||
//
|
||||
// Carried, identical, by every Go provider until the Go SDK has the loop: postgres and keycloak.
|
||||
// Each module's `harness_same_test.go` fails when its copy and the other's differ.
|
||||
// Each module's `harness_same_test.go` fails when its copy and the other's differ (this file and
|
||||
// retirement.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -37,8 +35,8 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -59,11 +57,21 @@ type Provision struct {
|
||||
|
||||
// Adapter is the per-service half.
|
||||
type Adapter interface {
|
||||
// Create brings the consumer into being under the mesh's login and password — and, for one that
|
||||
// was retired, re-enables it as it was and clears its mark to delete.
|
||||
Create(ctx context.Context, p Provision) error
|
||||
// Remove withdraws what Create made; derived is what the mesh last derived, remembered here.
|
||||
Remove(ctx context.Context, as string, derived map[string]any) error
|
||||
// Retire disables the consumer's access, reversibly, and marks its login and data to delete with
|
||||
// when and why. It deletes nothing (novox/hq ADR 0230). derived is what the mesh last derived,
|
||||
// remembered here; nil for a consumer this process did not make.
|
||||
Retire(ctx context.Context, as string, derived map[string]any, why string, at time.Time) error
|
||||
// Holds says whether the backend still holds the consumer exactly as p says. Read-only.
|
||||
Holds(ctx context.Context, p Provision) (bool, error)
|
||||
// Inventory is what the backend holds that the mesh made: consumers active and retired, read from
|
||||
// the backend itself so a restart forgets nothing. Read-only.
|
||||
Inventory(ctx context.Context) (Inventory, error)
|
||||
// Delete removes one retired consumer — its login and its data — for good. Only ever asked by a
|
||||
// person, through `cleanup delete`; the harness has checked it is retired and not asked for.
|
||||
Delete(ctx context.Context, r Retired) (freedBytes int64, err error)
|
||||
}
|
||||
|
||||
// Harness is the loop's settings and memory.
|
||||
@@ -85,19 +93,19 @@ type Harness struct {
|
||||
// missed the first hears the next, and a standing nobody repeats can be told from one that holds.
|
||||
FailingAfter time.Duration
|
||||
SayAgainEvery time.Duration
|
||||
// WithdrawAtOnce (1) and WithdrawFraction (0.5) bound what one pass may withdraw: more consumers
|
||||
// than WithdrawAtOnce, or a larger share of those held than WithdrawFraction, brakes the pass.
|
||||
// ReleaseEvery (1h) is how often a braked withdrawal lets one consumer go.
|
||||
WithdrawAtOnce int
|
||||
WithdrawFraction float64
|
||||
ReleaseEvery time.Duration
|
||||
// StablePasses (5) is how many consecutive passes must see the same set no longer asked for, and
|
||||
// StableFor (10m) how long it must have held since the first of them, before it is retired. RetireAtOnce (3) and RetireFraction (0.5) bound what is retired without a person:
|
||||
// more consumers than RetireAtOnce, or — where more than one is held — a larger share of those held
|
||||
// than RetireFraction, waits for `retire approve` (novox/hq ADR 0230).
|
||||
StablePasses int
|
||||
StableFor time.Duration
|
||||
RetireAtOnce int
|
||||
RetireFraction float64
|
||||
|
||||
verifiedAt time.Time
|
||||
// braked is every consumer a braked pass kept, by when it was first kept; releasedAt is when the
|
||||
// brake last let one go, and brakeSaid the set it last said, so a pass repeats nothing.
|
||||
braked map[string]time.Time
|
||||
releasedAt time.Time
|
||||
brakeSaid string
|
||||
// mu is held by a pass and by every tool a person asks, so the two never interleave.
|
||||
mu sync.Mutex
|
||||
verifiedAt time.Time
|
||||
r retirement
|
||||
applied map[string]appliedEntry
|
||||
lost map[string]brake
|
||||
waiting map[string]int
|
||||
@@ -131,9 +139,6 @@ const (
|
||||
ClassUnreachable = "unreachable"
|
||||
ClassSecret = "secret-unreadable"
|
||||
ClassRefused = "refused"
|
||||
// ClassWithdrawalBraked is a consumer the mesh no longer asks for, kept because the pass that would
|
||||
// withdraw it would withdraw more than its bound (ADR 0227 rule 4).
|
||||
ClassWithdrawalBraked = "withdrawal-braked"
|
||||
)
|
||||
|
||||
// Classifier is an adapter that can say what class an error of its own is.
|
||||
@@ -196,14 +201,17 @@ func (h *Harness) init() {
|
||||
if h.SayAgainEvery == 0 {
|
||||
h.SayAgainEvery = 15 * time.Minute
|
||||
}
|
||||
if h.WithdrawAtOnce == 0 {
|
||||
h.WithdrawAtOnce = 1
|
||||
if h.StablePasses == 0 {
|
||||
h.StablePasses = 5
|
||||
}
|
||||
if h.WithdrawFraction == 0 {
|
||||
h.WithdrawFraction = 0.5
|
||||
if h.StableFor == 0 {
|
||||
h.StableFor = StableFor
|
||||
}
|
||||
if h.ReleaseEvery == 0 {
|
||||
h.ReleaseEvery = time.Hour
|
||||
if h.RetireAtOnce == 0 {
|
||||
h.RetireAtOnce = 3
|
||||
}
|
||||
if h.RetireFraction == 0 {
|
||||
h.RetireFraction = 0.5
|
||||
}
|
||||
if h.Log == nil {
|
||||
h.Log = func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }
|
||||
@@ -215,7 +223,7 @@ func (h *Harness) init() {
|
||||
h.failing = map[string]failure{}
|
||||
h.trouble = map[string]*standing{}
|
||||
h.cleared = map[string]bool{}
|
||||
h.braked = map[string]time.Time{}
|
||||
h.r.retired = map[string]retiredEntry{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,7 +257,7 @@ func (h *Harness) warn(why string) {
|
||||
}
|
||||
|
||||
// readContributions answers the consumers asked for, or nil when the file says nothing usable.
|
||||
// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can withdraw.
|
||||
// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can retire anything.
|
||||
func (h *Harness) readContributions() []contribution {
|
||||
raw, err := os.ReadFile(h.Receives)
|
||||
if err != nil {
|
||||
@@ -299,15 +307,20 @@ func orEmpty(m map[string]any) map[string]any {
|
||||
|
||||
// Reconcile is one pass.
|
||||
func (h *Harness) Reconcile(ctx context.Context) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
given := h.readContributions()
|
||||
if given == nil {
|
||||
// Not a result: the passes that must agree before anything is retired start again.
|
||||
h.r.key, h.r.count = "", 0
|
||||
return
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, g := range given {
|
||||
want[g.As] = true
|
||||
}
|
||||
h.seed(ctx, want)
|
||||
verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery
|
||||
if verifying {
|
||||
h.verifiedAt = h.Now()
|
||||
@@ -396,6 +409,7 @@ func (h *Harness) Reconcile(ctx context.Context) {
|
||||
}
|
||||
h.succeeded(g.As)
|
||||
h.applied[g.As] = appliedEntry{hash: hash, derived: p.Derived, node: g.Node}
|
||||
h.reenabled(g.As, g.Node)
|
||||
if reapplying == 0 {
|
||||
delete(h.lost, g.As)
|
||||
} else {
|
||||
@@ -410,97 +424,24 @@ func (h *Harness) Reconcile(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Withdraw every login this process made that the mesh no longer asks for — within the bound.
|
||||
var withdrawing []string
|
||||
for as := range h.applied {
|
||||
if !want[as] {
|
||||
withdrawing = append(withdrawing, as)
|
||||
}
|
||||
}
|
||||
sort.Strings(withdrawing)
|
||||
for as := range h.braked {
|
||||
if want[as] {
|
||||
// Asked for again: the brake held what the mesh still wanted. Its standing was ended by this
|
||||
// pass's success above, as any consumer's is.
|
||||
delete(h.braked, as)
|
||||
}
|
||||
}
|
||||
if h.overTheBound(len(withdrawing), len(h.applied)) {
|
||||
withdrawing = h.brakeWithdrawal(withdrawing)
|
||||
} else if len(h.braked) > 0 {
|
||||
h.say("the withdrawal is within its bound again: %s withdrawn as asked", strings.Join(withdrawing, ", "))
|
||||
h.braked, h.brakeSaid = map[string]time.Time{}, ""
|
||||
}
|
||||
for _, as := range withdrawing {
|
||||
was := h.applied[as]
|
||||
h.say("%s: no longer in %s; withdrawing it from the backend", as, h.Receives)
|
||||
if err := h.Adapter.Remove(ctx, as, was.derived); err != nil {
|
||||
h.say("%s: remove failed, will retry: %v", as, err)
|
||||
continue
|
||||
}
|
||||
delete(h.applied, as)
|
||||
delete(h.lost, as)
|
||||
delete(h.braked, as)
|
||||
}
|
||||
// Retire what the mesh has stably stopped asking for — within the bound, or with a person.
|
||||
h.retireUnasked(ctx, want)
|
||||
h.r.lastWant = want
|
||||
for as := range h.failing {
|
||||
if !want[as] {
|
||||
delete(h.failing, as)
|
||||
}
|
||||
}
|
||||
// A consumer the mesh stopped asking for is no longer failed by anyone: said, so a standing
|
||||
// the controller keeps for it is cleared rather than left naming a consumer that is gone. One the
|
||||
// brake holds is still kept, and its standing stays.
|
||||
// A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by
|
||||
// anyone: said, so a standing the controller keeps for it is cleared rather than left naming a
|
||||
// consumer that is gone. One still held is said recovered when it is retired.
|
||||
for as := range h.trouble {
|
||||
if _, held := h.braked[as]; !want[as] && !held {
|
||||
h.recovered(as, "withdrawn")
|
||||
if _, held := h.applied[as]; !want[as] && !held {
|
||||
h.recovered(as, "no longer asked for")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// overTheBound says a pass withdrawing n of the held consumers would withdraw more than it may.
|
||||
func (h *Harness) overTheBound(n, held int) bool {
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
return n > h.WithdrawAtOnce || (held > 1 && float64(n) > h.WithdrawFraction*float64(held))
|
||||
}
|
||||
|
||||
// brakeWithdrawal keeps every consumer a pass over its bound would withdraw, announces each as failing
|
||||
// with the class withdrawal-braked, and answers the one it releases now, if one is due.
|
||||
func (h *Harness) brakeWithdrawal(withdrawing []string) []string {
|
||||
now := h.Now()
|
||||
set := strings.Join(withdrawing, ", ")
|
||||
if set != h.brakeSaid {
|
||||
h.say("WITHDRAWAL BRAKED: this pass would withdraw %d of the %d consumer(s) this provider holds (%s), "+
|
||||
"more than %d at once or %.0f%% of them. Nothing is withdrawn; each is announced as %s (%s), and one "+
|
||||
"is let go every %s while the mesh goes on not asking for them (novox/hq ADR 0227 rule 4)",
|
||||
len(withdrawing), len(h.applied), set, h.WithdrawAtOnce, h.WithdrawFraction*100, EventFailing,
|
||||
ClassWithdrawalBraked, h.ReleaseEvery)
|
||||
h.brakeSaid = set
|
||||
}
|
||||
if len(h.braked) == 0 {
|
||||
// The release clock starts with the brake, not at the last release of an earlier one.
|
||||
h.releasedAt = now
|
||||
}
|
||||
for _, as := range withdrawing {
|
||||
if _, kept := h.braked[as]; !kept {
|
||||
h.braked[as] = now
|
||||
}
|
||||
text := fmt.Sprintf("the mesh no longer asks for it, and the pass that would withdraw it would withdraw %d "+
|
||||
"consumers at once: kept until released (%s)", len(withdrawing), set)
|
||||
h.failed(as, h.applied[as].node, ClassWithdrawalBraked, text)
|
||||
}
|
||||
if now.Sub(h.releasedAt) < h.ReleaseEvery {
|
||||
return nil
|
||||
}
|
||||
h.releasedAt = now
|
||||
release := withdrawing[0]
|
||||
h.say("%s: released by the withdrawal brake after %s; %d more kept", release,
|
||||
now.Sub(h.braked[release]).Round(time.Second), len(withdrawing)-1)
|
||||
h.recovered(release, "withdrawn")
|
||||
return []string{release}
|
||||
}
|
||||
|
||||
// failed counts one more failure in a consumer's unbroken run, and announces the run once it has
|
||||
// lasted FailingAfter — then again every SayAgainEvery while it lasts.
|
||||
func (h *Harness) failed(as, node, class, text string) {
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
package main
|
||||
|
||||
// The provisioner loop is carried, identical, by every Go provider until the Go SDK has it
|
||||
// (harness.go). Two copies drift the moment one is fixed and the other is not — and the one left
|
||||
// behind is the provider that fails a consumer without saying so (novox/hq ADR 0224). This holds
|
||||
// them to one text. Skipped where keycloak is not beside this module, as in a build of this one alone.
|
||||
// (harness.go, retirement.go, and retirement_test.go which tests it). Two copies drift the moment one
|
||||
// is fixed and the other is not — and the one left behind is the provider that fails a consumer
|
||||
// without saying so (novox/hq ADR 0224), or retires one it should not (ADR 0230). This holds them to
|
||||
// one text. Skipped where keycloak is not beside this module, as in a build of this one alone.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -14,18 +15,20 @@ import (
|
||||
)
|
||||
|
||||
func TestTheHarnessIsTheSameAsKeycloaks(t *testing.T) {
|
||||
theirs, err := os.ReadFile("../../../keycloak/cmd/keycloak-provider/harness.go")
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
t.Skip("keycloak is not beside this module")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours, err := os.ReadFile("harness.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(ours, theirs) {
|
||||
t.Fatal("harness.go differs from keycloak/cmd/keycloak-provider/harness.go: change both, identically")
|
||||
for _, file := range []string{"harness.go", "retirement.go", "retirement_test.go"} {
|
||||
theirs, err := os.ReadFile("../../../keycloak/cmd/keycloak-provider/" + file)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
t.Skip("keycloak is not beside this module")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(ours, theirs) {
|
||||
t.Fatalf("%s differs from keycloak/cmd/keycloak-provider/%s: change both, identically", file, file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,34 +9,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type recorder struct {
|
||||
created []Provision
|
||||
removed []string
|
||||
held bool
|
||||
failing error
|
||||
holdsErr error
|
||||
}
|
||||
|
||||
func (r *recorder) Create(_ context.Context, p Provision) error {
|
||||
if r.failing != nil {
|
||||
return r.failing
|
||||
}
|
||||
r.created = append(r.created, p)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Remove(_ context.Context, as string, _ map[string]any) error {
|
||||
r.removed = append(r.removed, as)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Holds(context.Context, Provision) (bool, error) {
|
||||
if r.holdsErr != nil {
|
||||
return false, r.holdsErr
|
||||
}
|
||||
return r.held, nil
|
||||
}
|
||||
|
||||
type world struct {
|
||||
t *testing.T
|
||||
dir string
|
||||
@@ -102,18 +73,18 @@ func TestAddingExtensionsAppliesTheConsumerAgain(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConsumerNoLongerAskedForIsWithdrawn(t *testing.T) {
|
||||
func TestAConsumerNoLongerAskedForIsRetiredOnceStable(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.h.Reconcile(ctx)
|
||||
w.settle() // five passes and ten minutes
|
||||
if strings.Join(w.a.removed, ",") != "b" {
|
||||
t.Fatal(w.a.removed)
|
||||
}
|
||||
// Only a file that says nobody asks withdraws everybody.
|
||||
// Only a file that says nobody asks retires the last one.
|
||||
w.give()
|
||||
w.h.Reconcile(ctx)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "b,a" {
|
||||
t.Fatal(w.a.removed)
|
||||
}
|
||||
@@ -137,7 +108,7 @@ func TestNothingReadIsNotNobodyAsking(t *testing.T) {
|
||||
}
|
||||
w.h.Reconcile(ctx)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("withdrew %v on a file it could not use", w.a.removed)
|
||||
t.Fatalf("retired %v on a file it could not use", w.a.removed)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -206,7 +177,9 @@ func TestProvisionerCreatesTheDatabaseThenItsExtensions(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := f.statements()
|
||||
if !strings.HasPrefix(sql[len(sql)-1], `CREATE EXTENSION IF NOT EXISTS "vector"`) || !has(sql, `CREATE DATABASE "mesh_ace_letta"`) {
|
||||
// The extension once the database exists, and last the active mark (novox/hq ADR 0230).
|
||||
if !strings.HasPrefix(sql[len(sql)-2], `CREATE EXTENSION IF NOT EXISTS "vector"`) || !has(sql, `CREATE DATABASE "mesh_ace_letta"`) ||
|
||||
sql[len(sql)-1] != `COMMENT ON ROLE "mesh_ace_letta" IS '{"mesh":"consumer"}'` {
|
||||
t.Fatal(strings.Join(sql, "\n"))
|
||||
}
|
||||
if strings.Join(events, ",") != "database.provisioned" {
|
||||
@@ -228,7 +201,7 @@ func TestProvisionerCreatesTheDatabaseThenItsExtensions(t *testing.T) {
|
||||
|
||||
f.reset()
|
||||
f.answer(`FROM pg_roles`, []string{"?column?"}, []string{"1"})
|
||||
if err := a.Remove(ctx, "mesh_ace_letta", nil); err != nil {
|
||||
if err := a.Retire(ctx, "mesh_ace_letta", nil, "test", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
noDrop(t, f.statements())
|
||||
|
||||
@@ -6,13 +6,16 @@ package main
|
||||
// MESH_POSTGRES_LIVE=postgres://postgres:admin@127.0.0.1:55432/postgres?sslmode=disable go test ./...
|
||||
//
|
||||
// It proves what the fakes cannot: an untrusted extension is installed by the superuser in a fresh
|
||||
// consumer database and a second pass is a no-op; the consumer then holds; a withdrawn login cannot
|
||||
// log in and its data is still there; the reader cannot write, even past a COMMIT.
|
||||
// consumer database and a second pass is a no-op; the consumer then holds; a retired login cannot
|
||||
// log in, its data is still there and the backend lists it retired with when and why; asked for again
|
||||
// it is enabled as it was; only a deletion drops it, and only it; the reader cannot write, even past a
|
||||
// COMMIT (novox/hq ADR 0230).
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLive(t *testing.T) {
|
||||
@@ -64,21 +67,97 @@ func TestLive(t *testing.T) {
|
||||
t.Fatal(r, err)
|
||||
}
|
||||
|
||||
if err := a.Remove(ctx, p.As, nil); err != nil {
|
||||
// A neighbour that must survive everything below untouched.
|
||||
other := Provision{As: "mesh_test_other", Password: "other-pw"}
|
||||
if err := a.Create(ctx, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.DeleteRetired(ctx, Retired{Consumer: other.As}) //nolint — best effort, after a retire below
|
||||
|
||||
at := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
if err := a.Retire(ctx, p.As, nil, "the mesh stopped asking for it", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, err := a.Holds(ctx, p); err != nil || ok {
|
||||
t.Fatal("a withdrawn login still logs in:", ok, err)
|
||||
t.Fatal("a retired login still logs in:", ok, err)
|
||||
}
|
||||
r, err = c.Query(ctx, p.As, "SELECT count(*) FROM kept")
|
||||
if err != nil || len(r.Rows) != 1 {
|
||||
t.Fatal("withdrawing lost the data:", err)
|
||||
if err != nil || len(r.Rows) != 1 || cell(r.Rows[0], 0) != "1" {
|
||||
t.Fatal("retiring lost the data:", r, err)
|
||||
}
|
||||
// Coming back is given the same database.
|
||||
inv, err := a.Inventory(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found *Retired
|
||||
for i := range inv.Retired {
|
||||
if inv.Retired[i].Consumer == p.As {
|
||||
found = &inv.Retired[i]
|
||||
}
|
||||
}
|
||||
if found == nil || !found.RetiredAt.Equal(at) || found.Why != "the mesh stopped asking for it" || found.SizeBytes <= 0 {
|
||||
t.Fatalf("not listed retired with when, why and size: %+v", inv)
|
||||
}
|
||||
if !listHas(inv.Active, other.As) || listHas(inv.Active, p.As) {
|
||||
t.Fatalf("%+v", inv)
|
||||
}
|
||||
// An active consumer is never deleted, whatever is asked.
|
||||
if _, err := c.DeleteRetired(ctx, Retired{Consumer: other.As}); err == nil {
|
||||
t.Fatal("deleted an active consumer")
|
||||
}
|
||||
|
||||
// Asked for again: the same database, the same rows, the mark active.
|
||||
if err := a.Create(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, _ := a.Holds(ctx, p); !ok {
|
||||
t.Fatal("not held after coming back")
|
||||
}
|
||||
if r, err := c.as(ctx, Login{Database: p.As, User: p.As, Password: p.Password}, "SELECT count(*) FROM kept"); err != nil ||
|
||||
cell(r.Rows[0], 0) != "1" {
|
||||
t.Fatal("re-enabled without its data:", err)
|
||||
}
|
||||
if inv, _ := a.Inventory(ctx); !listHas(inv.Active, p.As) {
|
||||
t.Fatalf("not active again: %+v", inv)
|
||||
}
|
||||
|
||||
// Retired again and deleted: that database and role go; the neighbour stays.
|
||||
if err := a.Retire(ctx, p.As, nil, "again", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
freed, err := a.Delete(ctx, Retired{Consumer: p.As, Kind: KindConsumer})
|
||||
if err != nil || freed <= 0 {
|
||||
t.Fatal(freed, err)
|
||||
}
|
||||
if has, _ := c.exists(ctx, "SELECT 1 FROM pg_database WHERE datname = "+Literal(p.As)); has {
|
||||
t.Fatal("the database is still there")
|
||||
}
|
||||
if has, _ := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(p.As)); has {
|
||||
t.Fatal("the role is still there")
|
||||
}
|
||||
if ok, err := a.Holds(ctx, other); err != nil || !ok {
|
||||
t.Fatal("the neighbour was touched:", ok, err)
|
||||
}
|
||||
|
||||
// A database set aside by hand is listed since its date and can be deleted, alone.
|
||||
aside, err := c.RetireDatabase(ctx, other.As, at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv, _ = a.Inventory(ctx)
|
||||
var setAside *Retired
|
||||
for i := range inv.Retired {
|
||||
if inv.Retired[i].Consumer == aside {
|
||||
setAside = &inv.Retired[i]
|
||||
}
|
||||
}
|
||||
if setAside == nil || setAside.Kind != KindSetAside || setAside.RetiredAt.Format("20060102") != "20261006" {
|
||||
t.Fatalf("%+v", inv.Retired)
|
||||
}
|
||||
if _, err := a.Delete(ctx, *setAside); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if has, _ := c.exists(ctx, "SELECT 1 FROM pg_database WHERE datname = "+Literal(aside)); has {
|
||||
t.Fatal("the set-aside database is still there")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,10 +32,11 @@ func main() {
|
||||
}
|
||||
return
|
||||
}
|
||||
var h *Harness
|
||||
if receives := os.Getenv("MESH_RECEIVES"); receives == "" {
|
||||
say("MESH_RECEIVES is not set — the provisioner cannot run without it")
|
||||
} else {
|
||||
h := &Harness{
|
||||
h = &Harness{
|
||||
Resource: "postgres-database",
|
||||
Receives: receives,
|
||||
Adapter: provisioner{pg: pg, announce: announce},
|
||||
@@ -52,7 +53,9 @@ func main() {
|
||||
go h.Run(context.Background())
|
||||
}
|
||||
go audit()
|
||||
if err := stdio.Serve("", Tools(pg)); err != nil {
|
||||
// The retirement tools beside postgres's own: what is retired here, approving or rejecting what waits
|
||||
// for a person, and deleting a retired consumer (novox/hq ADR 0230).
|
||||
if err := stdio.Serve("", append(Tools(pg), RetirementTools(h)...)); err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -79,7 +82,7 @@ func audit() {
|
||||
case "postgres.database.provisioned":
|
||||
say("database provisioned for %s (db %s)", body.Consumer, body.Database)
|
||||
case "postgres.database.deprovisioned":
|
||||
say("database withdrawn, kept (db %s)", body.Database)
|
||||
say("database retired, kept (db %s)", body.Database)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -14,6 +14,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// provisioner is the adapter over the client; announce emits a lifecycle event.
|
||||
@@ -36,22 +37,36 @@ func (a provisioner) Create(ctx context.Context, p Provision) error {
|
||||
if err := a.pg.EnsureExtensions(ctx, database, extensions); err != nil {
|
||||
return err
|
||||
}
|
||||
// The active mark: a retired consumer asked for again loses its mark to delete here, its LOGIN
|
||||
// already set again by the role statement above (novox/hq ADR 0230).
|
||||
if err := a.pg.MarkActive(ctx, p.As, p.Consumer); err != nil {
|
||||
return err
|
||||
}
|
||||
a.announce("database.provisioned", map[string]string{"consumer": p.Consumer, "database": database, "user": p.As})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove withdraws, never drops (novox/hq issue 241). The login is locked and the database kept under
|
||||
// its own name: on 2026-10-04 a misread contributions file withdrew every consumer at once, and
|
||||
// dropping made that a loss of seven databases. Taking a database out of service is a person's act —
|
||||
// postgres_retire_database — and even that renames rather than drops.
|
||||
func (a provisioner) Remove(ctx context.Context, as string, _ map[string]any) error {
|
||||
if err := a.pg.LockRole(ctx, as); err != nil {
|
||||
// Retire locks the login, never drops (novox/hq issue 241, ADR 0230): the database is kept under its
|
||||
// own name, the role marked retired with when and why (retire_pg.go). On 2026-10-04 a misread
|
||||
// contributions file withdrew every consumer at once, and dropping made that a loss of seven databases.
|
||||
// Deleting is `cleanup delete`, a person's act; taking a database out of service by hand is
|
||||
// postgres_retire_database, which renames rather than drops.
|
||||
func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, why string, at time.Time) error {
|
||||
if err := a.pg.RetireRole(ctx, as, why, at); err != nil {
|
||||
return err
|
||||
}
|
||||
a.announce("database.deprovisioned", map[string]string{"database": as, "kept": "true"})
|
||||
a.announce("database.deprovisioned", map[string]string{"database": as, "kept": "true", "retired": "true"})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Inventory is what this server holds that the mesh made (retire_pg.go).
|
||||
func (a provisioner) Inventory(ctx context.Context) (Inventory, error) { return a.pg.Inventory(ctx) }
|
||||
|
||||
// Delete drops a retired consumer's database and role, or a database set aside — a person's act.
|
||||
func (a provisioner) Delete(ctx context.Context, r Retired) (int64, error) {
|
||||
return a.pg.DeleteRetired(ctx, r)
|
||||
}
|
||||
|
||||
// Holds is asked every minute: whether the consumer can still log in as the mesh gave it, and finds
|
||||
// the extensions it asked for, so a login or extension lost behind the provisioner's back is made
|
||||
// again (novox/hq issue 120).
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
package main
|
||||
|
||||
// What retired means in postgres (novox/hq ADR 0230).
|
||||
//
|
||||
// **Retired is the login locked, the database kept, and the role marked.** `ALTER ROLE … NOLOGIN` —
|
||||
// the consumer's login can no longer connect, as itself, to anything — and its open sessions are
|
||||
// ended. The database, its owner, its grants and every byte in it stay exactly as they were: CONNECT is
|
||||
// not revoked and the database still allows connections, because the backup contribution dumps every
|
||||
// database and a retired one is still worth backing up, and because re-enabling must give it back as it
|
||||
// was. The mark is the role's comment, a JSON object the mesh owns:
|
||||
//
|
||||
// {"mesh":"consumer","node":"ace"} active
|
||||
// {"mesh":"consumer","node":"ace","retired":"2026-10-06T…Z","why":"…"} retired
|
||||
//
|
||||
// Asked for again, the ordinary create sets LOGIN and the password again and writes the active mark.
|
||||
// Deleted — only through `cleanup delete` — the database is dropped, then the role, unless it still owns
|
||||
// another database (a set-aside copy), which is said and kept.
|
||||
//
|
||||
// **What the mesh made is recognised by its mark, or by its shape before the mark existed**: a role
|
||||
// valid until 'infinity' (only the mesh's role statement says that) that owns a database of its own
|
||||
// name. A role of any other shape is somebody else's and is never listed, retired or deleted. A database
|
||||
// renamed aside — `<name>_deleted_<yyyymmdd>`, by postgres_retire_database or by hand — is listed as
|
||||
// retired too, since that date, so a person sees it and can delete it.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// KindSetAside is a database renamed aside, listed for deletion beside the retired consumers.
|
||||
const KindSetAside = "set-aside-database"
|
||||
|
||||
// roleMark is the comment the mesh keeps on a consumer's role.
|
||||
type roleMark struct {
|
||||
Mesh string `json:"mesh"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Retired string `json:"retired,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
func readMark(comment string) (roleMark, bool) {
|
||||
var m roleMark
|
||||
if comment == "" || json.Unmarshal([]byte(comment), &m) != nil || m.Mesh != KindConsumer {
|
||||
return roleMark{}, false
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
|
||||
// MarkStatement is the comment that marks a role as the mesh's consumer, active or retired.
|
||||
func MarkStatement(role string, m roleMark) string {
|
||||
m.Mesh = KindConsumer
|
||||
b, _ := json.Marshal(m)
|
||||
return fmt.Sprintf("COMMENT ON ROLE %s IS %s", Ident(role), Literal(string(b)))
|
||||
}
|
||||
|
||||
// MarkActive writes the active mark — after create, which has already set LOGIN.
|
||||
func (c *Client) MarkActive(ctx context.Context, role, node string) error {
|
||||
_, err := c.Query(ctx, "", MarkStatement(role, roleMark{Node: node}))
|
||||
return err
|
||||
}
|
||||
|
||||
// RetireRole locks the login, ends its sessions and marks it retired with when and why. Its database
|
||||
// is not touched. A role that does not exist has nothing to retire.
|
||||
func (c *Client) RetireRole(ctx context.Context, role, why string, at time.Time) error {
|
||||
r, err := c.Query(ctx, "", "SELECT coalesce(shobj_description(oid, 'pg_authid'), '') FROM pg_roles WHERE rolname = "+
|
||||
Literal(role))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(r.Rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
prev, _ := readMark(cell(r.Rows[0], 0))
|
||||
if err := c.LockRole(ctx, role); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = c.Query(ctx, "", MarkStatement(role, roleMark{Node: prev.Node, Retired: at.UTC().Format(time.RFC3339), Why: why}))
|
||||
return err
|
||||
}
|
||||
|
||||
var setAside = regexp.MustCompile(`_deleted_(\d{8})$`)
|
||||
|
||||
// InventoryStatement lists every role that may be the mesh's, with its login, mark, shape and the size
|
||||
// of its own database.
|
||||
const InventoryStatement = `SELECT r.rolname, r.rolcanlogin, coalesce(shobj_description(r.oid, 'pg_authid'), '') AS mark,
|
||||
coalesce(r.rolvaliduntil = 'infinity', false) AS mesh_shaped,
|
||||
(SELECT pg_database_size(d.datname) FROM pg_database d WHERE d.datname = r.rolname AND d.datdba = r.oid) AS size
|
||||
FROM pg_roles r
|
||||
WHERE r.rolname !~ '^pg_' AND NOT r.rolsuper AND r.rolname <> ` + "'" + Reader + "'" + `
|
||||
ORDER BY r.rolname`
|
||||
|
||||
// SetAsideStatement lists the databases renamed aside.
|
||||
const SetAsideStatement = `SELECT datname, pg_database_size(datname) FROM pg_database WHERE datname ~ '_deleted_[0-9]{8}$' ORDER BY datname`
|
||||
|
||||
// Inventory is what this server holds that the mesh made.
|
||||
func (c *Client) Inventory(ctx context.Context) (Inventory, error) {
|
||||
inv := Inventory{Active: []string{}, Retired: []Retired{}}
|
||||
r, err := c.Query(ctx, "", InventoryStatement)
|
||||
if err != nil {
|
||||
return inv, err
|
||||
}
|
||||
for _, row := range r.Rows {
|
||||
name, login, comment, shaped, size := cell(row, 0), cell(row, 1) == "t", cell(row, 2), cell(row, 3) == "t", cell(row, 4)
|
||||
m, marked := readMark(comment)
|
||||
if !marked && !(shaped && size != "") {
|
||||
continue // not the mesh's
|
||||
}
|
||||
if login && m.Retired == "" {
|
||||
inv.Active = append(inv.Active, name)
|
||||
continue
|
||||
}
|
||||
at, _ := time.Parse(time.RFC3339, m.Retired)
|
||||
inv.Retired = append(inv.Retired, Retired{Consumer: name, Node: m.Node, RetiredAt: at, Why: m.Why,
|
||||
SizeBytes: sizeOf(size), Kind: KindConsumer})
|
||||
}
|
||||
r, err = c.Query(ctx, "", SetAsideStatement)
|
||||
if err != nil {
|
||||
return inv, err
|
||||
}
|
||||
for _, row := range r.Rows {
|
||||
name := cell(row, 0)
|
||||
m := setAside.FindStringSubmatch(name)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
at, _ := time.Parse("20060102", m[1])
|
||||
inv.Retired = append(inv.Retired, Retired{Consumer: name, RetiredAt: at, SizeBytes: sizeOf(cell(row, 1)),
|
||||
Why: "renamed aside — by postgres_retire_database, or by hand", Kind: KindSetAside})
|
||||
}
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
// DeleteRetired drops what a retired entry names: a consumer's database and then its role, or one
|
||||
// database set aside. Answers the bytes freed.
|
||||
func (c *Client) DeleteRetired(ctx context.Context, r Retired) (int64, error) {
|
||||
if r.Kind == KindSetAside {
|
||||
if !setAside.MatchString(r.Consumer) {
|
||||
return 0, fmt.Errorf("%s is not a database set aside", r.Consumer)
|
||||
}
|
||||
return c.dropDatabase(ctx, r.Consumer)
|
||||
}
|
||||
// Only a retired one: the login must be locked here and now, whatever the caller believed.
|
||||
row, err := c.Query(ctx, "", "SELECT rolcanlogin FROM pg_roles WHERE rolname = "+Literal(r.Consumer))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(row.Rows) > 0 && cell(row.Rows[0], 0) == "t" {
|
||||
return 0, fmt.Errorf("%s can log in — it is active, not retired, and is not deleted", r.Consumer)
|
||||
}
|
||||
freed, err := c.dropDatabase(ctx, r.Consumer)
|
||||
if err != nil {
|
||||
return freed, err
|
||||
}
|
||||
if len(row.Rows) == 0 {
|
||||
return freed, nil
|
||||
}
|
||||
owns, err := c.Query(ctx, "", "SELECT datname FROM pg_database WHERE datdba = (SELECT oid FROM pg_roles WHERE rolname = "+
|
||||
Literal(r.Consumer)+")")
|
||||
if err != nil {
|
||||
return freed, err
|
||||
}
|
||||
if len(owns.Rows) > 0 {
|
||||
return freed, fmt.Errorf("%s's database is dropped; the role is kept because it still owns %s — delete that first",
|
||||
r.Consumer, cell(owns.Rows[0], 0))
|
||||
}
|
||||
_, err = c.Query(ctx, "", fmt.Sprintf("DROP ROLE %s", Ident(r.Consumer)))
|
||||
return freed, err
|
||||
}
|
||||
|
||||
func (c *Client) dropDatabase(ctx context.Context, database string) (int64, error) {
|
||||
r, err := c.Query(ctx, "", "SELECT pg_database_size(datname) FROM pg_database WHERE datname = "+Literal(database))
|
||||
if err != nil || len(r.Rows) == 0 {
|
||||
return 0, err
|
||||
}
|
||||
freed := sizeOf(cell(r.Rows[0], 0))
|
||||
if _, err := c.Query(ctx, "", "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = "+
|
||||
Literal(database)+" AND pid <> pg_backend_pid()"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := c.Query(ctx, "", fmt.Sprintf("DROP DATABASE %s", Ident(database))); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return freed, nil
|
||||
}
|
||||
|
||||
func cell(row []*string, i int) string {
|
||||
if i < len(row) && row[i] != nil {
|
||||
return *row[i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sizeOf(s string) int64 {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
// What retired means in postgres, held against the statements sent (retire_pg.go); the server's side
|
||||
// of it — the login refused, the data kept, a deletion dropping only its own — is live_test.go's.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRetiringLocksMarksAndDropsNothing(t *testing.T) {
|
||||
f, c := newFake(admin)
|
||||
f.answer(`shobj_description\(oid`, []string{"c"}, []string{`{"mesh":"consumer","node":"ace"}`})
|
||||
f.answer(`SELECT 1 FROM pg_roles`, []string{"?column?"}, []string{"1"})
|
||||
at := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
||||
if err := c.RetireRole(ctx, "mesh_ace_letta", "the mesh stopped asking for it", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sql := f.statements()
|
||||
if !has(sql, `ALTER ROLE "mesh_ace_letta" NOLOGIN`) || !has(sql, `pg_terminate_backend.*usename = 'mesh_ace_letta'`) ||
|
||||
!has(sql, `COMMENT ON ROLE "mesh_ace_letta" IS '\{"mesh":"consumer","node":"ace","retired":"2026-10-06T12:00:00Z","why":"the mesh stopped asking for it"\}'`) {
|
||||
t.Fatal(strings.Join(sql, "\n"))
|
||||
}
|
||||
noDrop(t, sql)
|
||||
if has(sql, `REVOKE|ALLOW_CONNECTIONS|RENAME`) {
|
||||
t.Fatal("retiring touched the database:", strings.Join(sql, "\n"))
|
||||
}
|
||||
|
||||
// No such role: nothing to retire, nothing done.
|
||||
f, c = newFake(admin)
|
||||
if err := c.RetireRole(ctx, "gone", "x", at); err != nil || has(f.statements(), `ALTER|COMMENT`) {
|
||||
t.Fatal(f.statements(), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) {
|
||||
f, c := newFake(admin)
|
||||
f.answer(`FROM pg_roles r`, []string{"rolname", "rolcanlogin", "mark", "mesh_shaped", "size"},
|
||||
[]string{"active_marked", "t", `{"mesh":"consumer","node":"ace"}`, "f", "100"},
|
||||
[]string{"active_shaped", "t", "", "t", "200"},
|
||||
[]string{"retired_marked", "f", `{"mesh":"consumer","node":"ace","retired":"2026-10-06T12:00:00Z","why":"gone"}`, "t", "300"},
|
||||
[]string{"locked_before", "f", "", "t", "400"},
|
||||
[]string{"hal_registry", "t", "", "f", "500"},
|
||||
[]string{"jochens", "t", "", "t", ""},
|
||||
)
|
||||
f.answer(`_deleted_`, []string{"datname", "size"}, []string{"mesh_novox_gitea_deleted_20261005", "7771827"})
|
||||
inv, err := c.Inventory(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(inv.Active, ",") != "active_marked,active_shaped" {
|
||||
t.Fatalf("active: %v", inv.Active)
|
||||
}
|
||||
if len(inv.Retired) != 3 {
|
||||
t.Fatalf("%+v", inv.Retired)
|
||||
}
|
||||
r := inv.Retired[0]
|
||||
if r.Consumer != "retired_marked" || r.Node != "ace" || r.Why != "gone" || r.SizeBytes != 300 ||
|
||||
r.RetiredAt.Format(time.RFC3339) != "2026-10-06T12:00:00Z" {
|
||||
t.Fatalf("%+v", r)
|
||||
}
|
||||
if r := inv.Retired[1]; r.Consumer != "locked_before" || !r.RetiredAt.IsZero() || r.Kind != KindConsumer {
|
||||
t.Fatalf("found locked without a mark: %+v", r)
|
||||
}
|
||||
if r := inv.Retired[2]; r.Kind != KindSetAside || r.RetiredAt.Format("20060102") != "20261005" || r.SizeBytes != 7771827 {
|
||||
t.Fatalf("set aside: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletingRefusesALoginThatCanConnect(t *testing.T) {
|
||||
f, c := newFake(admin)
|
||||
f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"t"})
|
||||
if _, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_baserow", Kind: KindConsumer}); err == nil {
|
||||
t.Fatal("deleted an active consumer")
|
||||
}
|
||||
noDrop(t, f.statements())
|
||||
|
||||
f, c = newFake(admin)
|
||||
if _, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_baserow", Kind: KindSetAside}); err == nil {
|
||||
t.Fatal("dropped a database not set aside")
|
||||
}
|
||||
noDrop(t, f.statements())
|
||||
}
|
||||
|
||||
func TestDeletingDropsTheDatabaseThenTheRole(t *testing.T) {
|
||||
f, c := newFake(admin)
|
||||
f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"f"})
|
||||
f.answer(`SELECT pg_database_size`, []string{"size"}, []string{"9000"})
|
||||
freed, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_letta", Kind: KindConsumer})
|
||||
if err != nil || freed != 9000 {
|
||||
t.Fatal(freed, err)
|
||||
}
|
||||
sql := strings.Join(f.statements(), "\n")
|
||||
db, role := strings.Index(sql, `DROP DATABASE "mesh_ace_letta"`), strings.Index(sql, `DROP ROLE "mesh_ace_letta"`)
|
||||
if db < 0 || role < db {
|
||||
t.Fatal(sql)
|
||||
}
|
||||
// A role still owning a set-aside database is kept, and said.
|
||||
f, c = newFake(admin)
|
||||
f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"f"})
|
||||
f.answer(`SELECT datname FROM pg_database WHERE datdba`, []string{"datname"}, []string{"mesh_ace_letta_deleted_20261005"})
|
||||
if _, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_letta", Kind: KindConsumer}); err == nil ||
|
||||
!strings.Contains(err.Error(), "still owns") || has(f.statements(), `DROP ROLE`) {
|
||||
t.Fatal(err, f.statements())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,603 @@
|
||||
package main
|
||||
|
||||
// What becomes of a consumer the mesh no longer asks for (novox/hq ADR 0230 — the operator's model,
|
||||
// decided 2026-10-06; it replaces ADR 0229's withdrawal brake, which let one consumer go every hour).
|
||||
//
|
||||
// A consumer — a login, a client, a key, and the data behind it — is in one of three states:
|
||||
//
|
||||
// 1. ACTIVE.
|
||||
// 2. RETIRED. The mesh stopped asking for it: its access is disabled, reversibly, and its login and
|
||||
// data are marked "to delete" with when and why. Nothing is deleted. Asked for again, the ordinary
|
||||
// create re-enables it at once, as it was.
|
||||
// 3. DELETED, only through `cleanup delete`, a person's act, which this provider executes because it
|
||||
// owns its backend.
|
||||
//
|
||||
// **Stable removals.** A consumer is retired only once the same set of consumers has gone unasked BOTH
|
||||
// in StablePasses (5) consecutive passes that read the file AND for at least StableFor (10 minutes)
|
||||
// since the first pass that saw it. Five passes alone are twenty-five seconds — shorter than a
|
||||
// controller restart, a store reconnecting or a file half written. A pass that could not read the file
|
||||
// is not a result and starts both again; so does a different set. Additions and changes are never
|
||||
// delayed.
|
||||
//
|
||||
// **Too many is a person.** A stable set of more than RetireAtOnce (3), or — where more than one is
|
||||
// held — of more than RetireFraction (half) of those held, retires nothing: it WAITS, said in the
|
||||
// journal and announced `provisioner.retirement` `waiting` (again every SayAgainEvery), which the
|
||||
// controller raises as an urgent condition, until `retire approve <node> <module>` or `retire reject`.
|
||||
// An unassignment the controller made itself is no exception: many changes at once means a person is
|
||||
// at work, and a person confirms once. On 2026-10-04 one misread file withdrew seven consumers at once
|
||||
// (issue 241); under this rule that is a question, not an act.
|
||||
//
|
||||
// **The backend remembers, not this process.** What is retired, since when and why is read from the
|
||||
// backend's own mark (Adapter.Inventory), so a restart forgets nothing; a consumer the backend holds
|
||||
// active that the mesh no longer asks for is retired by the same rules after a restart as before one.
|
||||
// A consumer found disabled without the mark — withdrawn before this record — is ADOPTED as retired:
|
||||
// marked, said, announced `adopted`, and its clock starts then.
|
||||
//
|
||||
// **A rejection lives as long as this process.** Rejected, the set is kept active and not asked about
|
||||
// again while it stays the same set; a restart asks again — loudly, never silently.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// EventRetirement is the one event a provider says about retirement, its `change` saying what
|
||||
// happened. The controller derives the permission to emit it for every module that receives
|
||||
// contributions, as it does the standing events (novox/hq ADR 0224, 0230).
|
||||
const EventRetirement = "provisioner.retirement"
|
||||
|
||||
// The changes EventRetirement carries.
|
||||
const (
|
||||
ChangeWaiting = "waiting" // a stable set over the bound waits for a person
|
||||
ChangeSettled = "settled" // a waiting or rejected set ended without being retired
|
||||
ChangeApproved = "approved" // a person approved the waiting (or rejected) set
|
||||
ChangeRejected = "rejected" // a person kept the waiting set active
|
||||
ChangeRetired = "retired" // consumers disabled and marked to delete
|
||||
ChangeReenabled = "reenabled" // a retired consumer asked for again, enabled as it was
|
||||
ChangeDeleted = "deleted" // a retired consumer deleted, by a person
|
||||
ChangeAdopted = "adopted" // found disabled without the mark, now retired on record
|
||||
)
|
||||
|
||||
// StableFor is the least time the same unasked set must hold before it is retired (novox/hq ADR 0230):
|
||||
// longer than a controller restart, a store reconnecting or a file half written.
|
||||
const StableFor = 10 * time.Minute
|
||||
|
||||
// KindConsumer is a retired consumer. A backend may list other things set aside for deletion under a
|
||||
// word of its own (postgres: a database renamed aside).
|
||||
const KindConsumer = "consumer"
|
||||
|
||||
// Retired is one thing a provider holds retired, as its backend's mark says.
|
||||
type Retired struct {
|
||||
Consumer string
|
||||
// Node is the consumer's machine, where the mark records it.
|
||||
Node string
|
||||
// RetiredAt is when it was retired; zero for one found disabled without the mesh's mark.
|
||||
RetiredAt time.Time
|
||||
Why string
|
||||
// SizeBytes is what deleting it would free; -1 when the backend cannot say.
|
||||
SizeBytes int64
|
||||
Kind string
|
||||
}
|
||||
|
||||
// Inventory is what a backend holds that the mesh made.
|
||||
type Inventory struct {
|
||||
Active []string
|
||||
Retired []Retired
|
||||
}
|
||||
|
||||
// retirement is the loop's memory of the sets it is counting, waiting on and was told to keep.
|
||||
type retirement struct {
|
||||
key string // the unasked set, joined
|
||||
count int // consecutive passes that saw it
|
||||
firstSeen time.Time
|
||||
waiting *waitingSet
|
||||
rejected *rejectedSet
|
||||
retired map[string]retiredEntry
|
||||
lastWant map[string]bool
|
||||
seeded bool
|
||||
seedSaid string
|
||||
}
|
||||
|
||||
type waitingSet struct {
|
||||
set []string
|
||||
since time.Time
|
||||
saidAt time.Time
|
||||
held int
|
||||
}
|
||||
|
||||
type rejectedSet struct {
|
||||
set []string
|
||||
by, why string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
type retiredEntry struct {
|
||||
node string
|
||||
at time.Time
|
||||
why string
|
||||
}
|
||||
|
||||
// seed reads what the backend holds, once per process: an active consumer the mesh no longer asks
|
||||
// for is held, so it is retired by the same rules as one this process made; one found disabled
|
||||
// without the mark is adopted as retired.
|
||||
func (h *Harness) seed(ctx context.Context, want map[string]bool) {
|
||||
if h.r.seeded {
|
||||
return
|
||||
}
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
if said := err.Error(); said != h.r.seedSaid {
|
||||
h.say("cannot list what the backend holds (%v); a consumer the mesh stopped asking for while this "+
|
||||
"provider was down is not retired until it can", err)
|
||||
h.r.seedSaid = said
|
||||
}
|
||||
return
|
||||
}
|
||||
h.r.seeded = true
|
||||
for _, as := range inv.Active {
|
||||
if _, held := h.applied[as]; !held && !want[as] {
|
||||
// No hash: asked for again, it is applied again, which is harmless and marks it.
|
||||
h.applied[as] = appliedEntry{}
|
||||
h.say("%s: held by the backend and no longer asked for; retired once that holds for %d passes "+
|
||||
"and %s (novox/hq ADR 0230)", as, h.StablePasses, h.StableFor)
|
||||
}
|
||||
}
|
||||
now := h.Now()
|
||||
for _, r := range inv.Retired {
|
||||
if r.Kind != "" && r.Kind != KindConsumer {
|
||||
continue
|
||||
}
|
||||
if !r.RetiredAt.IsZero() {
|
||||
h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: r.RetiredAt, why: r.Why}
|
||||
continue
|
||||
}
|
||||
if want[r.Consumer] {
|
||||
continue // asked for: this pass's create enables it
|
||||
}
|
||||
why := "found disabled without the mesh's mark — withdrawn before retirement was recorded " +
|
||||
"(novox/hq ADR 0230); retired as found"
|
||||
if err := h.Adapter.Retire(ctx, r.Consumer, nil, why, now); err != nil {
|
||||
h.say("%s: found disabled and could not be marked retired, will try at the next start: %v", r.Consumer, err)
|
||||
continue
|
||||
}
|
||||
h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: now, why: why}
|
||||
h.say("%s: ADOPTED as retired — %s", r.Consumer, why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeAdopted, []map[string]any{
|
||||
{"consumer": r.Consumer, "node": r.Node, "retired_at": stamp(now), "why": why, "size_bytes": r.SizeBytes},
|
||||
}, map[string]any{"why": why}))
|
||||
}
|
||||
}
|
||||
|
||||
// retireUnasked counts the passes that saw the same set no longer asked for and, once it is stable,
|
||||
// retires it — or, past the bound, waits for a person.
|
||||
func (h *Harness) retireUnasked(ctx context.Context, want map[string]bool) {
|
||||
var unasked []string
|
||||
for as := range h.applied {
|
||||
if !want[as] {
|
||||
unasked = append(unasked, as)
|
||||
}
|
||||
}
|
||||
sort.Strings(unasked)
|
||||
key := strings.Join(unasked, "\x00")
|
||||
now := h.Now()
|
||||
switch {
|
||||
case len(unasked) == 0:
|
||||
h.settle("every consumer held is asked for again")
|
||||
h.r.key, h.r.count = "", 0
|
||||
return
|
||||
case key != h.r.key:
|
||||
if h.r.waiting != nil || h.r.rejected != nil {
|
||||
h.settle("the set the mesh no longer asks for changed")
|
||||
}
|
||||
h.r.key, h.r.count, h.r.firstSeen = key, 1, now
|
||||
h.say("no longer asked for: %s; retired once the same set holds for %d passes and %s",
|
||||
strings.Join(unasked, ", "), h.StablePasses, h.StableFor)
|
||||
default:
|
||||
h.r.count++
|
||||
}
|
||||
if h.r.rejected != nil || h.r.count < h.StablePasses || now.Sub(h.r.firstSeen) < h.StableFor {
|
||||
return
|
||||
}
|
||||
if h.overTheBound(len(unasked), len(h.applied)) {
|
||||
h.wait(unasked)
|
||||
return
|
||||
}
|
||||
why := fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s, from %s",
|
||||
h.r.count, now.Sub(h.r.firstSeen).Round(time.Second), stamp(h.r.firstSeen))
|
||||
h.retire(ctx, unasked, why, nil)
|
||||
}
|
||||
|
||||
// overTheBound says retiring n of the held consumers at once needs a person.
|
||||
func (h *Harness) overTheBound(n, held int) bool {
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
return n > h.RetireAtOnce || (held > 1 && float64(n) > h.RetireFraction*float64(held))
|
||||
}
|
||||
|
||||
func (h *Harness) bound(held int) string {
|
||||
return fmt.Sprintf("more than %d at once, or more than %.0f%% of the %d held", h.RetireAtOnce,
|
||||
h.RetireFraction*100, held)
|
||||
}
|
||||
|
||||
// wait holds a stable set over the bound for a person, said once and announced again every
|
||||
// SayAgainEvery so a controller that missed the first hears the next.
|
||||
func (h *Harness) wait(set []string) {
|
||||
now := h.Now()
|
||||
w := h.r.waiting
|
||||
if w == nil {
|
||||
w = &waitingSet{set: set, since: now, held: len(h.applied)}
|
||||
h.r.waiting = w
|
||||
h.say("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d consumer(s) this provider "+
|
||||
"holds (%s), %s. Nothing is retired; each stays active until `retire approve %s <module>` or "+
|
||||
"`retire reject` (novox/hq ADR 0230)", len(set), w.held, strings.Join(set, ", "), h.bound(w.held), h.Node)
|
||||
} else if now.Sub(w.saidAt) < h.SayAgainEvery {
|
||||
return
|
||||
}
|
||||
w.saidAt = now
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeWaiting, h.named(set), map[string]any{
|
||||
"held": w.held, "bound": h.bound(w.held), "since": stamp(w.since),
|
||||
}))
|
||||
}
|
||||
|
||||
// settle ends a waiting or rejected set that the mesh's own answer made moot.
|
||||
func (h *Harness) settle(why string) {
|
||||
var set []string
|
||||
switch {
|
||||
case h.r.waiting != nil:
|
||||
set = h.r.waiting.set
|
||||
case h.r.rejected != nil:
|
||||
set = h.r.rejected.set
|
||||
default:
|
||||
return
|
||||
}
|
||||
h.r.waiting, h.r.rejected = nil, nil
|
||||
h.say("retirement of %s settled without retiring: %s", strings.Join(set, ", "), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeSettled, h.named(set), map[string]any{"why": why}))
|
||||
}
|
||||
|
||||
// retire disables and marks each consumer of set; one that fails stays held and is tried again once
|
||||
// its set is stable again.
|
||||
func (h *Harness) retire(ctx context.Context, set []string, why string, extra map[string]any) []string {
|
||||
now := h.Now()
|
||||
held := len(h.applied)
|
||||
var done []string
|
||||
var said []map[string]any
|
||||
for _, as := range set {
|
||||
was, ok := h.applied[as]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := h.Adapter.Retire(ctx, as, was.derived, why, now); err != nil {
|
||||
h.say("%s: retiring failed, will try again: %v", as, err)
|
||||
continue
|
||||
}
|
||||
delete(h.applied, as)
|
||||
delete(h.lost, as)
|
||||
delete(h.failing, as)
|
||||
h.r.retired[as] = retiredEntry{node: was.node, at: now, why: why}
|
||||
h.recovered(as, "retired")
|
||||
h.say("%s: RETIRED — its access disabled and its data kept, marked to delete (%s). Asked for again "+
|
||||
"it is re-enabled as it was; it is deleted only by `cleanup delete` (novox/hq ADR 0230)", as, why)
|
||||
done = append(done, as)
|
||||
said = append(said, map[string]any{"consumer": as, "node": was.node, "retired_at": stamp(now), "why": why})
|
||||
}
|
||||
h.r.key, h.r.count, h.r.waiting, h.r.rejected = "", 0, nil, nil
|
||||
if len(done) > 0 {
|
||||
body := map[string]any{"held": held, "why": why}
|
||||
for k, v := range extra {
|
||||
body[k] = v
|
||||
}
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeRetired, said, body))
|
||||
}
|
||||
return done
|
||||
}
|
||||
|
||||
// reenabled says a retired consumer was asked for again and its create enabled it.
|
||||
func (h *Harness) reenabled(as, node string) {
|
||||
e, was := h.r.retired[as]
|
||||
if !was {
|
||||
return
|
||||
}
|
||||
delete(h.r.retired, as)
|
||||
h.say("%s: asked for again — re-enabled as it was, its mark to delete cleared (retired %s: %s)", as,
|
||||
stamp(e.at), e.why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeReenabled, []map[string]any{
|
||||
{"consumer": as, "node": node, "retired_at": stamp(e.at), "why": e.why},
|
||||
}, nil))
|
||||
}
|
||||
|
||||
// Approve retires exactly the set waiting (or the set rejected) — a person's confirmation.
|
||||
func (h *Harness) Approve(ctx context.Context, consumers []string, why, by, via string) ([]string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("approve: say why")
|
||||
}
|
||||
set := sorted(consumers)
|
||||
var held []string
|
||||
switch {
|
||||
case h.r.waiting != nil && same(set, h.r.waiting.set):
|
||||
held = h.r.waiting.set
|
||||
case h.r.rejected != nil && same(set, h.r.rejected.set):
|
||||
held = h.r.rejected.set
|
||||
default:
|
||||
return nil, fmt.Errorf("approve: %s is not the set waiting here — %s", orNone(set), h.waitingWords())
|
||||
}
|
||||
h.say("retirement of %s APPROVED by %s: %s", strings.Join(held, ", "), orSomebody(by), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeApproved, h.named(held),
|
||||
map[string]any{"why": why, "by": by, "via": via}))
|
||||
reason := fmt.Sprintf("the mesh stopped asking for it; approved by %s: %s", orSomebody(by), why)
|
||||
return h.retire(ctx, held, reason, map[string]any{"by": by, "via": via}), nil
|
||||
}
|
||||
|
||||
// Reject keeps the waiting set active; it is not asked about again while it stays the same set.
|
||||
func (h *Harness) Reject(consumers []string, why, by, via string) ([]string, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("reject: say why")
|
||||
}
|
||||
set := sorted(consumers)
|
||||
if h.r.waiting == nil || !same(set, h.r.waiting.set) {
|
||||
return nil, fmt.Errorf("reject: %s is not the set waiting here — %s", orNone(set), h.waitingWords())
|
||||
}
|
||||
h.r.rejected = &rejectedSet{set: h.r.waiting.set, by: by, why: why, at: h.Now()}
|
||||
h.r.waiting = nil
|
||||
h.say("retirement of %s REJECTED by %s: %s. Kept active, and not asked about again while the mesh goes on "+
|
||||
"not asking for exactly these (until this provider restarts)", strings.Join(set, ", "), orSomebody(by), why)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeRejected, h.named(set),
|
||||
map[string]any{"why": why, "by": by, "via": via}))
|
||||
return set, nil
|
||||
}
|
||||
|
||||
// DeleteRetired deletes one retired consumer, by a person's word: never an active one, never one the
|
||||
// mesh asks for.
|
||||
func (h *Harness) DeleteRetired(ctx context.Context, consumer, why, by, via string) (int64, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return 0, errors.New("delete: say why")
|
||||
}
|
||||
if h.r.lastWant[consumer] {
|
||||
return 0, fmt.Errorf("delete: the mesh asks for %s — it is not retired", consumer)
|
||||
}
|
||||
if _, held := h.applied[consumer]; held {
|
||||
return 0, fmt.Errorf("delete: %s is active here — only a retired consumer is deleted", consumer)
|
||||
}
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete: what the backend holds cannot be read, so nothing is deleted: %w", err)
|
||||
}
|
||||
var found *Retired
|
||||
for i := range inv.Retired {
|
||||
if inv.Retired[i].Consumer == consumer {
|
||||
found = &inv.Retired[i]
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
return 0, fmt.Errorf("delete: %s is not retired here — only a retired consumer is deleted", consumer)
|
||||
}
|
||||
freed, err := h.Adapter.Delete(ctx, *found)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
delete(h.r.retired, consumer)
|
||||
h.say("%s: DELETED by %s: %s (retired %s: %s; %d bytes freed)", consumer, orSomebody(by), why,
|
||||
stampOr(found.RetiredAt), found.Why, freed)
|
||||
h.announce(EventRetirement, h.retirementBody(ChangeDeleted, []map[string]any{{
|
||||
"consumer": consumer, "node": found.Node, "retired_at": stampOr(found.RetiredAt), "why": found.Why,
|
||||
"size_bytes": found.SizeBytes, "kind": kindOf(*found),
|
||||
}}, map[string]any{"why": why, "by": by, "via": via, "freed_bytes": freed}))
|
||||
return freed, nil
|
||||
}
|
||||
|
||||
// Retirement is what a person (and the controller's `cleanup list` and its probe) asks: what is
|
||||
// held, waiting, rejected and retired here.
|
||||
func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.init()
|
||||
inv, err := h.Adapter.Inventory(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var held []string
|
||||
for as := range h.applied {
|
||||
held = append(held, as)
|
||||
}
|
||||
sort.Strings(held)
|
||||
retired := []map[string]any{}
|
||||
for _, r := range inv.Retired {
|
||||
retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node,
|
||||
"retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)})
|
||||
}
|
||||
out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held),
|
||||
"stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil,
|
||||
"retired": retired}
|
||||
if w := h.r.waiting; w != nil {
|
||||
out["waiting"] = map[string]any{"consumers": h.named(w.set), "since": stamp(w.since), "held": w.held}
|
||||
}
|
||||
if r := h.r.rejected; r != nil {
|
||||
out["rejected"] = map[string]any{"consumers": h.named(r.set), "by": r.by, "why": r.why, "at": stamp(r.at)}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *Harness) waitingWords() string {
|
||||
switch {
|
||||
case h.r.waiting != nil:
|
||||
return "waiting: " + strings.Join(h.r.waiting.set, ", ")
|
||||
case h.r.rejected != nil:
|
||||
return "nothing waits; rejected and kept: " + strings.Join(h.r.rejected.set, ", ")
|
||||
}
|
||||
return "nothing waits for a person here"
|
||||
}
|
||||
|
||||
// named is a set with each consumer's machine, as the events and the tools say it.
|
||||
func (h *Harness) named(set []string) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(set))
|
||||
for _, as := range set {
|
||||
out = append(out, map[string]any{"consumer": as, "node": h.applied[as].node})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *Harness) retirementBody(change string, consumers []map[string]any, extra map[string]any) map[string]any {
|
||||
body := map[string]any{"provider": h.Resource, "provider-node": h.Node, "change": change,
|
||||
"consumers": consumers, "at": stamp(h.Now())}
|
||||
for k, v := range extra {
|
||||
body[k] = v
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// RetirementTools are the four tools every provider serves for retirement, the same names in every
|
||||
// module: the controller's `retire` and `cleanup` verbs ask them on the provider's machine.
|
||||
func RetirementTools(h *Harness) []stdio.Tool {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
ask := func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
}
|
||||
who := map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why — required, kept in the hand-act log"},
|
||||
"by": map[string]any{"type": "string", "description": "who decided"},
|
||||
"via": map[string]any{"type": "string", "description": "mesh-controller when asked through its verbs"},
|
||||
}
|
||||
withSet := map[string]any{"consumers": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the set, exactly as provisioner_retirement names it"}}
|
||||
for k, v := range who {
|
||||
withSet[k] = v
|
||||
}
|
||||
withOne := map[string]any{
|
||||
"consumer": map[string]any{"type": "string", "description": "the retired consumer to delete"},
|
||||
"confirm": map[string]any{"type": "string", "description": "the consumer's name again, to say this is meant"},
|
||||
}
|
||||
for k, v := range who {
|
||||
withOne[k] = v
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: "provisioner_retirement",
|
||||
Description: "What this provider holds, what waits for a person to approve its retirement, what was rejected, " +
|
||||
"and every retired consumer with when, why and its size (novox/hq ADR 0230).",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
return h.Retirement(ctx)
|
||||
}},
|
||||
{Name: "provisioner_retire_approve",
|
||||
Description: "Retire exactly the set waiting for a person (or the set rejected earlier): access disabled, data " +
|
||||
"kept and marked to delete. Use the controller's `retire approve`, which records the hand act.",
|
||||
Input: withSet,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
done, err := h.Approve(ctx, strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"retired": orEmptyList(done)}, nil
|
||||
}},
|
||||
{Name: "provisioner_retire_reject",
|
||||
Description: "Keep the set waiting for a person active. Use the controller's `retire reject`.",
|
||||
Input: withSet,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
kept, err := h.Reject(strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"kept": kept}, nil
|
||||
}},
|
||||
{Name: "provisioner_delete",
|
||||
Description: "Delete one RETIRED consumer — its login and its data — for good. Refused for anything active or " +
|
||||
"asked for. Use the controller's `cleanup delete`, which records the hand act.",
|
||||
Input: withOne,
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
consumer := argString(args, "consumer")
|
||||
if consumer == "" || argString(args, "confirm") != consumer {
|
||||
return nil, errors.New("delete: name the consumer, and repeat it in confirm")
|
||||
}
|
||||
ctx, cancel := ask()
|
||||
defer cancel()
|
||||
freed, err := h.DeleteRetired(ctx, consumer, argString(args, "why"), argString(args, "by"), argString(args, "via"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"deleted": consumer, "freed_bytes": freed}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func strs(v any) []string {
|
||||
list, _ := v.([]any)
|
||||
out := []string{}
|
||||
for _, x := range list {
|
||||
if s, ok := x.(string); ok && s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sorted(list []string) []string {
|
||||
out := append([]string(nil), list...)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func same(a, b []string) bool {
|
||||
return strings.Join(sorted(a), "\x00") == strings.Join(sorted(b), "\x00")
|
||||
}
|
||||
|
||||
func orNone(set []string) string {
|
||||
if len(set) == 0 {
|
||||
return "an empty set"
|
||||
}
|
||||
return strings.Join(set, ", ")
|
||||
}
|
||||
|
||||
func orSomebody(by string) string {
|
||||
if by == "" {
|
||||
return "a person"
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
func orEmptyList(list []string) []string {
|
||||
if list == nil {
|
||||
return []string{}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func kindOf(r Retired) string {
|
||||
if r.Kind == "" {
|
||||
return KindConsumer
|
||||
}
|
||||
return r.Kind
|
||||
}
|
||||
|
||||
func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339) }
|
||||
|
||||
func stampOr(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return stamp(t)
|
||||
}
|
||||
|
||||
func argString(args map[string]any, key string) string {
|
||||
s, _ := args[key].(string)
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
package main
|
||||
|
||||
// Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is
|
||||
// retired only after the same result in five consecutive passes, too many at once wait for a person,
|
||||
// asked for again it is re-enabled, and only a person deletes. The same file in every Go provider.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// recorder is a backend that remembers what it holds, active and retired, as a real one's mark does.
|
||||
type recorder struct {
|
||||
created []Provision
|
||||
removed []string // retired, in order
|
||||
deleted []string
|
||||
held bool
|
||||
failing error
|
||||
holdsErr error
|
||||
retErr error
|
||||
inv Inventory
|
||||
invErr error
|
||||
}
|
||||
|
||||
func (r *recorder) Create(_ context.Context, p Provision) error {
|
||||
if r.failing != nil {
|
||||
return r.failing
|
||||
}
|
||||
r.created = append(r.created, p)
|
||||
r.inv.Retired = withoutRetired(r.inv.Retired, p.As)
|
||||
if !listHas(r.inv.Active, p.As) {
|
||||
r.inv.Active = append(r.inv.Active, p.As)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error {
|
||||
if r.retErr != nil {
|
||||
return r.retErr
|
||||
}
|
||||
r.removed = append(r.removed, as)
|
||||
r.inv.Active = without(r.inv.Active, as)
|
||||
r.inv.Retired = append(withoutRetired(r.inv.Retired, as),
|
||||
Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) Holds(context.Context, Provision) (bool, error) {
|
||||
if r.holdsErr != nil {
|
||||
return false, r.holdsErr
|
||||
}
|
||||
return r.held, nil
|
||||
}
|
||||
|
||||
func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr }
|
||||
|
||||
func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) {
|
||||
r.deleted = append(r.deleted, x.Consumer)
|
||||
r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer)
|
||||
return x.SizeBytes, nil
|
||||
}
|
||||
|
||||
func listHas(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func without(list []string, s string) []string {
|
||||
var out []string
|
||||
for _, x := range list {
|
||||
if x != s {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func withoutRetired(list []Retired, s string) []Retired {
|
||||
var out []Retired
|
||||
for _, x := range list {
|
||||
if x.Consumer != s {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// holding gives the provider n consumers c1…cn and applies them.
|
||||
func holding(w *world, n int) []map[string]any {
|
||||
var given []map[string]any
|
||||
for i := 1; i <= n; i++ {
|
||||
given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"})
|
||||
}
|
||||
w.give(given...)
|
||||
w.h.Reconcile(ctx)
|
||||
return given
|
||||
}
|
||||
|
||||
// pass is one reconcile five seconds after the last.
|
||||
func (w *world) pass() {
|
||||
w.now = w.now.Add(5 * time.Second)
|
||||
w.h.Reconcile(ctx)
|
||||
}
|
||||
|
||||
func retirements(said []announced) []string {
|
||||
var out []string
|
||||
for _, a := range said {
|
||||
if a.event == EventRetirement {
|
||||
out = append(out, a.body["change"].(string))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func lastRetirement(t *testing.T, said []announced) map[string]any {
|
||||
t.Helper()
|
||||
for i := len(said) - 1; i >= 0; i-- {
|
||||
if said[i].event == EventRetirement {
|
||||
return said[i].body
|
||||
}
|
||||
}
|
||||
t.Fatal("nothing about retirement was announced")
|
||||
return nil
|
||||
}
|
||||
|
||||
func namesOf(body map[string]any) string {
|
||||
var out []string
|
||||
for _, c := range body["consumers"].([]map[string]any) {
|
||||
out = append(out, c["consumer"].(string))
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
// settle passes every five seconds until the same answer has held for StableFor, and one pass more.
|
||||
func (w *world) settle() { w.passes(StableFor + 5*time.Second) }
|
||||
|
||||
func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 1)
|
||||
w.give()
|
||||
for i := 0; i < 4; i++ {
|
||||
w.pass()
|
||||
}
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
|
||||
t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said))
|
||||
}
|
||||
}
|
||||
|
||||
// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold:
|
||||
// five passes AND ten minutes of the same answer (novox/hq ADR 0230).
|
||||
func TestFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
holding(w, 1)
|
||||
w.give()
|
||||
for i := 0; i < 5; i++ {
|
||||
w.pass()
|
||||
}
|
||||
if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
|
||||
t.Fatalf("five passes in 25 s retired %v", w.a.removed)
|
||||
}
|
||||
w.passes(StableFor - 30*time.Second)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed)
|
||||
}
|
||||
w.passes(time.Minute)
|
||||
if strings.Join(w.a.removed, ",") != "c1" {
|
||||
t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed)
|
||||
}
|
||||
// Ten minutes in one slow pass are not five passes.
|
||||
w2 := newWorld(t)
|
||||
holding(w2, 1)
|
||||
w2.give()
|
||||
w2.pass()
|
||||
w2.now = w2.now.Add(StableFor)
|
||||
w2.pass()
|
||||
if len(w2.a.removed) != 0 {
|
||||
t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed)
|
||||
}
|
||||
w2.passes(15 * time.Second)
|
||||
if len(w2.a.removed) != 1 {
|
||||
t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 3)
|
||||
w.give(given[:2]...)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "c3" {
|
||||
t.Fatalf("retired %v", w.a.removed)
|
||||
}
|
||||
body := lastRetirement(t, *said)
|
||||
if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" ||
|
||||
!strings.Contains(body["why"].(string), "consecutive passes over 10m") {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" {
|
||||
t.Fatalf("the backend does not hold it retired: %+v", w.a.inv)
|
||||
}
|
||||
// Asked for again: the ordinary create, on the first pass, and said.
|
||||
created := len(w.a.created)
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if len(w.a.created) != created+1 || w.a.created[created].As != "c3" {
|
||||
t.Fatalf("not created again: %v", w.a.created)
|
||||
}
|
||||
if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if len(w.a.inv.Retired) != 0 {
|
||||
t.Fatalf("still marked retired: %+v", w.a.inv.Retired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadablePassStartsTheCountAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
holding(w, 1)
|
||||
w.give()
|
||||
w.passes(StableFor - time.Minute)
|
||||
os.WriteFile(w.receives, []byte("{"), 0o600)
|
||||
w.pass()
|
||||
w.give()
|
||||
w.passes(StableFor - time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("an unreadable pass counted: %v", w.a.removed)
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if len(w.a.removed) != 1 {
|
||||
t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADifferentSetStartsTheCountAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
given := holding(w, 5)
|
||||
w.give(given[:4]...)
|
||||
w.passes(StableFor - time.Minute)
|
||||
w.give(given[:3]...)
|
||||
w.passes(StableFor - time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("a changed set kept its count: %v", w.a.removed)
|
||||
}
|
||||
w.passes(2 * time.Minute)
|
||||
if strings.Join(w.a.removed, ",") != "c4,c5" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdditionsAndChangesAreNeverDelayed(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
holding(w, 1)
|
||||
w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"})
|
||||
w.pass()
|
||||
if len(w.a.created) != 2 || w.a.created[1].As != "c2" {
|
||||
t.Fatalf("an addition waited: %v", w.a.created)
|
||||
}
|
||||
w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"})
|
||||
w.pass()
|
||||
if len(w.a.created) != 3 || w.a.created[2].As != "c1" {
|
||||
t.Fatalf("a change waited: %v", w.a.created)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
holding(w, 4)
|
||||
w.give()
|
||||
w.passes(15 * time.Minute)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("four of four were retired without a person: %v", w.a.removed)
|
||||
}
|
||||
if got := strings.Join(retirements(*said), ","); got != ChangeWaiting {
|
||||
t.Fatalf("said %q, want one waiting", got)
|
||||
}
|
||||
body := lastRetirement(t, *said)
|
||||
if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" {
|
||||
t.Fatalf("%v", body)
|
||||
}
|
||||
if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") {
|
||||
t.Fatal("the wait was not said")
|
||||
}
|
||||
// Said again every quarter of an hour while it waits.
|
||||
w.passes(11 * time.Minute)
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
|
||||
if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil {
|
||||
t.Fatal("approved a set that is not the one waiting")
|
||||
}
|
||||
if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil {
|
||||
t.Fatal("approved without a why")
|
||||
}
|
||||
done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller")
|
||||
if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" {
|
||||
t.Fatal(done, err, w.a.removed)
|
||||
}
|
||||
changes := retirements(*said)
|
||||
if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" {
|
||||
t.Fatalf("%v", changes)
|
||||
}
|
||||
if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" ||
|
||||
!strings.Contains(b["why"].(string), "the old machine is gone") {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
// Nothing more waits.
|
||||
w.passes(time.Minute)
|
||||
if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil {
|
||||
t.Fatal("rejected a set that is not the one waiting")
|
||||
}
|
||||
kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller")
|
||||
if err != nil || len(kept) != 4 {
|
||||
t.Fatal(kept, err)
|
||||
}
|
||||
w.passes(time.Hour)
|
||||
if len(w.a.removed) != 0 {
|
||||
t.Fatalf("a rejected set was retired: %v", w.a.removed)
|
||||
}
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" {
|
||||
t.Fatalf("asked again after a rejection: %q", got)
|
||||
}
|
||||
r, _ := w.h.Retirement(ctx)
|
||||
if r["rejected"] == nil || r["waiting"] != nil {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
// One of them asked for again: a different answer, so the rejection is settled and counting
|
||||
// starts over — three of four is over the bound again, and waits again.
|
||||
w.give(given[0])
|
||||
w.pass()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
w.settle()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
// A rejected set can still be approved later.
|
||||
w2, _ := standingWorld(t)
|
||||
holding(w2, 4)
|
||||
w2.give()
|
||||
w2.settle()
|
||||
w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "")
|
||||
if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 {
|
||||
t.Fatal(done, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingSetAskedForAgainSettles(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
w.give(given...)
|
||||
w.pass()
|
||||
if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 {
|
||||
t.Fatalf("%q %v", got, w.a.removed)
|
||||
}
|
||||
if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil {
|
||||
t.Fatal("approved a set that no longer waits")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 5)
|
||||
w.give(given[:3]...)
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "c4,c5" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted an active consumer")
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil {
|
||||
t.Fatal("deleted without a why")
|
||||
}
|
||||
if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted something not retired")
|
||||
}
|
||||
freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller")
|
||||
if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" {
|
||||
t.Fatal(freed, err, w.a.deleted)
|
||||
}
|
||||
if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
r, _ := w.h.Retirement(ctx)
|
||||
if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" {
|
||||
t.Fatalf("%v", r)
|
||||
}
|
||||
// Retired and asked for again before anybody deleted it: refused, it is the mesh's again.
|
||||
w.give(given[:4]...)
|
||||
w.pass()
|
||||
if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil {
|
||||
t.Fatal("deleted a consumer the mesh asks for")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBound(t *testing.T) {
|
||||
h := &Harness{}
|
||||
h.init()
|
||||
for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} {
|
||||
if h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d waits for a person", c.n, c.held)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} {
|
||||
if !h.overTheBound(c.n, c.held) {
|
||||
t.Errorf("%d of %d is retired without a person", c.n, c.held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{
|
||||
{Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer},
|
||||
{Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"},
|
||||
}}
|
||||
w.give(map[string]any{"as": "kept"})
|
||||
w.h.Reconcile(ctx)
|
||||
if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" {
|
||||
t.Fatalf("%v", b)
|
||||
}
|
||||
if strings.Join(w.a.removed, ",") != "locked-before" {
|
||||
t.Fatalf("adopting did not mark it: %v", w.a.removed)
|
||||
}
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "locked-before,orphan" {
|
||||
t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.a.invErr = errors.New("connection refused")
|
||||
holding(w, 1)
|
||||
if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") {
|
||||
t.Fatal(w.said)
|
||||
}
|
||||
w.a.invErr = nil
|
||||
w.a.inv = Inventory{Active: []string{"c1", "orphan"}}
|
||||
w.pass()
|
||||
w.settle()
|
||||
if strings.Join(w.a.removed, ",") != "orphan" {
|
||||
t.Fatalf("%v", w.a.removed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsAnswer(t *testing.T) {
|
||||
w, _ := standingWorld(t)
|
||||
holding(w, 4)
|
||||
w.give()
|
||||
w.settle()
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tool := range RetirementTools(w.h) {
|
||||
tools[tool.Name] = tool.Run
|
||||
}
|
||||
if len(tools) != 4 {
|
||||
t.Fatalf("%d tools", len(tools))
|
||||
}
|
||||
got, err := tools["provisioner_retirement"](nil)
|
||||
if err != nil || got.(map[string]any)["waiting"] == nil {
|
||||
t.Fatal(got, err)
|
||||
}
|
||||
set := []any{"c1", "c2", "c3", "c4"}
|
||||
if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil {
|
||||
t.Fatal("approved without a why")
|
||||
}
|
||||
if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil {
|
||||
t.Fatal("deleted without confirm")
|
||||
}
|
||||
if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(w.a.deleted, ",") != "c1" {
|
||||
t.Fatal(w.a.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailingConsumerRetiredIsSaidRecovered(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
given := holding(w, 2)
|
||||
w.a.held = false
|
||||
w.a.failing = errors.New("boom")
|
||||
w.passes(7 * time.Minute)
|
||||
w.give(given[0])
|
||||
w.settle()
|
||||
recovered := false
|
||||
for _, a := range *said {
|
||||
if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" {
|
||||
recovered = true
|
||||
}
|
||||
}
|
||||
if !recovered {
|
||||
t.Fatalf("%v", *said)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func TestAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWithdrawnConsumerIsNoLongerFailing(t *testing.T) {
|
||||
func TestAConsumerNoLongerAskedForIsNoLongerFailing(t *testing.T) {
|
||||
w, said := standingWorld(t)
|
||||
w.a.failing = errors.New("boom")
|
||||
w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
|
||||
@@ -139,7 +139,7 @@ func TestAWithdrawnConsumerIsNoLongerFailing(t *testing.T) {
|
||||
w.give(map[string]any{"as": "a"})
|
||||
w.passes(5 * time.Second)
|
||||
last := (*said)[len(*said)-1]
|
||||
if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "withdrawn" {
|
||||
if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "no longer asked for" {
|
||||
t.Fatalf("%v", *said)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user