From cc9e7a72eec19d4eb730e7839a04d97cf8cb7808 Mon Sep 17 00:00:00 2001 From: "fix-cat-pas@novox" Date: Fri, 16 Oct 2026 19:29:30 +0200 Subject: [PATCH 01/11] Name the vault's credential vault-name in the client's bound-entry fixtures (issue 945) --- .../bound_entry_keys_test.go | 28 +++++++++++++++++++ .../bound_gitea_test.go | 2 +- .../mesh-provisioner-client/consumers_test.go | 2 +- .../moved_forge_test.go | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_entry_keys_test.go diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_entry_keys_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_entry_keys_test.go new file mode 100644 index 00000000..0b117768 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_entry_keys_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// Issue 945: the SDK names the vault's credential of a bound entry `vault-name`, and the state guard refuses an entry +// written under the old key `credential`. The client never reads that key, but a fixture that carries it is a bound +// entry the mesh would refuse, so no fixture of this client writes one. +func TestNoFixtureCarriesABoundEntryKeyTheStateGuardRefuses(t *testing.T) { + files, err := filepath.Glob("*_test.go") + if err != nil { + t.Fatal(err) + } + old := `"credential"` + `:"pair/` // a bound entry's credential, under its old key + for _, f := range files { + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), old) { + t.Errorf("%s has a bound entry with the key credential; the SDK names it vault-name (issue 945)", f) + } + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go index 23664ee4..c7ed03eb 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go @@ -63,7 +63,7 @@ func giteaRun(t *testing.T, entry string, extra string) (*engine, *aBus, *report func TestNatsAHandedGiteasServerEnvIsFilledAsTheControllerComposedIt(t *testing.T) { e, _, _ := giteaRun(t, `{"provider":"novox","provider-module":"postgres","provision":"postgres-database",`+ `"consumer":"g14","module":"gitea","as":"mesh_g14_gitea","user":"mesh_g14_gitea",`+ - `"values":{"at":"10.7.0.1","from":"novox","port":5432},"credential":"pair/novox/postgres-database/g14/gitea"}`, + `"values":{"at":"10.7.0.1","from":"novox","port":5432},"vault-name":"pair/novox/postgres-database/g14/gitea"}`, "FROM=${bound:postgres-database:from}\n") eventually(t, "server.env is filled to the controller's text", func() bool { return e.read(t, giteaEnvPath) == giteaBoundExpected("tok-never-on-the-bus", "pw-never-on-the-bus")+"FROM=novox\n" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go index 469e4396..b6a75b20 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go @@ -224,7 +224,7 @@ func TestNatsAConsumersBoundValuesAreFilledFromItsProvidersEntry(t *testing.T) { }) b.put(t, "postgres_bound", node+".app.postgres-database", `{"provider":"novox","provider-module":"postgres",`+ `"provision":"postgres-database","consumer":"g14","module":"app","as":"mesh_g14_app","user":"mesh_g14_app",`+ - `"values":{"host":"novox.mesh","port":5432},"credential":"pair/novox/postgres-database/g14/app"}`) + `"values":{"host":"novox.mesh","port":5432},"vault-name":"pair/novox/postgres-database/g14/app"}`) eventually(t, "the file is filled from its provider's entry", func() bool { return e.read(t, appBound) == "HOST=novox.mesh\nPORT=5432\nUSER=mesh_g14_app\n" }) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go index 848227ef..882670f9 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go @@ -86,7 +86,7 @@ func TestNatsAMovedGiteaAndClaudeCodeHaveTheirFilesFilledFromTheBus(t *testing.T } b.put(t, "postgres_bound", node+".gitea.postgres-database", `{"provider":"novox","provider-module":"postgres",`+ `"provision":"postgres-database","consumer":"g14","module":"gitea","as":"mesh_g14_gitea","user":"mesh_g14_gitea",`+ - `"values":{"at":"10.7.0.1","from":"novox","port":5432},"credential":"pair/novox/postgres-database/g14/gitea"}`) + `"values":{"at":"10.7.0.1","from":"novox","port":5432},"vault-name":"pair/novox/postgres-database/g14/gitea"}`) b.put(t, "route-proxy_bound", node+".gitea.route", `{"provider":"novox","provider-module":"route-proxy",`+ `"provision":"route","consumer":"g14","module":"gitea","as":"mesh_g14_gitea",`+ `"values":{"name":"git.example.org","internal-name":"git.novox.internal","name-web":"git.example.org"}}`) From 0b056edaaae9448ae0bbf619e5f5852e44540a97 Mon Sep 17 00:00:00 2001 From: "fix-cat-pas@novox" Date: Fri, 16 Oct 2026 19:29:30 +0200 Subject: [PATCH 02/11] Prove the client fills gitea's origins and keycloak's hostname from route-proxy's entry for a controller-shaped plan entry (issue 923) --- .../route_names_test.go | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go new file mode 100644 index 00000000..42b27ecf --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "os" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/provider" + "github.com/nats-io/nats.go/jetstream" +) + +// Issue 923: route-proxy serves a consumer of its route the names the controller composed, and the consumer's client +// fills gitea's origins and keycloak's hostname.env from them. The route-proxy provider is run here as its module +// runs it (the SDK helper, `route` offered with no credential and nothing served), over a plan entry shaped as the +// controller writes it (mesh-controller handed.go PlanProvision: `at`, `from`, `as` and `names`), and the entry it +// writes is the one the client reads. + +// memBus is the provider helper's bus over in-memory buckets. +type memBus struct { + mu sync.Mutex + buckets map[string]map[string][]byte +} + +func (m *memBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { + m.mu.Lock() + defer m.mu.Unlock() + for k, v := range m.buckets[bucket] { + handle(k, v) + } + return nil +} + +func (m *memBus) Put(_ context.Context, bucket, key string, value []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.buckets[bucket] == nil { + m.buckets[bucket] = map[string][]byte{} + } + m.buckets[bucket][key] = append([]byte(nil), value...) + return nil +} + +func (m *memBus) Delete(_ context.Context, bucket, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.buckets[bucket], key) + return nil +} + +// routeEntryFor is the entry route-proxy's helper writes for module's plan entry on the route provision. +func routeEntryFor(t *testing.T, module, planEntry string) string { + t.Helper() + bus := &memBus{buckets: map[string]map[string][]byte{provider.AssignmentsBucket: { + node + "." + module: []byte(`{"node":"` + node + `","module":"` + module + `","provisioning":{"provisions":[` + + planEntry + `]}}`)}}} + p := provider.New(provider.Options{Node: "novox", Module: "route-proxy", Bus: bus, Logf: t.Logf, + Offers: map[string]provider.Offer{"route": {Credential: provider.NoCredential, Serves: map[string]any{}}}}) + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].Status != provider.Answered { + t.Fatalf("the provider did %+v", out) + } + return string(bus.buckets[provider.BucketOf("route-proxy")][node+"."+module+".route"]) +} + +func routeRun(t *testing.T, module, id, path, templatePath, entryKey, firstEntry string) (*engine, *aBus, *reports) { + t.Helper() + raw, err := os.ReadFile(templatePath) + if err != nil { + t.Fatal(err) + } + b := startBus(t) + registryOf679(t, b) + startVault(t, b) + heard := b.listen(t) + e := startEngine(t, map[string]string{path: "owned"}) + tpl, _ := templates(t, map[string]string{id: string(raw)}) + b.assign(t, module, Plan{Templates: tpl, Files: []PlannedFile{{ID: id, Path: path, Template: id, + OwnerAtRuntime: "owned"}}, + Provisions: []PlannedProvision{{Provision: "route", Provider: "novox", ProviderModule: "route-proxy"}}}) + kv, err := b.domain.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: "route-proxy_bound"}) + if err != nil { + t.Fatal(err) + } + b.kv["route-proxy_bound"] = kv + if firstEntry != "" { + b.put(t, "route-proxy_bound", node+"."+entryKey, firstEntry) + } + aClient(t, b, e, t.TempDir(), time.Hour) + return e, b, heard +} + +// gitea's plan entry for its `web` route, as the controller writes it for a node whose public domain is example.org. +const giteaRoutePlanEntry = `{"provision":"route","provider":"novox","provider-module":"route-proxy",` + + `"at":"novox.internal","from":"novox","as":"mesh_g14_gitea",` + + `"names":{"name-web":"git.example.org","internal-name-web":"git.novox.internal",` + + `"name-internal-api-refused":"git.example.org","internal-name-internal-api-refused":"git.novox.internal"}}` + +const keycloakRoutePlanEntry = `{"provision":"route","provider":"novox","provider-module":"route-proxy",` + + `"at":"novox.internal","from":"novox","as":"mesh_g14_keycloak",` + + `"names":{"name":"keycloak.example.org","internal-name":"keycloak.novox.internal"}}` + +func TestNatsGiteasOriginsAreFilledFromRouteProxysEntryForThePlanEntryTheControllerWrites(t *testing.T) { + e, _, _ := routeRun(t, "gitea", "origins", "/etc/gitea/origins", "../../../gitea/files/origins", "gitea.route", + routeEntryFor(t, "gitea", giteaRoutePlanEntry)) + eventually(t, "origins is filled with the public name", func() bool { + return e.read(t, "/etc/gitea/origins") == "https://git.example.org\n" + }) +} + +func TestNatsKeycloaksHostnameIsFilledFromRouteProxysEntryForThePlanEntryTheControllerWrites(t *testing.T) { + e, _, _ := routeRun(t, "keycloak", "hostname", "/etc/keycloak/hostname.env", "../../../keycloak/files/hostname.env", + "keycloak.route", routeEntryFor(t, "keycloak", keycloakRoutePlanEntry)) + eventually(t, "hostname.env is filled with the public name", func() bool { + return e.read(t, "/etc/keycloak/hostname.env") == "KC_HOSTNAME=https://keycloak.example.org\n" + }) +} + +// A module that failed on a missing name is filled once the provider's entry carries it: the entry first held no +// names (as the provider wrote it before the controller composed them), then the controller's plan entry arrives. +func TestNatsAModuleThatFailedOnAMissingNameIsFilledWhenTheNameArrives(t *testing.T) { + bare := routeEntryFor(t, "gitea", `{"provision":"route","provider":"novox","provider-module":"route-proxy",`+ + `"as":"mesh_g14_gitea"}`) + e, b, heard := routeRun(t, "gitea", "origins", "/etc/gitea/origins", "../../../gitea/files/origins", "gitea.route", bare) + eventually(t, "the module fails for the missing name", func() bool { + r, _ := heard.last("gitea") + return r.Status == StatusFailed && strings.Contains(r.Reason, `no "name-web"`) + }) + b.put(t, "route-proxy_bound", node+".gitea.route", routeEntryFor(t, "gitea", giteaRoutePlanEntry)) + eventually(t, "origins is filled once the name arrives", func() bool { + return e.read(t, "/etc/gitea/origins") == "https://git.example.org\n" + }) +} From 0dab4475cd8882045d991574f4b5b7e76ba807f2 Mon Sep 17 00:00:00 2001 From: "fix-vault-pas@novox" Date: Fri, 16 Oct 2026 19:30:15 +0200 Subject: [PATCH 03/11] Answer an unreadable assignment as not yet, never a refusal (issue 928) --- modules/mesh-vault/cmd/mesh-vault/askers.go | 9 +++- .../mesh-vault/cmd/mesh-vault/notyet_test.go | 48 +++++++++++++++++++ modules/mesh-vault/cmd/mesh-vault/pair.go | 5 +- modules/mesh-vault/cmd/mesh-vault/vault.go | 6 +++ 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-vault/cmd/mesh-vault/notyet_test.go diff --git a/modules/mesh-vault/cmd/mesh-vault/askers.go b/modules/mesh-vault/cmd/mesh-vault/askers.go index 2e385c3a..037bce99 100644 --- a/modules/mesh-vault/cmd/mesh-vault/askers.go +++ b/modules/mesh-vault/cmd/mesh-vault/askers.go @@ -143,8 +143,8 @@ func (d Declared) May(caller, name string) (bool, string) { } raw, err := d.Read(node + "." + module) if err != nil { - return false, fmt.Sprintf("the assignment %s.%s could not be read, so what it names is not known: %v", node, - module, err) + return false, fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read, so what it names is not known: %v", + node, module, err) } if raw == nil { return false, fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", module, @@ -173,3 +173,8 @@ func (d Declared) May(caller, name string) (bool, string) { } return false, fmt.Sprintf("%s's plan on %s names no secret %s", module, node, secret) } + +// notYetPrefix begins a reason that is not a refusal (issue 928): what the rule needs could not be read just now, a +// bus error rather than an answer. The vault says "holds no secret named ... yet", which the client waits on and asks +// again, and records no refusal. +const notYetPrefix = "not yet: " diff --git a/modules/mesh-vault/cmd/mesh-vault/notyet_test.go b/modules/mesh-vault/cmd/mesh-vault/notyet_test.go new file mode 100644 index 00000000..951ae1b6 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/notyet_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// Issue 928: an assignment the vault could not read (the controller was being pushed, "no responders") is not a +// refusal the client keeps for ten minutes: it is "not yet", the answer the client waits on and asks again. +func TestAnAssignmentThatCannotBeReadIsNotYetAndNotARefusal(t *testing.T) { + down := errors.New("nats: no responders available for request") + read := func(string) ([]byte, error) { return nil, down } + v := newVault(t, Declared{Read: read}) + asker, _ := seal.GenerateKeyPair() + for _, name := range []string{"own/g14/claude-code/token", "provision/novox/gitea-read/g14/claude-code"} { + _, err := v.Get("provisioner.g14", name, asker.PublicKey) + if err == nil { + t.Fatalf("%s: answered while the assignment cannot be read and the vault holds nothing", name) + } + if !strings.Contains(err.Error(), "holds no secret named") || !strings.Contains(err.Error(), "no responders") { + t.Errorf("%s: the answer is %q, not the not-yet the client waits on, with the reason", name, err) + } + } + hs, _ := v.Store.Handed(10) + for _, h := range hs { + if h.Outcome == "refused" { + t.Errorf("a read error was recorded as a refusal: %+v", h) + } + } +} + +// A pair row asked for while the consumer's assignment cannot be read is not yet either, and a row whose +// assignment reads as absent stays "not on". +func TestAPairRowWhoseAssignmentCannotBeReadIsNotYet(t *testing.T) { + down := errors.New("nats: no responders available for request") + parts := []string{"pair", "novox", "db", "g14", "claude-code"} + why := pairRowRefusal(func(string) ([]byte, error) { return nil, down }, "g14", parts) + if !strings.HasPrefix(why, notYetPrefix) || !strings.Contains(why, "no responders") { + t.Errorf("a read error answered %q, want a not-yet carrying the reason", why) + } + why = pairRowRefusal(func(string) ([]byte, error) { return nil, nil }, "g14", parts) + if strings.HasPrefix(why, notYetPrefix) || !strings.Contains(why, "is not on g14") { + t.Errorf("an absent assignment answered %q, want the refusal 'not on'", why) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/pair.go b/modules/mesh-vault/cmd/mesh-vault/pair.go index ac3df924..2417df12 100644 --- a/modules/mesh-vault/cmd/mesh-vault/pair.go +++ b/modules/mesh-vault/cmd/mesh-vault/pair.go @@ -780,7 +780,10 @@ func pairRowRefusal(read AssignmentReader, node string, parts []string) string { return "the vault cannot read the assignments, so it cannot tell what " + node + "'s plans name" } raw, err := read(consumer + "." + module) - if err != nil || raw == nil { + if err != nil { + return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", consumer, module, err) + } + if raw == nil { return fmt.Sprintf("%s is not on %s", module, consumer) } var a plannedProvisions diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index 817bb3b3..0fffa275 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "regexp" + "strings" "sync" "time" @@ -237,6 +238,11 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) { return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) } } else if ok, why := askers.May(caller, name); !ok { + if reason, notYet := strings.CutPrefix(why, notYetPrefix); notYet { + // Not a refusal (issue 928): the client waits and asks again, and nothing is recorded against it. + v.log("not yet %s to %s: %s", name, caller, reason) + return Answer{}, fmt.Errorf("the vault holds no secret named %s yet: %s", name, reason) + } v.log("refused %s to %s: %s", name, caller, why) _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) From 044a548cee8d8009a937b03cb21b624b095b7835 Mon Sep 17 00:00:00 2001 From: "fix-vault-pas@novox" Date: Fri, 16 Oct 2026 19:31:40 +0200 Subject: [PATCH 04/11] Pin that the client asks a refused or not-yet secret again (issue 928) --- .../asked_again_test.go | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go new file mode 100644 index 00000000..844aa47e --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +// scriptedSecrets answers each Get from the next of its errors, then with a value, counting the asks. +type scriptedSecrets struct { + errs []error + asked int +} + +func (f *scriptedSecrets) Get(context.Context, string) (string, int, error) { + f.asked++ + if f.asked <= len(f.errs) { + return "", 0, f.errs[f.asked-1] + } + return "tok", 1, nil +} + +// Issue 928: a refusal is asked again after RefusedFor, and a not-yet on every pass, so a vault that was down or +// that holds the value later fills the file with no change of the plan. +func TestARefusedSecretIsAskedAgainAfterRefusedForAndFilledWhenTheVaultHasIt(t *testing.T) { + now := time.Date(2026, 10, 16, 16, 20, 0, 0, time.UTC) + fake := &scriptedSecrets{errs: []error{errors.New("the vault refused it: the assignment g14.claude-code could not be read")}} + c := &Client{Node: "g14", Secrets: fake, Now: func() time.Time { return now }} + s := &state{secrets: map[string]heldSecret{}} + n := Need{Kind: "secret", Name: "gitea-read"} + name := "provision/novox/gitea-read/g14/claude-code" + + if _, _, fail := c.vaultValue(context.Background(), s, n, name); fail == "" { + t.Fatal("a refusal did not fail the need") + } + now = now.Add(RefusedFor - time.Second) + if _, _, fail := c.vaultValue(context.Background(), s, n, name); fail == "" || fake.asked != 1 { + t.Fatalf("within RefusedFor the vault was asked again (%d asks) or the refusal was dropped", fake.asked) + } + now = now.Add(time.Second) + v, wait, fail := c.vaultValue(context.Background(), s, n, name) + if fake.asked != 2 || v != "tok" || wait != "" || fail != "" { + t.Fatalf("after RefusedFor: %d asks, value %q, waits %q, fails %q; want asked again and filled", fake.asked, v, wait, fail) + } +} + +func TestANotYetSecretIsAskedAgainOnEveryPass(t *testing.T) { + now := time.Date(2026, 10, 16, 16, 20, 0, 0, time.UTC) + notYet := fmt.Errorf("%w: the vault holds no secret named x yet", errNotYet) + fake := &scriptedSecrets{errs: []error{notYet, notYet}} + c := &Client{Node: "g14", Secrets: fake, Now: func() time.Time { return now }} + s := &state{secrets: map[string]heldSecret{}} + n := Need{Kind: "secret", Name: "gitea-read"} + for i := 0; i < 2; i++ { + if _, wait, fail := c.vaultValue(context.Background(), s, n, "x"); wait == "" || fail != "" { + t.Fatalf("pass %d: waits %q fails %q, want a wait", i, wait, fail) + } + } + if v, _, _ := c.vaultValue(context.Background(), s, n, "x"); v != "tok" || fake.asked != 3 { + t.Fatalf("value %q after %d asks, want filled on the third", v, fake.asked) + } +} From a51e2c0fb42e9c98bdda16ee0b843e62ff8c76ce Mon Sep 17 00:00:00 2001 From: "fix-cat-pas@novox" Date: Fri, 16 Oct 2026 19:32:20 +0200 Subject: [PATCH 05/11] Pin route-proxy's acme templates beside its manifest (issue 849) --- .../cmd/route-provider/provider_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/route-proxy/cmd/route-provider/provider_test.go b/modules/route-proxy/cmd/route-provider/provider_test.go index f5c4b7c2..6c580b97 100644 --- a/modules/route-proxy/cmd/route-provider/provider_test.go +++ b/modules/route-proxy/cmd/route-provider/provider_test.go @@ -105,4 +105,21 @@ func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) { if got := hex.EncodeToString(sum[:]); got != "57a39de0943d8b33b9db6cb998ce97f8793602cca6cdfa5849c1c392390454a3" { t.Fatalf("route-proxy's own manifest changed: digest %s", got) } + // The two acme templates are pinned the same way (issue 849): they are what the proxy's certificates are asked + // with, so a change to either is a change to the public ingress. + for file, want := range map[string]string{ + "acme.env": "d66eba145c893080e30e013329fc00e51395f0c31fc8621a20c7b26abc11b52f", + "internal-acme.env": "baf4da75379183758c8ae505be4dd8a79d5b089ae1e7c8ded4d232df40251df5", + } { + text, err := os.ReadFile("../../files/" + file) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(text) + if got := hex.EncodeToString(sum[:]); got != want { + t.Fatalf("route-proxy's files/%s changed: digest %s, pinned %s; if the change is meant, check that the "+ + "public and internal ingress still ask for the certificates they should, then pin the new digest here", + file, got, want) + } + } } From 5a00596908182730955ddb37439526911f5d085c Mon Sep 17 00:00:00 2001 From: "fix-vault-pas@novox" Date: Fri, 16 Oct 2026 19:34:11 +0200 Subject: [PATCH 06/11] Make the vault ask a software-issuing provider's tool itself for holders that hold nothing (issue 928) --- modules/mesh-vault/cmd/mesh-vault/intake.go | 16 +++- .../cmd/mesh-vault/own_issue_test.go | 84 +++++++++++++++++ modules/mesh-vault/cmd/mesh-vault/ownissue.go | 89 +++++++++++++++++++ modules/mesh-vault/cmd/mesh-vault/pair.go | 14 ++- .../mesh-vault/cmd/mesh-vault/pair_test.go | 10 ++- 5 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-vault/cmd/mesh-vault/own_issue_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/ownissue.go diff --git a/modules/mesh-vault/cmd/mesh-vault/intake.go b/modules/mesh-vault/cmd/mesh-vault/intake.go index d7898574..329b2c8e 100644 --- a/modules/mesh-vault/cmd/mesh-vault/intake.go +++ b/modules/mesh-vault/cmd/mesh-vault/intake.go @@ -444,6 +444,11 @@ func (v *Vault) IssuedOpen(caller, provision, provider, module string) (Opened, if err := fromTheController(caller, "issued"); err != nil { return Opened{}, err } + return v.openIssued(provision, provider, module) +} + +// openIssued is IssuedOpen for a call the vault makes in its own name (ownIssue, issue 928). +func (v *Vault) openIssued(provision, provider, module string) (Opened, error) { secret, err := ProvisionSecret(provider, provision) if err != nil { return Opened{}, err @@ -584,6 +589,15 @@ func (v *Vault) TakeIssued(caller, provision, provider, module, tool string) (Is // credential, as before) or one per holder (`holders`: node, module and sealed each, kept as that holder's own and // handed to it alone, perholder.go). Never both. func (v *Vault) TakeIssuedFor(caller, provision, provider, module, tool string, holders []Holder) (Issued, error) { + if err := fromTheController(caller, "issued"); err != nil { + return Issued{}, err + } + return v.takeIssuedFor(caller, provision, provider, module, tool, holders) +} + +// takeIssuedFor is TakeIssuedFor for whoever asks: the controller's verb, or the vault's own pass (ownIssue). caller +// is who the kept values are recorded as taken by. +func (v *Vault) takeIssuedFor(caller, provision, provider, module, tool string, holders []Holder) (Issued, error) { if v.Ask == nil { return Issued{}, errors.New("nothing was taken: this vault reaches no provider") } @@ -593,7 +607,7 @@ func (v *Vault) TakeIssuedFor(caller, provision, provider, module, tool string, if err := checkHolders(holders); err != nil { return Issued{}, fmt.Errorf("nothing was asked: %w", err) } - opened, err := v.IssuedOpen(caller, provision, provider, module) + opened, err := v.openIssued(provision, provider, module) if err != nil { return Issued{}, err } diff --git a/modules/mesh-vault/cmd/mesh-vault/own_issue_test.go b/modules/mesh-vault/cmd/mesh-vault/own_issue_test.go new file mode 100644 index 00000000..84bacaac --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/own_issue_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "strings" + "testing" +) + +// Issue 928 (ADR 0349 clause 10): the vault takes a software-issued value by itself. With claude-code's need +// recorded on two machines and the forge's tool answering, one pass asks the tool for both holders and keeps a +// value for each; the next pass leaves them be, and a holder bound later is the only one asked. +func TestThePassAsksAnIssuingProvidersToolForTheHoldersItHoldsNothingFor(t *testing.T) { + mirror := assignments{ + "novox.gitea": handedWith("novox", "gitea"), + "anchor.claude-code": handedWith("anchor", "claude-code", "gitea-read@novox/gitea=gitea-read+issued~gitea_agent_token"), + } + v := pairVault(t, mirror) + var asked []map[string]any + v.Ask = forgeMesh(perHolderProvider(agentTokens, &asked)).ask + + o := pass(t, v, "provision/novox/gitea-read") + if o.Outcome != PairPerHolder { + t.Fatalf("outcome %+v", o) + } + if len(asked) != 1 || !strings.Contains(toString(asked[0]["holders"]), "anchor") { + t.Fatalf("the tool was asked %v, want once for anchor", asked) + } + if _, held, _ := v.Store.Read(anchorsToken); !held { + t.Fatal("the vault took no value for claude-code on anchor by itself") + } + if got := handedTo(t, v, "provisioner.anchor", anchorsToken); got != agentTokens["anchor/claude-code"] { + t.Errorf("anchor got %q", got) + } + + // Held is left untouched: no second ask, so the forge does not mint anew (rotation is the software's). + pass(t, v, "provision/novox/gitea-read") + if len(asked) != 1 { + t.Errorf("the tool was asked again for a holder that holds its value (%d asks)", len(asked)) + } + + // A holder bound since is the only one asked. + mirror["laptop.claude-code"] = handedWith("laptop", "claude-code", "gitea-read@novox/gitea=gitea-read+issued~gitea_agent_token") + pass(t, v, "provision/novox/gitea-read") + if len(asked) != 2 || strings.Contains(toString(asked[1]["holders"]), "anchor") { + t.Fatalf("asks %v, want a second naming laptop alone", asked) + } + if _, held, _ := v.Store.Read(laptopsToken); !held { + t.Error("no value for the holder bound later") + } +} + +// A failed ask keeps nothing, is said as a held outcome, and is asked again at the next pass; a plan naming no +// tool is said, not guessed. +func TestAFailedAskOfTheToolIsAskedAgainAndNoToolIsSaid(t *testing.T) { + mirror := assignments{ + "novox.gitea": handedWith("novox", "gitea"), + "anchor.claude-code": handedWith("anchor", "claude-code", "gitea-read@novox/gitea=gitea-read+issued~gitea_agent_token"), + } + v := pairVault(t, mirror) + v.Ask = forgeMesh(func(string, map[string]any) (map[string]any, error) { return nil, errTest }).ask + o := pass(t, v, "provision/novox/gitea-read") + if o.Outcome != PairHeld || !strings.Contains(o.Why, "could not be asked") { + t.Fatalf("a failed ask is %+v", o) + } + var asked []map[string]any + v.Ask = forgeMesh(perHolderProvider(agentTokens, &asked)).ask + pass(t, v, "provision/novox/gitea-read") + if _, held, _ := v.Store.Read(anchorsToken); !held || len(asked) != 1 { + t.Fatal("the next pass did not ask again and keep the value") + } + + noTool := pairVault(t, assignments{ + "novox.gitea": handedWith("novox", "gitea"), + "anchor.claude-code": handedWith("anchor", "claude-code", "gitea-read@novox/gitea=gitea-read+issued"), + }) + noTool.Ask = v.Ask + if o := pass(t, noTool, "provision/novox/gitea-read"); o.Outcome != PairHeld || !strings.Contains(o.Why, "name no tool") { + t.Errorf("a plan with no tool is %+v", o) + } +} + +var errTest = errString("nats: no responders available for request") + +func toString(v any) string { return strings.TrimSpace(strings.ReplaceAll(fmt.Sprint(v), "map", "")) } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownissue.go b/modules/mesh-vault/cmd/mesh-vault/ownissue.go new file mode 100644 index 00000000..aebf9bc4 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ownissue.go @@ -0,0 +1,89 @@ +package main + +// The vault asks a software-issuing provider's tool itself (novox/hq ADR 0349 clause 10, issue 928). +// +// A provider whose offer says `issued-by: software` has its value taken from its tool, sealed to the vault, "when the +// provider starts, when the software's value changes and at least hourly". Only the controller's `secret issued` made +// the vault ask until now, so a holder that bound after the last such act waited for ever. The pass (once a minute and +// when the vault starts) now asks the tool for the holders the plans name that hold nothing yet, in the vault's own +// name and through the same path as the controller's verb. A holder that holds its value is left untouched: the +// forge mints anew on every ask, and a rotation is the software's (ADR 0305 §5, §9). A failed ask is said, keeps +// nothing, and is asked again at the next pass. + +import ( + "fmt" + "sort" + "strings" +) + +// ownCaller is who a value the vault asked for itself is recorded as taken by. +const ownCaller = "mesh-vault" + +// perHolderWhy is what a pass says of a provision whose plans name a credential per holder. +const perHolderWhy = "a plan names a credential per holder for it (issue 757): there is no one shared value" + +// ownIssue is the pass's part for a provision whose plans name a credential per holder: when its offer is issued by +// the provider's software and the plans name the tool, the holders that hold nothing yet are asked of it. +func (v *Vault) ownIssue(k pairKey, b *binds) PairOutcome { + name := k.name() + said := PairOutcome{Name: name, Outcome: PairPerHolder, Why: perHolderWhy} + if v.Ask == nil || len(b.forms) != 1 || !b.forms[FormIssued] { + return said + } + held := func(why string) PairOutcome { + return PairOutcome{Name: name, Outcome: PairHeld, Why: why} + } + var tools []string + for t := range b.tools { + if t != "" { + tools = append(tools, t) + } + } + sort.Strings(tools) + switch { + case len(tools) == 0: + return held(fmt.Sprintf("the plans name %s from %s as issued by its software and name no tool that answers it; "+ + "the vault cannot ask for it (issue 928)", k.provision, k.provider)) + case len(tools) > 1 || b.tools[""]: + return held(fmt.Sprintf("the plans naming %s from %s disagree on its tool (%s); the vault asks none", k.provision, + k.provider, strings.Join(tools, ", "))) + case len(b.providerModules) != 1: + return held(fmt.Sprintf("no one plan names the module serving %s on %s, so the vault cannot tell whose tool to ask", + k.provision, k.provider)) + } + var module string + for m := range b.providerModules { + module = m + } + var missing []Holder + for h := range b.holders { + secret, err := HolderSecret(k.provider, k.provision, h) + if err != nil { + continue + } + if _, have, err := v.Store.Read(secret); err == nil && !have { + missing = append(missing, h) + } + } + if len(missing) == 0 { + return said + } + sort.Slice(missing, func(i, j int) bool { return missing[i].String() < missing[j].String() }) + got, err := v.takeIssuedFor(ownCaller, k.provision, k.provider, module, tools[0], missing) + if err != nil { + return held(err.Error()) + } + if len(got.NotAnswered) > 0 { + var who []string + for _, h := range got.NotAnswered { + who = append(who, h.String()) + } + return held(fmt.Sprintf("its tool %s gave no value for %s", tools[0], strings.Join(who, ", "))) + } + var who []string + for _, h := range got.Holders { + who = append(who, h.Holder.String()) + } + return PairOutcome{Name: name, Outcome: PairPerHolder, Why: fmt.Sprintf("the vault asked its tool %s and keeps a value "+ + "for %s (issue 928)", tools[0], strings.Join(who, ", "))} +} diff --git a/modules/mesh-vault/cmd/mesh-vault/pair.go b/modules/mesh-vault/cmd/mesh-vault/pair.go index 2417df12..40221732 100644 --- a/modules/mesh-vault/cmd/mesh-vault/pair.go +++ b/modules/mesh-vault/cmd/mesh-vault/pair.go @@ -139,6 +139,10 @@ type binds struct { consumers map[string]bool perHolder bool forms map[string]bool + // holders are the modules on a machine whose plan names a credential per holder for it, and tools the + // `credential-tool` those entries name (issue 928): what the vault asks the provider's tool with. + holders map[Holder]bool + tools map[string]bool } // The credential forms a plan entry names (issue 768), as the provider's offer declares them. @@ -203,6 +207,9 @@ type plannedProvisions struct { Provider string `json:"provider"` ProviderModule string `json:"provider-module"` CredentialForm string `json:"credential-form"` + // CredentialTool is the provider's tool that answers the credential of an offer issued by its software + // (issue 928): the vault asks it itself. + CredentialTool string `json:"credential-tool"` // Contributes is what the consumer tells its provider (issue 776); the vault does not act on it. Contributes map[string]any `json:"contributes"` } `json:"provisions"` @@ -244,7 +251,7 @@ func readBindings(m Mirror) (bindings, error) { k := pairKey{p.Provider, p.Provision} if b.bound[k] == nil { b.bound[k] = &binds{providerModules: map[string]bool{}, consumers: map[string]bool{}, - forms: map[string]bool{}} + forms: map[string]bool{}, holders: map[Holder]bool{}, tools: map[string]bool{}} } b.bound[k].forms[p.CredentialForm] = true if p.ProviderModule != "" { @@ -254,6 +261,8 @@ func readBindings(m Mirror) (bindings, error) { // for one holder: the provider still gets its logins as rows. if p.Name != "" && p.CredentialForm != FormPair { b.bound[k].perHolder = true + b.bound[k].holders[Holder{Node: a.Node, Module: a.Module}] = true + b.bound[k].tools[p.CredentialTool] = true } b.bound[k].consumers[consumerOf(a.Node, a.Module, p.Local)] = true } @@ -428,8 +437,7 @@ func (v *Vault) PairPass() ([]PairOutcome, error) { func (v *Vault) pairOne(k pairKey, b *binds, rows []pairRow, seen bindings) (PairOutcome, error) { name := k.name() if b != nil && b.perHolder { - return PairOutcome{Name: name, Outcome: PairPerHolder, - Why: "a plan names a credential per holder for it (issue 757): there is no one shared value"}, nil + return v.ownIssue(k, b), nil } rec, held, err := v.Store.Read(name) if err != nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/pair_test.go b/modules/mesh-vault/cmd/mesh-vault/pair_test.go index 4ec4ad0f..f241624a 100644 --- a/modules/mesh-vault/cmd/mesh-vault/pair_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/pair_test.go @@ -27,11 +27,14 @@ func (a assignments) keys() ([]string, error) { } // handedWith is an assignment handed to the provisioner, its plan naming each provision given as -// provision@provider/provider-module[#local][=name][+credential-form]. +// provision@provider/provider-module[#local][=name][+credential-form][~credential-tool] (the tool last). func handedWith(node, module string, provisions ...string) string { var entries []string for _, p := range provisions { - form := "" + form, tool := "", "" + if i := strings.Index(p, "~"); i >= 0 { + p, tool = p[:i], p[i+1:] + } if i := strings.Index(p, "+"); i >= 0 { p, form = p[:i], p[i+1:] } @@ -55,6 +58,9 @@ func handedWith(node, module string, provisions ...string) string { if form != "" { e += `,"credential-form":"` + form + `"` } + if tool != "" { + e += `,"credential-tool":"` + tool + `"` + } entries = append(entries, e+"}") } return `{"node":"` + node + `","module":"` + module + `","provisioning":{"templates":{"source":"x","digest":"y"},` + From 33759f8b5790c3b78347e0d4c5185356719200c2 Mon Sep 17 00:00:00 2001 From: "fix-cat-pas@novox" Date: Fri, 16 Oct 2026 19:35:22 +0200 Subject: [PATCH 07/11] Run postgres on the SDK's harness and moved path, dropping its copies (issue 849) --- .../cmd/postgres-provider/early_test.go | 62 -- .../postgres-provider/grants_moved_test.go | 123 ---- .../postgres/cmd/postgres-provider/harness.go | 574 ---------------- .../postgres-provider/harness_same_test.go | 34 - .../cmd/postgres-provider/harness_test.go | 141 +--- .../cmd/postgres-provider/live_test.go | 15 +- .../postgres/cmd/postgres-provider/main.go | 9 +- .../postgres/cmd/postgres-provider/moved.go | 298 +-------- .../cmd/postgres-provider/moved_test.go | 306 +-------- .../cmd/postgres-provider/provisioner.go | 11 +- .../cmd/postgres-provider/retire_pg.go | 17 +- .../cmd/postgres-provider/retire_pg_test.go | 11 +- .../cmd/postgres-provider/retirement.go | 623 ------------------ .../cmd/postgres-provider/retirement_test.go | 571 ---------------- .../cmd/postgres-provider/standing_test.go | 193 ------ 15 files changed, 78 insertions(+), 2910 deletions(-) delete mode 100644 modules/postgres/cmd/postgres-provider/early_test.go delete mode 100644 modules/postgres/cmd/postgres-provider/grants_moved_test.go delete mode 100644 modules/postgres/cmd/postgres-provider/harness.go delete mode 100644 modules/postgres/cmd/postgres-provider/harness_same_test.go delete mode 100644 modules/postgres/cmd/postgres-provider/retirement.go delete mode 100644 modules/postgres/cmd/postgres-provider/retirement_test.go delete mode 100644 modules/postgres/cmd/postgres-provider/standing_test.go diff --git a/modules/postgres/cmd/postgres-provider/early_test.go b/modules/postgres/cmd/postgres-provider/early_test.go deleted file mode 100644 index 592c8b53..00000000 --- a/modules/postgres/cmd/postgres-provider/early_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -// The moved path started before its runtime answers asks again until it does (issue 943). - -import ( - "context" - "errors" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// earlyBus refuses its first watches as the runtime does before Serve runs, then answers as the bucket bus does. -type earlyBus struct { - *bucketBus - refuse int -} - -func (b *earlyBus) Watch(ctx context.Context, bucket string, handle func(string, []byte)) error { - if b.refuse > 0 { - b.refuse-- - return errors.New("before Serve: a bundle reaches the bus through the runtime that launched it") - } - return b.bucketBus.Watch(ctx, bucket, handle) -} - -func TestAMovedPathStartedTooEarlyAsksAgainUntilTheRuntimeAnswers(t *testing.T) { - w := newMovedWorld(t, map[[2]string]string{}) - w.run.FirstRetry = time.Millisecond - w.run.loaded = false - bus := &earlyBus{bucketBus: w.bus, refuse: 3} - p := provider.New(provider.Options{Node: "novox", Module: "postgres", Bus: w.bus, - Vault: provider.ConsumersFile{Paths: []string{w.file}}, Make: w.run.Make, Gone: w.run.Gone, - Every: time.Hour, Logf: func(string, ...any) {}}) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- w.run.Run(ctx, p, bus) }() - deadline := time.After(time.Second) - for { - w.run.mu.Lock() - loaded := w.run.loaded - w.run.mu.Unlock() - if loaded { - break - } - select { - case err := <-done: - cancel() - t.Fatalf("Run stopped before the runtime answered: %v", err) - case <-deadline: - cancel() - t.Fatal("the plans were never watched") - case <-time.After(time.Millisecond): - } - } - cancel() - <-done - if bus.refuse != 0 { - t.Fatalf("%d refusals left", bus.refuse) - } -} diff --git a/modules/postgres/cmd/postgres-provider/grants_moved_test.go b/modules/postgres/cmd/postgres-provider/grants_moved_test.go deleted file mode 100644 index 0d2b8a6f..00000000 --- a/modules/postgres/cmd/postgres-provider/grants_moved_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -// postgres handed to the provisioner still serves the consumers the controller's grant file gives (issue 937): a -// consumer not moved has no plan naming postgres, only its grant, until R1 (issue 682). - -import ( - "encoding/json" - "os" - "path/filepath" - "reflect" - "testing" - "time" -) - -// granting gives the moved world a grant file, as the controller writes it for the consumers not moved. -func (w *movedWorld) granting(given ...map[string]any) { - if w.h.Receives == "" { - w.h.Receives = filepath.Join(w.t.TempDir(), "mesh.json") - } - dir := filepath.Dir(w.h.Receives) - for _, g := range given { - secret := filepath.Join(dir, g["as"].(string)+".secret") - if err := os.WriteFile(secret, []byte("pw-"+g["as"].(string)+"\n"), 0o600); err != nil { - w.t.Fatal(err) - } - g["secret"] = secret - } - if given == nil { - given = []map[string]any{} - } - raw, _ := json.Marshal(map[string]any{"requirement": ProvisionName, "given": given}) - if err := os.WriteFile(w.h.Receives, raw, 0o600); err != nil { - w.t.Fatal(err) - } -} - -// settled is enough passes and time for a retirement to be stable. -func (w *movedWorld) settled() { - for i := 0; i < 8; i++ { - w.pass() - } - w.now = w.now.Add(15 * time.Minute) - w.pass() - w.pass() -} - -// The logins the grant file gives are asked for: made as the grant path makes them, never offered for retirement, -// beside a moved consumer the helper makes from its plan. -func TestAConsumerOnlyTheGrantFileGivesIsAskedForAndMade(t *testing.T) { - bound, logins := bindings(1) - w := newMovedWorld(t, bound) - w.a.inv.Active = []string{"mesh_ace_baserow", "mesh_ace_letta", logins[0]} - w.bus.bind("ace", "app1", nil) - w.granting( - map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}}, - map[string]any{"as": "mesh_ace_letta", "node": "ace", - "values": map[string]any{"name": "letta", "extensions": []any{"vector"}}}, - ) - w.settled() - if len(w.a.removed) != 0 || w.h.r.waiting != nil { - t.Fatalf("retired %v, waiting %+v: the grant file's logins were counted unasked", w.a.removed, w.h.r.waiting) - } - made := map[string]Provision{} - for _, c := range w.a.created { - made[c.As] = c - } - letta := made["mesh_ace_letta"] - if letta.Password != "pw-mesh_ace_letta" || letta.Consumer != "ace" || - !reflect.DeepEqual(letta.Values, map[string]any{"name": "letta", "extensions": []any{"vector"}}) { - t.Fatalf("made %+v for letta, as its grant gives it", letta) - } - if _, ok := made["mesh_ace_baserow"]; !ok { - t.Fatalf("baserow not made: %+v", w.a.created) - } - if _, ok := made[logins[0]]; !ok { - t.Fatalf("the moved consumer not made: %+v", w.a.created) - } -} - -// A login both a plan and the grant file name is the plan's: made once, with the plan's credential. -func TestALoginAPlanNamesIsNotMadeFromTheGrantFile(t *testing.T) { - bound, logins := bindings(1) - w := newMovedWorld(t, bound) - w.bus.bind("ace", "app1", nil) - w.granting(map[string]any{"as": logins[0], "node": "ace", "values": map[string]any{"name": "app1"}}) - w.pass() - w.pass() - for _, c := range w.a.created { - if c.As == logins[0] && c.Password != "pw-app1" { - t.Fatalf("made %s with the grant's password %q, not the plan's", c.As, c.Password) - } - } -} - -// An unreadable grant file is not nobody asking (issue 241): nothing is retired while it cannot be read. -func TestAnUnreadableGrantFileRetiresNothing(t *testing.T) { - bound, logins := bindings(1) - w := newMovedWorld(t, bound) - w.a.inv.Active = []string{"mesh_ace_baserow", logins[0]} - w.bus.bind("ace", "app1", nil) - w.h.Receives = filepath.Join(t.TempDir(), "absent.json") - w.settled() - if len(w.a.removed) != 0 || w.h.r.waiting != nil { - t.Fatalf("retired %v, waiting %+v with the grant file unreadable", w.a.removed, w.h.r.waiting) - } -} - -// A login the grant file stops giving is retired by the usual rules: stable, then retired, never dropped. -func TestALoginTheGrantFileStopsGivingIsRetired(t *testing.T) { - bound, _ := bindings(1) - w := newMovedWorld(t, bound) - w.bus.bind("ace", "app1", nil) - w.granting( - map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}}, - map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}}, - ) - w.pass() - w.granting(map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}}) - w.settled() - if !reflect.DeepEqual(w.a.removed, []string{"mesh_ace_letta"}) || len(w.a.deleted) != 0 { - t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) - } -} diff --git a/modules/postgres/cmd/postgres-provider/harness.go b/modules/postgres/cmd/postgres-provider/harness.go deleted file mode 100644 index 5e348ce5..00000000 --- a/modules/postgres/cmd/postgres-provider/harness.go +++ /dev/null @@ -1,574 +0,0 @@ -package main - -// The reconcile loop every provider shares, as the TypeScript SDK's runProvisioner runs it -// (@novox/mesh-sdk/provisioner, 0.1.12). The Go SDK has no provisioner yet, so this module carries -// the loop itself, line for line in behaviour; when the Go SDK grows one, this file is what moves -// there (novox/hq ADR 0039: the loop is the SDK's, the adapter is the module's). -// -// Read the contributions the mesh delivered; bring each consumer's resource into being through the -// adapter, under the login and password the mesh minted; retire what the mesh no longer asks for. -// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).** -// -// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer -// whose create, check or secret has failed without one success in between for FailingAfter is -// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and -// again every SayAgainEvery while it lasts; the first success after that is `provisioner.recovered`. -// The controller keeps the newest per provider and consumer and `status` names it. On 2026-10-05 the -// identity provider failed every consumer 31,000 times in a day and said so only in its journal -// (novox/hq issue 179). -// -// **A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person -// (novox/hq ADR 0230, the operator's model of 2026-10-06).** Retiring disables its access — reversibly — -// and marks its login and data "to delete" with when and why; nothing is deleted. Asked for again, the -// ordinary create re-enables it as it was. It is retired only once the same set has gone unasked in -// StablePasses consecutive passes and for StableFor, and a set larger than the bound waits for a -// person. retirement.go. -// -// Carried, identical, by every Go provider until the Go SDK has the loop: postgres and keycloak. -// Each module's `harness_same_test.go` fails when its copy and the other's differ (this file and -// retirement.go). - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/url" - "os" - "strings" - "sync" - "time" -) - -// Provision is one consumer's resource to bring into being — everything the mesh derived and delivered. -type Provision struct { - // As is the login the mesh derived and gave the consumer to present. - As string - // Password is the one the mesh minted, read from the file the host unsealed. - Password string - // Values are what the consumer contributed (e.g. {"name": "letta", "extensions": ["vector"]}). - Values map[string]any - // Derived is what this provider's own definition derives for the consumer (novox/hq ADR 0201). - Derived map[string]any - // At is where the consumer is; Consumer is its node. - At string - Consumer string -} - -// Adapter is the per-service half. -type Adapter interface { - // Create brings the consumer into being under the mesh's login and password — and, for one that - // was retired, re-enables it as it was and clears its mark to delete. - Create(ctx context.Context, p Provision) error - // Retire disables the consumer's access, reversibly, and marks its login and data to delete with - // when and why. It deletes nothing (novox/hq ADR 0230). derived is what the mesh last derived, - // remembered here; nil for a consumer this process did not make. - Retire(ctx context.Context, as string, derived map[string]any, why string, at time.Time) error - // Holds says whether the backend still holds the consumer exactly as p says. Read-only. - Holds(ctx context.Context, p Provision) (bool, error) - // Inventory is what the backend holds that the mesh made: consumers active and retired, read from - // the backend itself so a restart forgets nothing. Read-only. - Inventory(ctx context.Context) (Inventory, error) - // Delete removes one retired consumer — its login and its data — for good. Only ever asked by a - // person, through `cleanup delete`; the harness has checked it is retired and not asked for. - Delete(ctx context.Context, r Retired) (freedBytes int64, err error) -} - -// Harness is the loop's settings and memory. -type Harness struct { - Resource string - Receives string - Adapter Adapter - Every time.Duration // 5s - VerifyEvery time.Duration // 60s - HoldsTimeout time.Duration // 30s - Log func(format string, args ...any) - Now func() time.Time - // Announce publishes one of the provider's standing events; nil announces nothing. Node is the - // machine this provider runs on, said in each. - Announce func(event string, body map[string]any) - Node string - // FailingAfter is how long a consumer fails without a success before it is announced (5m); - // SayAgainEvery is how often it is announced again while it lasts (15m), so a controller that - // missed the first hears the next, and a standing nobody repeats can be told from one that holds. - FailingAfter time.Duration - SayAgainEvery time.Duration - // StablePasses (5) is how many consecutive passes must see the same set no longer asked for, and - // StableFor (10m) how long it must have held since the first of them, before it is retired. RetireAtOnce (3) and RetireFraction (0.5) bound what is retired without a person: - // more consumers than RetireAtOnce, or — where more than one is held — a larger share of those held - // than RetireFraction, waits for `retire approve` (novox/hq ADR 0230). - StablePasses int - StableFor time.Duration - RetireAtOnce int - RetireFraction float64 - - // mu is held by a pass and by every tool a person asks, so the two never interleave. - mu sync.Mutex - verifiedAt time.Time - r retirement - applied map[string]appliedEntry - lost map[string]brake - waiting map[string]int - failing map[string]failure - trouble map[string]*standing - cleared map[string]bool - lastWarning string -} - -// standing is one consumer's unbroken run of failures: since when, how often, and the last error. -type standing struct { - node string - since time.Time - attempts int - class string - text string - saidAt time.Time -} - -// The events a provider's standing is announced as (novox/hq ADR 0224). The controller derives the -// permission to emit them for every module that receives contributions; no manifest lists them. -const ( - EventFailing = "provisioner.failing" - EventRecovered = "provisioner.recovered" -) - -// The classes of error a standing is announced with: what a person reading `status` needs to know -// before reading the journal. An adapter may say better (Classifier). -const ( - ClassCredentials = "credentials-rejected" - ClassUnreachable = "unreachable" - ClassSecret = "secret-unreadable" - ClassRefused = "refused" -) - -// Classifier is an adapter that can say what class an error of its own is. -type Classifier interface { - Class(err error) string -} - -type appliedEntry struct { - hash string - derived map[string]any - node string -} - -type brake struct { - times int - nextAt time.Time -} - -type failure struct { - text string - times int -} - -// The longest a consumer whose create keeps failing to satisfy holds waits between checks. -const maxBackoff = time.Hour - -// How many passes a secret may be unreadable before it stops being called a race (issue 225), and -// once said loudly, how often it is repeated. The same cadence quiets a create that keeps failing -// the same way. -const ( - patiently = 12 - loudlyEvery = 240 -) - -type contribution struct { - As string `json:"as"` - Secret string `json:"secret"` - Node string `json:"node"` - At string `json:"at"` - Values map[string]any `json:"values"` - Derived map[string]any `json:"derived"` -} - -func (h *Harness) init() { - if h.Every == 0 { - h.Every = 5 * time.Second - } - if h.VerifyEvery == 0 { - h.VerifyEvery = time.Minute - } - if h.HoldsTimeout == 0 { - h.HoldsTimeout = 30 * time.Second - } - if h.Now == nil { - h.Now = time.Now - } - if h.FailingAfter == 0 { - h.FailingAfter = 5 * time.Minute - } - if h.SayAgainEvery == 0 { - h.SayAgainEvery = 15 * time.Minute - } - if h.StablePasses == 0 { - h.StablePasses = 5 - } - if h.StableFor == 0 { - h.StableFor = StableFor - } - if h.RetireAtOnce == 0 { - h.RetireAtOnce = 3 - } - if h.RetireFraction == 0 { - h.RetireFraction = 0.5 - } - if h.Log == nil { - h.Log = func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) } - } - if h.applied == nil { - h.applied = map[string]appliedEntry{} - h.lost = map[string]brake{} - h.waiting = map[string]int{} - h.failing = map[string]failure{} - h.trouble = map[string]*standing{} - h.cleared = map[string]bool{} - h.r.retired = map[string]retiredEntry{} - } -} - -// Run reconciles until ctx ends. One consumer's failure never stops the others'. -func (h *Harness) Run(ctx context.Context) { - h.init() - for { - h.Reconcile(ctx) - select { - case <-ctx.Done(): - return - case <-time.After(h.Every): - } - } -} - -func (h *Harness) say(format string, args ...any) { - h.Log("[provisioner:"+h.Resource+"] "+format, args...) -} - -func (h *Harness) warn(why string) { - if why == h.lastWarning { - return - } - if why != "" { - h.say("%s; nothing applied or removed until it can be read", why) - } else { - h.say("contributions file readable again") - } - h.lastWarning = why -} - -// readContributions answers the consumers asked for, or nil when the file says nothing usable. -// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can retire anything. -func (h *Harness) readContributions() []contribution { - raw, err := os.ReadFile(h.Receives) - if err != nil { - h.warn(fmt.Sprintf("contributions file unreadable (%s): %v", h.Receives, err)) - return nil - } - var doc struct { - Requirement string `json:"requirement"` - Given json.RawMessage `json:"given"` - } - if err := json.Unmarshal(raw, &doc); err != nil { - h.warn(fmt.Sprintf("contributions file is not JSON (%s): %v", h.Receives, err)) - return nil - } - if doc.Requirement != "" && doc.Requirement != h.Resource { - h.warn(fmt.Sprintf("%s is for %s, not %s", h.Receives, doc.Requirement, h.Resource)) - return nil - } - var given []contribution - if len(doc.Given) == 0 || string(doc.Given) == "null" || json.Unmarshal(doc.Given, &given) != nil { - h.warn(fmt.Sprintf("%s has no given list", h.Receives)) - return nil - } - h.warn("") - out := []contribution{} - for _, g := range given { - // No `as` is not a credential grant: nothing to create for it. - if g.As != "" && g.Secret != "" { - out = append(out, g) - } - } - return out -} - -func hashOf(as, password string, values, derived map[string]any) string { - // Derived is in the hash: a provider that renames what it derives gave a different resource. - b, _ := json.Marshal([]any{as, password, orEmpty(values), orEmpty(derived)}) - return string(b) -} - -func orEmpty(m map[string]any) map[string]any { - if m == nil { - return map[string]any{} - } - return m -} - -// Reconcile is one pass. -func (h *Harness) Reconcile(ctx context.Context) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - given := h.readContributions() - if given == nil { - // Not a result: the passes that must agree before anything is retired start again. - h.r.key, h.r.count = "", 0 - return - } - want := map[string]bool{} - for _, g := range given { - want[g.As] = true - } - h.seed(ctx, want) - h.apply(ctx, given) - - // Retire what the mesh has stably stopped asking for — within the bound, or with a person. - h.retireUnasked(ctx, want) - h.r.lastWant = want - for as := range h.failing { - if !want[as] { - delete(h.failing, as) - } - } - // A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by - // anyone: said, so a standing the controller keeps for it is cleared rather than left naming a - // consumer that is gone. One still held is said recovered when it is retired. - for as := range h.trouble { - if _, held := h.applied[as]; !want[as] && !held { - h.recovered(as, "no longer asked for") - } - } -} - -// apply makes each consumer the grant file gives, and checks every VerifyEvery that the backend still holds it. The -// caller holds h.mu. One consumer's failure never stops the others'. -func (h *Harness) apply(ctx context.Context, given []contribution) { - verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery - if verifying { - h.verifiedAt = h.Now() - } - - for _, g := range given { - raw, err := os.ReadFile(g.Secret) - if err != nil { - // A secret the host has not written yet is a race on the first pass; past a minute it is - // a person's to look at, and said so (novox/hq issue 225). - n := h.waiting[g.As] + 1 - h.waiting[g.As] = n - if n <= patiently { - h.say("%s: secret not readable yet (%s): %v", g.As, g.Secret, err) - } else if n == patiently+1 || n%loudlyEvery == 0 { - h.say("%s: CANNOT READ the secret after %d attempts (%s): %v. This is not a race any more — "+ - "nothing has been provisioned for this consumer and nothing will be until somebody looks. "+ - "Check who owns the file and who this process runs as (novox/hq issue 225)", g.As, n, g.Secret, err) - } - h.failed(g.As, g.Node, ClassSecret, fmt.Sprintf("secret not readable (%s): %v", g.Secret, err)) - continue - } - delete(h.waiting, g.As) - password := strings.TrimSuffix(string(raw), "\n") - p := Provision{As: g.As, Password: password, Values: orEmpty(g.Values), Derived: orEmpty(g.Derived), At: g.At, Consumer: g.Node} - hash := hashOf(g.As, password, g.Values, g.Derived) - - reapplying := 0 - if was, ok := h.applied[g.As]; ok && was.hash == hash { - if !verifying { - continue - } - b, braked := h.lost[g.As] - if braked && h.Now().Before(b.nextAt) { - continue - } - hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) - held, err := h.Adapter.Holds(hctx, p) - timedOut := errors.Is(hctx.Err(), context.DeadlineExceeded) - cancel() - if err != nil { - // Unable to ask is not evidence of loss. A backend that timed out will time out for - // the next consumer too, so the rest of this pass is not asked. - text := scrub(err, password) - h.say("%s: could not check the backend, will ask again: %s", g.As, text) - h.failed(g.As, g.Node, h.classOf(err, text), text) - if timedOut { - verifying = false - } - continue - } - if held { - delete(h.lost, g.As) - h.succeeded(g.As) - continue - } - reapplying = b.times + 1 - if reapplying == 1 { - h.say("%s: the backend no longer holds it; applying again", g.As) - } else { - h.say("%s: still not held after being applied again (%d times in a row) — create does not "+ - "produce what holds checks; applying again", g.As, reapplying) - } - } - if err := h.Adapter.Create(ctx, p); err != nil { - text := scrub(err, password) - f := h.failing[g.As] - if f.text != text { - f = failure{text: text} - } - f.times++ - h.failing[g.As] = f - // Said each time it changes, and while it stays the same, as rarely as a lost secret. - if f.times == 1 || f.times%loudlyEvery == 0 { - h.say("%s: create failed, will retry: %s", g.As, text) - } - h.failed(g.As, g.Node, h.classOf(err, text), text) - if reapplying > 0 { - h.lost[g.As] = brake{times: reapplying - 1} - } - continue - } - if f, was := h.failing[g.As]; was { - h.say("%s: created, after %d failed attempt(s)", g.As, f.times) - delete(h.failing, g.As) - } - h.succeeded(g.As) - h.applied[g.As] = appliedEntry{hash: hash, derived: p.Derived, node: g.Node} - h.reenabled(g.As, g.Node) - if reapplying == 0 { - delete(h.lost, g.As) - } else { - wait := h.VerifyEvery << (reapplying - 1) - if wait > maxBackoff || wait <= 0 { - wait = maxBackoff - } - h.lost[g.As] = brake{times: reapplying, nextAt: h.Now().Add(wait)} - if reapplying > 1 { - h.say("%s: next check in %s", g.As, wait.Round(time.Second)) - } - } - } -} - -// failed counts one more failure in a consumer's unbroken run, and announces the run once it has -// lasted FailingAfter — then again every SayAgainEvery while it lasts. -func (h *Harness) failed(as, node, class, text string) { - now := h.Now() - s := h.trouble[as] - if s == nil { - s = &standing{since: now} - h.trouble[as] = s - } - s.node, s.class, s.text = node, class, text - s.attempts++ - if now.Sub(s.since) < h.FailingAfter { - return - } - if !s.saidAt.IsZero() && now.Sub(s.saidAt) < h.SayAgainEvery { - return - } - first := s.saidAt.IsZero() - s.saidAt = now - if first { - h.say("%s: FAILING for %s (%d attempts, %s): %s. Announced as %s; `status` names it until it "+ - "succeeds (novox/hq ADR 0224)", as, now.Sub(s.since).Round(time.Second), s.attempts, class, text, EventFailing) - } - h.announce(EventFailing, map[string]any{ - "provider": h.Resource, "provider-node": h.Node, - "consumer": as, "node": node, - "class": class, "error": clip(text), - "since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts, - }) -} - -// succeeded ends a consumer's run of failures; one that was announced is announced recovered. -// -// **And the first success for a consumer since this process started is announced too**, failing or -// not: a provider that announced a failure and was restarted has forgotten it, and without this the -// controller would name the consumer failing for ever after it recovered unheard. -func (h *Harness) succeeded(as string) { - if h.trouble[as] == nil && !h.cleared[as] { - h.cleared[as] = true - h.announce(EventRecovered, map[string]any{ - "provider": h.Resource, "provider-node": h.Node, "consumer": as, "why": "first-success", - }) - return - } - h.cleared[as] = true - h.recovered(as, "") -} - -func (h *Harness) recovered(as, why string) { - s := h.trouble[as] - if s == nil { - return - } - delete(h.trouble, as) - if s.saidAt.IsZero() { - return // never announced, so there is nothing to take back - } - if why == "" { - h.say("%s: recovered after %s and %d failed attempt(s)", as, h.Now().Sub(s.since).Round(time.Second), s.attempts) - } - body := map[string]any{ - "provider": h.Resource, "provider-node": h.Node, "consumer": as, "node": s.node, - "since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts, - } - if why != "" { - body["why"] = why - } - h.announce(EventRecovered, body) -} - -func (h *Harness) announce(event string, body map[string]any) { - if h.Announce != nil { - h.Announce(event, body) - } -} - -// classOf is an error's class: the adapter's word when it has one, else read from the text. -func (h *Harness) classOf(err error, text string) string { - if c, ok := h.Adapter.(Classifier); ok { - if class := c.Class(err); class != "" { - return class - } - } - return ClassOf(text) -} - -// ClassOf reads an error's class from its text — the words the backends the mesh runs use. -func ClassOf(text string) string { - t := strings.ToLower(text) - for _, w := range []string{"invalid_grant", "invalid user credentials", "password authentication failed", - "authentication failed", "unauthorized", " 401"} { - if strings.Contains(t, w) { - return ClassCredentials - } - } - for _, w := range []string{"connection refused", "no such host", "i/o timeout", "deadline exceeded", - "connection reset", "network is unreachable", "no route to host", "eof"} { - if strings.Contains(t, w) { - return ClassUnreachable - } - } - return ClassRefused -} - -// clip keeps an announced error to what belongs in a status line. -func clip(text string) string { - const most = 300 - if len(text) <= most { - return text - } - return text[:most] + "…" -} - -// scrub is an error's text with the consumer's password removed, raw and URL-encoded. -func scrub(err error, password string) string { - text := err.Error() - if password == "" { - return text - } - for _, form := range []string{password, url.QueryEscape(password), url.PathEscape(password)} { - text = strings.ReplaceAll(text, form, "***") - } - return text -} diff --git a/modules/postgres/cmd/postgres-provider/harness_same_test.go b/modules/postgres/cmd/postgres-provider/harness_same_test.go deleted file mode 100644 index d9977141..00000000 --- a/modules/postgres/cmd/postgres-provider/harness_same_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -// The provisioner loop is carried, identical, by every Go provider until the Go SDK has it -// (harness.go, retirement.go, and retirement_test.go which tests it). Two copies drift the moment one -// is fixed and the other is not — and the one left behind is the provider that fails a consumer -// without saying so (novox/hq ADR 0224), or retires one it should not (ADR 0230). This holds them to -// one text. Skipped where keycloak is not beside this module, as in a build of this one alone. - -import ( - "bytes" - "errors" - "io/fs" - "os" - "testing" -) - -func TestTheHarnessIsTheSameAsKeycloaks(t *testing.T) { - for _, file := range []string{"harness.go", "retirement.go", "retirement_test.go"} { - theirs, err := os.ReadFile("../../../keycloak/cmd/keycloak-provider/" + file) - if errors.Is(err, fs.ErrNotExist) { - t.Skip("keycloak is not beside this module") - } - if err != nil { - t.Fatal(err) - } - ours, err := os.ReadFile(file) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(ours, theirs) { - t.Fatalf("%s differs from keycloak/cmd/keycloak-provider/%s: change both, identically", file, file) - } - } -} diff --git a/modules/postgres/cmd/postgres-provider/harness_test.go b/modules/postgres/cmd/postgres-provider/harness_test.go index 7e4733b2..26b0246c 100644 --- a/modules/postgres/cmd/postgres-provider/harness_test.go +++ b/modules/postgres/cmd/postgres-provider/harness_test.go @@ -1,7 +1,9 @@ package main import ( + "context" "encoding/json" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "os" "path/filepath" "strings" @@ -14,20 +16,38 @@ type world struct { dir string receives string now time.Time - h *Harness + h *harness.Harness a *recorder said []string } func newWorld(t *testing.T) *world { - w := &world{t: t, dir: t.TempDir(), now: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), a: &recorder{held: true}} + w := &world{t: t, dir: t.TempDir(), now: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), a: &recorder{}} w.receives = filepath.Join(w.dir, "mesh.json") - w.h = &Harness{Resource: "postgres-database", Receives: w.receives, Adapter: w.a, + w.h = &harness.Harness{Resource: "postgres-database", Receives: w.receives, Adapter: w.a, Now: func() time.Time { return w.now }, Log: func(f string, a ...any) { w.said = append(w.said, f) }} return w } +// recorder is an adapter that remembers what it was asked to create; the loop's own behaviour is proven in the SDK's +// harness package, and this module's tests hold only what the module adds (issue 849). +type recorder struct{ created []harness.Provision } + +func (r *recorder) Create(_ context.Context, p harness.Provision) error { + r.created = append(r.created, p) + return nil +} + +func (r *recorder) Retire(context.Context, string, map[string]any, string, time.Time) error { + return nil +} +func (r *recorder) Holds(context.Context, harness.Provision) (bool, error) { return true, nil } +func (r *recorder) Inventory(context.Context) (harness.Inventory, error) { + return harness.Inventory{}, nil +} +func (r *recorder) Delete(context.Context, harness.Retired) (int64, error) { return 0, nil } + func (w *world) give(given ...map[string]any) { for _, g := range given { secret := filepath.Join(w.dir, g["as"].(string)+".secret") @@ -45,20 +65,6 @@ func (w *world) give(given ...map[string]any) { } } -func TestAConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}}) - w.h.Reconcile(ctx) - w.h.Reconcile(ctx) - if len(w.a.created) != 1 { - t.Fatalf("created %d times", len(w.a.created)) - } - p := w.a.created[0] - if p.As != "mesh_ace_letta" || p.Password != "pw-mesh_ace_letta" || p.Consumer != "ace" { - t.Fatalf("%+v", p) - } -} - func TestAddingExtensionsAppliesTheConsumerAgain(t *testing.T) { w := newWorld(t) w.give(map[string]any{"as": "mesh_ace_letta", "values": map[string]any{"name": "letta"}}) @@ -73,106 +79,16 @@ func TestAddingExtensionsAppliesTheConsumerAgain(t *testing.T) { } } -func TestAConsumerNoLongerAskedForIsRetiredOnceStable(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}) - w.h.Reconcile(ctx) - w.give(map[string]any{"as": "a"}) - w.settle() // five passes and ten minutes - if strings.Join(w.a.removed, ",") != "b" { - t.Fatal(w.a.removed) - } - // Only a file that says nobody asks retires the last one. - w.give() - w.settle() - if strings.Join(w.a.removed, ",") != "b,a" { - t.Fatal(w.a.removed) - } -} - -func TestNothingReadIsNotNobodyAsking(t *testing.T) { - for name, content := range map[string]string{ - "unreadable": "", - "not JSON": "{", - "no given": `{"requirement": "postgres-database"}`, - "another": `{"requirement": "mssql-database", "given": []}`, - } { - t.Run(name, func(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "a"}) - w.h.Reconcile(ctx) - if content == "" { - os.Remove(w.receives) - } else { - os.WriteFile(w.receives, []byte(content), 0o600) - } - w.h.Reconcile(ctx) - if len(w.a.removed) != 0 { - t.Fatalf("retired %v on a file it could not use", w.a.removed) - } - }) - } -} - -func TestALostConsumerIsAppliedAgainAndBraked(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "a"}) - w.h.Reconcile(ctx) - w.a.held = false - w.now = w.now.Add(2 * time.Minute) - w.h.Reconcile(ctx) - if len(w.a.created) != 2 { - t.Fatalf("created %d times", len(w.a.created)) - } - // Still not held a minute later: applied again, then braked for two minutes. - w.now = w.now.Add(61 * time.Second) - w.h.Reconcile(ctx) - w.now = w.now.Add(61 * time.Second) - w.h.Reconcile(ctx) - if len(w.a.created) != 3 { - t.Fatalf("not braked: created %d times", len(w.a.created)) - } -} - -func TestAFailingCreateIsRetriedAndSaidOnce(t *testing.T) { - w := newWorld(t) - w.a.failing = &pgErr{"extension \"nope\" refused, password pw-a"} - w.give(map[string]any{"as": "a"}) - for i := 0; i < 5; i++ { - w.h.Reconcile(ctx) - } - n := 0 - for _, s := range w.said { - if strings.Contains(s, "create failed") { - n++ - } - } - if n != 1 { - t.Fatalf("said %d times", n) - } - w.a.failing = nil - w.h.Reconcile(ctx) - if len(w.a.created) != 1 { - t.Fatal("not retried") - } -} - type pgErr struct{ s string } func (e *pgErr) Error() string { return e.s } -func TestScrubRemovesThePassword(t *testing.T) { - if got := scrub(&pgErr{"bad pw a/b c and a%2Fb+c"}, "a/b c"); strings.Contains(got, "a/b c") || strings.Contains(got, "a%2Fb+c") { - t.Fatal(got) - } -} - func TestProvisionerCreatesTheDatabaseThenItsExtensions(t *testing.T) { f, c := newFake(admin) f.answer(`FROM pg_available_extensions`, []string{"name"}, []string{"vector"}) var events []string a := provisioner{pg: c, announce: func(e string, _ map[string]string) { events = append(events, e) }} - p := Provision{As: "mesh_ace_letta", Password: "pw", Values: map[string]any{"name": "letta", "extensions": []any{"vector"}}} + p := harness.Provision{As: "mesh_ace_letta", Password: "pw", Values: map[string]any{"name": "letta", "extensions": []any{"vector"}}} if err := a.Create(ctx, p); err != nil { t.Fatal(err) } @@ -209,3 +125,12 @@ func TestProvisionerCreatesTheDatabaseThenItsExtensions(t *testing.T) { t.Fatal(f.statements()) } } + +func listHas(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} diff --git a/modules/postgres/cmd/postgres-provider/live_test.go b/modules/postgres/cmd/postgres-provider/live_test.go index 33e337c1..38f91ef5 100644 --- a/modules/postgres/cmd/postgres-provider/live_test.go +++ b/modules/postgres/cmd/postgres-provider/live_test.go @@ -12,6 +12,7 @@ package main // COMMIT (novox/hq ADR 0230). import ( + "git.novox.be/novox/mesh-sdk/go/provider/harness" "net/url" "os" "testing" @@ -31,7 +32,7 @@ func TestLive(t *testing.T) { t.Fatal(err) } a := provisioner{pg: c, announce: func(string, map[string]string) {}} - p := Provision{As: "mesh_test_letta", Password: "consumer-pw", Values: map[string]any{"name": "letta", "extensions": []any{"vector"}}} + p := harness.Provision{As: "mesh_test_letta", Password: "consumer-pw", Values: map[string]any{"name": "letta", "extensions": []any{"vector"}}} // vector is not trusted: the consumer cannot install it itself. if err := c.CreateDatabaseAndRole(ctx, p.As, p.As, p.Password); err != nil { @@ -68,11 +69,11 @@ func TestLive(t *testing.T) { } // A neighbour that must survive everything below untouched. - other := Provision{As: "mesh_test_other", Password: "other-pw"} + other := harness.Provision{As: "mesh_test_other", Password: "other-pw"} if err := a.Create(ctx, other); err != nil { t.Fatal(err) } - defer c.DeleteRetired(ctx, Retired{Consumer: other.As}) //nolint — best effort, after a retire below + defer c.DeleteRetired(ctx, harness.Retired{Consumer: other.As}) //nolint — best effort, after a retire below at := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) if err := a.Retire(ctx, p.As, nil, "the mesh stopped asking for it", at); err != nil { @@ -89,7 +90,7 @@ func TestLive(t *testing.T) { if err != nil { t.Fatal(err) } - var found *Retired + var found *harness.Retired for i := range inv.Retired { if inv.Retired[i].Consumer == p.As { found = &inv.Retired[i] @@ -102,7 +103,7 @@ func TestLive(t *testing.T) { t.Fatalf("%+v", inv) } // An active consumer is never deleted, whatever is asked. - if _, err := c.DeleteRetired(ctx, Retired{Consumer: other.As}); err == nil { + if _, err := c.DeleteRetired(ctx, harness.Retired{Consumer: other.As}); err == nil { t.Fatal("deleted an active consumer") } @@ -125,7 +126,7 @@ func TestLive(t *testing.T) { if err := a.Retire(ctx, p.As, nil, "again", at); err != nil { t.Fatal(err) } - freed, err := a.Delete(ctx, Retired{Consumer: p.As, Kind: KindConsumer}) + freed, err := a.Delete(ctx, harness.Retired{Consumer: p.As, Kind: harness.KindConsumer}) if err != nil || freed <= 0 { t.Fatal(freed, err) } @@ -145,7 +146,7 @@ func TestLive(t *testing.T) { t.Fatal(err) } inv, _ = a.Inventory(ctx) - var setAside *Retired + var setAside *harness.Retired for i := range inv.Retired { if inv.Retired[i].Consumer == aside { setAside = &inv.Retired[i] diff --git a/modules/postgres/cmd/postgres-provider/main.go b/modules/postgres/cmd/postgres-provider/main.go index c3f9e09e..a1eb199e 100644 --- a/modules/postgres/cmd/postgres-provider/main.go +++ b/modules/postgres/cmd/postgres-provider/main.go @@ -10,6 +10,7 @@ import ( "context" "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "os" "time" @@ -32,12 +33,12 @@ func main() { } return } - var h *Harness + var h *harness.Harness if consumers := handedConsumersFile(os.Getenv, os.Stat); consumers != "" { // Handed to the provisioner (novox/hq ADR 0373): the provider helper answers the consumers its // machine's client fills the file with, and the grant file still gives the consumers not moved // (issue 937). Retirement is the harness's, never run, so its guards and its tools stand (issue 780). - h = &Harness{ + h = &harness.Harness{ Resource: ProvisionName, Receives: os.Getenv("MESH_RECEIVES"), Adapter: provisioner{pg: pg, announce: announce}, @@ -62,7 +63,7 @@ func main() { } else if receives := os.Getenv("MESH_RECEIVES"); receives == "" { say("MESH_RECEIVES is not set — the provisioner cannot run without it") } else { - h = &Harness{ + h = &harness.Harness{ Resource: "postgres-database", Receives: receives, Adapter: provisioner{pg: pg, announce: announce}, @@ -93,7 +94,7 @@ func main() { go pg.RunReader(context.Background(), os.Getenv("MESH_RECEIVES"), time.Minute, say) // The retirement tools beside postgres's own: what is retired here, approving or rejecting what waits // for a person, and deleting a retired consumer (novox/hq ADR 0230). - if err := stdio.Serve("", append(Tools(pg), RetirementTools(h)...)); err != nil { + if err := stdio.Serve("", append(Tools(pg), harness.RetirementTools(h)...)); err != nil { say("%v", err) os.Exit(1) } diff --git a/modules/postgres/cmd/postgres-provider/moved.go b/modules/postgres/cmd/postgres-provider/moved.go index 25fd1ea2..a639f7b6 100644 --- a/modules/postgres/cmd/postgres-provider/moved.go +++ b/modules/postgres/cmd/postgres-provider/moved.go @@ -16,14 +16,8 @@ package main // that cannot be read is not nobody asking (issue 241): nothing is retired until it is read again. import ( - "context" - "encoding/json" - "errors" - "fmt" - "sync" - "time" - "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provider/harness" ) // ProvisionName is what postgres provides, as its manifest says. @@ -32,286 +26,12 @@ const ProvisionName = "postgres-database" // servedPort is the port its manifest serves (`serves.postgres-database.port`). const servedPort = 5432 -// movedRun is postgres on the provider helper, with the harness's own memory and retirement behind it: what is held, -// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's -// Make and Gone and the loop below drive the same state, so the guards are the harness's, not a copy. -type movedRun struct { - h *Harness - - // FirstRetry is the first wait before a watch asked too early is asked again (issue 943); a second when zero. - FirstRetry time.Duration - - mu sync.Mutex // guards assignments and loaded - assignments map[string][]byte - loaded bool - verified map[string]time.Time -} - -func newMovedRun(h *Harness) *movedRun { - h.mu.Lock() - h.init() - h.mu.Unlock() - return &movedRun{h: h, assignments: map[string][]byte{}, verified: map[string]time.Time{}} -} - -// Make brings one consumer's database and role into being under its login and the vault's credential, as the -// harness does: at once when it is new or changed, and otherwise only when the server no longer holds it, which is -// asked every VerifyEvery. -func (m *movedRun) Make(ctx context.Context, c provider.Consumer) error { - h, e := m.h, c.Entry - if e.User == "" { - return fmt.Errorf("%s has no login to make", e.Key()) - } - if len(c.Credential) == 0 { - return fmt.Errorf("%s has no credential to make its login with", e.Key()) - } - h.mu.Lock() - defer h.mu.Unlock() - as, password := e.User, string(c.Credential) - p := Provision{As: as, Password: password, Values: orEmpty(e.Contributes), Consumer: e.Consumer} - hash := hashOf(as, password, p.Values, nil) - if was, ok := h.applied[as]; ok && was.hash == hash { - if h.Now().Sub(m.verified[as]) < h.VerifyEvery { - return nil - } - m.verified[as] = h.Now() - hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) - held, err := h.Adapter.Holds(hctx, p) - cancel() - if err != nil { - text := scrub(err, password) - h.failed(as, e.Consumer, h.classOf(err, text), text) - return errors.New(text) - } - if held { - h.succeeded(as) - return nil - } - h.say("%s: the backend no longer holds it; applying again", as) - } - if err := h.Adapter.Create(ctx, p); err != nil { - text := scrub(err, password) - h.failed(as, e.Consumer, h.classOf(err, text), text) - return errors.New(text) - } - h.succeeded(as) - h.applied[as] = appliedEntry{hash: hash, node: e.Consumer} - m.verified[as] = h.Now() - h.reenabled(as, e.Consumer) - return nil -} - -// Gone is the helper asking whether it may forget a consumer that no longer binds. **It never retires**: retiring is -// the harness's rule, the same unasked set over StablePasses passes and StableFor, at most RetireAtOnce at a time -// (retirement.go, ADR 0230), run by retire below. So a consumer missing from one read of the plans loses nothing: -// the helper is told no, and asks again at its next pass, until the login is retired, or asked for again. -func (m *movedRun) Gone(_ context.Context, e provider.Entry) error { - h := m.h - as := e.User - if as == "" { - return nil // refused: nothing was made for it - } - // The plans and the grant file as they are now, not as the last retirement pass read them: with either not - // read, nothing is let go. - want, read := m.asked(h.Node) - h.mu.Lock() - defer h.mu.Unlock() - _, granted, ok := m.granted(want) - read = read && ok - asked := want[as] || granted[as] - _, held := h.applied[as] - _, retired := h.r.retired[as] - switch { - case !read: - case asked: - return nil // another holder of the same login still binds: nothing to retire - case retired, !held && h.r.seeded: - return nil - } - return fmt.Errorf("%s is kept until its retirement is stable (%d passes and %s) and within the bound, or a "+ - "person approves it", as, h.StablePasses, h.StableFor) -} - -// plan is as much of an assignment as is read to learn who binds postgres. -type plan struct { - Node string `json:"node"` - Module string `json:"module"` - Provisioning *struct { - Provisions []struct { - Provision string `json:"provision"` - Provider string `json:"provider"` - ProviderModule string `json:"provider-module"` - // As is who the consumer is at its provider's end, its slug first (issue 788). - As string `json:"as"` - } `json:"provisions"` - } `json:"provisioning"` -} - -// asked is the logins the plans bind postgres for, and whether they could all be read: a plan that is not the -// controller's, or no read yet, is not an answer, so nothing is counted unasked from it (novox/hq issue 241). -func (m *movedRun) asked(node string) (map[string]bool, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if !m.loaded || len(m.assignments) == 0 { - return nil, false - } - want := map[string]bool{} - for key, raw := range m.assignments { - var a plan - if err := json.Unmarshal(raw, &a); err != nil || a.Node+"."+a.Module != key { - return nil, false - } - if a.Provisioning == nil { - continue - } - for _, b := range a.Provisioning.Provisions { - if b.Provider == node && b.ProviderModule == "postgres" && b.Provision == ProvisionName { - as := b.As - if as == "" { - as = provider.ConsumerIdentity(a.Node, a.Module) // a plan that names none - } - want[as] = true - } - } - } - return want, true -} - -// granted is what the grant file gives that no plan names (issue 937), and every login it gives; and whether it -// could be read. The caller holds h.mu. With no grant file named there is nothing given, and that is a read. -func (m *movedRun) granted(planned map[string]bool) ([]contribution, map[string]bool, bool) { - h := m.h - if h.Receives == "" { - return nil, map[string]bool{}, true - } - given := h.readContributions() - if given == nil { - return nil, nil, false - } - var own []contribution - logins := map[string]bool{} - for _, g := range given { - logins[g.As] = true - if !planned[g.As] { - own = append(own, g) - } - } - return own, logins, true -} - -// retire is one pass of the harness's retirement over what the plans bind and the grant file gives now; the logins -// only the grant file gives are made and checked first, as the grant path does. -func (m *movedRun) retire(ctx context.Context) { - h := m.h - want, ok := m.asked(h.Node) - h.mu.Lock() - defer h.mu.Unlock() - var given []contribution - if ok { - var granted map[string]bool - given, granted, ok = m.granted(want) - for as := range granted { - want[as] = true - } - } - if !ok { - // Not a result: the passes that must agree before anything is retired start again. - h.r.key, h.r.count = "", 0 - return - } - h.seed(ctx, want) - h.apply(ctx, given) - h.retireUnasked(ctx, want) - h.r.lastWant = want - for as := range h.failing { - if !want[as] { - delete(h.failing, as) - } - } - for as := range h.trouble { - if _, held := h.applied[as]; !want[as] && !held { - h.recovered(as, "no longer asked for") - } - } -} - -// watch keeps the plans as the bus holds them, from the first read on. -func (m *movedRun) watch(ctx context.Context, bus provider.Bus) error { - err := bus.Watch(ctx, provider.AssignmentsBucket, func(key string, value []byte) { - m.mu.Lock() - defer m.mu.Unlock() - if value == nil { - delete(m.assignments, key) - } else { - m.assignments[key] = append([]byte(nil), value...) - } - }) - if err == nil { - m.mu.Lock() - m.loaded = true - m.mu.Unlock() - } - return err -} - -// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends. -// -// The bus is reached through the runtime that launched this bundle, which answers only once Serve runs, and Run starts -// beside Serve: a watch asked too early is asked again, after a wait that doubles from FirstRetry up to a minute, -// until it is answered or ctx ends, so a start that loses the race never leaves the consumers unanswered (issue 943). -func (m *movedRun) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error { - wait := m.FirstRetry - if wait <= 0 { - wait = time.Second - } - for tries := 1; ; tries++ { - err := m.watch(ctx, bus) - if err == nil { - break - } - if tries == 4 || tries%20 == 0 { - m.h.say("the plans cannot be watched yet (%d tries), asking again: %v", tries, err) - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(wait): - } - wait = min(wait*2, time.Minute) - } - go func() { - for { - m.retire(ctx) - select { - case <-ctx.Done(): - return - case <-time.After(m.h.Every): - } - } - }() - return p.Run(ctx) -} - -// moved is postgres handed to the provisioner: the helper answering its consumers, and the harness (never run) whose -// retirement and tools it shares. consumersFile is where its machine's client fills its consumers. -func moved(consumersFile string, h *Harness) (*movedRun, *provider.Provider, provider.Bus, error) { - if consumersFile == "" { - return nil, nil, nil, errors.New("no consumers' file") - } - run := newMovedRun(h) - bus := provider.StateBus("postgres") - return run, provider.New(provider.Options{ - Node: h.Node, - Module: "postgres", - Offers: map[string]provider.Offer{ProvisionName: { - Credential: provider.PerConsumer, - Serves: map[string]any{"port": servedPort}, - }}, - Bus: bus, - Vault: provider.ConsumersFile{Paths: []string{consumersFile}}, - Make: run.Make, - Gone: run.Gone, - Every: h.Every, - Logf: func(f string, a ...any) { h.say(f, a...) }, - }), bus, nil +// moved is postgres handed to the provisioner: the SDK's moved path (harness.Moved) answering its consumers, and the +// harness (never run) whose retirement and tools it shares. consumersFile is where its machine's client fills its +// consumers. What differs from keycloak is only this declaration (issue 849). +func moved(consumersFile string, h *harness.Harness) (*harness.Moved, *provider.Provider, provider.Bus, error) { + return harness.NewMoved(consumersFile, h, harness.MovedSpec{Module: "postgres", Provision: ProvisionName, + Offer: provider.Offer{Credential: provider.PerConsumer, Serves: map[string]any{"port": servedPort}}, + Login: func(e provider.Entry) string { return e.User }, + Noun: "login", Made: "login", Backend: "the backend", WithGrants: true}) } diff --git a/modules/postgres/cmd/postgres-provider/moved_test.go b/modules/postgres/cmd/postgres-provider/moved_test.go index db51aed4..4070507c 100644 --- a/modules/postgres/cmd/postgres-provider/moved_test.go +++ b/modules/postgres/cmd/postgres-provider/moved_test.go @@ -1,287 +1,13 @@ package main import ( - "context" - "encoding/json" - "fmt" "os" "path/filepath" - "reflect" - "sync" "testing" "time" - - "git.novox.be/novox/mesh-sdk/go/provider" ) -// bucketBus is the helper's bus over in-memory buckets. -type bucketBus struct { - mu sync.Mutex - buckets map[string]map[string][]byte -} - -func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { - b.mu.Lock() - defer b.mu.Unlock() - for k, v := range b.buckets[bucket] { - handle(k, v) - } - return nil -} - -func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { - b.mu.Lock() - defer b.mu.Unlock() - if b.buckets[bucket] == nil { - b.buckets[bucket] = map[string][]byte{} - } - b.buckets[bucket][key] = append([]byte(nil), value...) - return nil -} - -func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { - b.mu.Lock() - defer b.mu.Unlock() - delete(b.buckets[bucket], key) - return nil -} - -// bind is one consumer's assignment, whose plan names this machine's postgres as the provider and carries what the -// consumer contributes (issue 776). -func (b *bucketBus) bind(node, module string, contributes map[string]any) { - plan, _ := json.Marshal(map[string]any{"node": node, "module": module, "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "postgres", - "contributes": contributes}}}}) - if err := b.Put(context.Background(), provider.AssignmentsBucket, node+"."+module, plan); err != nil { - panic(err) - } -} - -// consumersFile writes the file the machine's client fills: one credential per consumer. -func consumersFile(t *testing.T, bound map[[2]string]string) string { - var list []provider.ConsumerCredential - for k, password := range bound { - list = append(list, provider.ConsumerCredential{Consumer: k[0], Module: k[1], - Credential: provider.PairName("novox", ProvisionName, k[0], k[1], ""), Value: password}) - } - raw, _ := json.Marshal(list) - at := filepath.Join(t.TempDir(), "consumers.json") - if err := os.WriteFile(at, raw, 0o600); err != nil { - t.Fatal(err) - } - return at -} - -// movedWorld is postgres on the helper over an in-memory bus, with a recorder for a server and a clock to move. -type movedWorld struct { - t *testing.T - a *recorder - bus *bucketBus - h *Harness - run *movedRun - p *provider.Provider - now time.Time - file string -} - -func newMovedWorld(t *testing.T, bound map[[2]string]string) *movedWorld { - w := &movedWorld{t: t, a: &recorder{}, bus: &bucketBus{buckets: map[string]map[string][]byte{}}, - now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)} - w.file = consumersFile(t, bound) - w.h = &Harness{Resource: ProvisionName, Adapter: w.a, Node: "novox", Now: func() time.Time { return w.now }, - Log: func(string, ...any) {}} - var bus provider.Bus - w.run, w.p, bus, _ = moved(w.file, w.h) - _ = bus - if err := w.run.watch(ctx, w.bus); err != nil { - t.Fatal(err) - } - return w -} - -// pass is one helper pass and one retirement pass, five seconds after the last. -func (w *movedWorld) pass() []provider.Outcome { - w.now = w.now.Add(5 * time.Second) - // the plans as the bus holds them now, as the watch would hand them - w.run.assignments = map[string][]byte{} - for k, v := range w.bus.buckets[provider.AssignmentsBucket] { - w.run.assignments[k] = v - } - // the helper reads its buckets from the in-memory bus - w.p = provider.New(provider.Options{Node: "novox", Module: "postgres", Bus: w.bus, - Vault: provider.ConsumersFile{Paths: []string{w.file}}, - Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer, - Serves: map[string]any{"port": servedPort}}}, - Make: w.run.Make, Gone: w.run.Gone, Logf: func(string, ...any) {}}) - out, err := w.p.Pass(ctx) - if err != nil { - w.t.Fatal(err) - } - w.run.retire(ctx) - return out -} - -func bindings(n int) (map[[2]string]string, []string) { - bound := map[[2]string]string{} - var logins []string - for i := 1; i <= n; i++ { - module := fmt.Sprintf("app%d", i) - bound[[2]string{"ace", module}] = "pw-" + module - logins = append(logins, provider.ConsumerIdentity("ace", module)) - } - return bound, logins -} - -// The consumers' file makes the same calls on the adapter as the equivalent grant files do (the harness). -func TestTheConsumersFileMakesWhatTheGrantFilesMake(t *testing.T) { - letta := provider.ConsumerIdentity("ace", "letta") - plain := provider.ConsumerIdentity("ace", "plain") - - // Today: the grant files, through the harness. - g := newWorld(t) - g.give( - map[string]any{"as": letta, "node": "ace", "values": map[string]any{"name": "letta", "extensions": []any{"vector"}}}, - map[string]any{"as": plain, "node": "ace", "values": map[string]any{"name": "plain"}}, - ) - g.h.Reconcile(ctx) - - // The consumers' file, through the provider helper. - w := newMovedWorld(t, map[[2]string]string{{"ace", "letta"}: "pw-" + letta, {"ace", "plain"}: "pw-" + plain}) - w.bus.bind("ace", "letta", map[string]any{"name": "letta", "extensions": []any{"vector"}}) - w.bus.bind("ace", "plain", map[string]any{"name": "plain"}) - for _, o := range w.pass() { - if o.Status != provider.Answered { - t.Fatalf("%+v", o) - } - } - byAs := map[string]Provision{} - for _, c := range w.a.created { - byAs[c.As] = c - } - if len(w.a.created) != 2 { - t.Fatalf("created %d, want 2", len(w.a.created)) - } - for _, want := range g.a.created { - c := byAs[want.As] - if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer || - !reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) { - t.Fatalf("made %+v, the grant files make %+v", c, want) - } - } - // Unchanged, and held: nothing is made again, within a minute or after it. - w.a.held = true - w.pass() - w.now = w.now.Add(2 * time.Minute) - w.pass() - if len(w.a.created) != 2 { - t.Fatalf("an unchanged pass created again: %d", len(w.a.created)) - } - // No longer held: made again. - w.a.held = false - w.now = w.now.Add(2 * time.Minute) - w.pass() - if len(w.a.created) != 4 { - t.Fatalf("not made again: %d", len(w.a.created)) - } -} - -// One read without a consumer retires nothing: it takes the harness's stable passes and time, and then the entry -// goes and the login is retired, never dropped. -func TestAConsumerMissingFromOneReadRetiresNothing(t *testing.T) { - bound, logins := bindings(2) - w := newMovedWorld(t, bound) - w.bus.bind("ace", "app1", nil) - w.bus.bind("ace", "app2", nil) - w.pass() - - // One read without app2. - saved := w.bus.buckets[provider.AssignmentsBucket]["ace.app2"] - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v on one read", w.a.removed) - } - if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; !kept { - t.Fatal("the entry went on one read") - } - // It is back: nothing was lost, and the entry never left the bucket. - w.bus.Put(ctx, provider.AssignmentsBucket, "ace.app2", saved) - for i := 0; i < 8; i++ { - w.pass() - } - w.now = w.now.Add(15 * time.Minute) - w.pass() - if len(w.a.removed) != 0 || len(w.bus.buckets[provider.BucketOf("postgres")]) != 2 { - t.Fatalf("retired %v, entries %d", w.a.removed, len(w.bus.buckets[provider.BucketOf("postgres")])) - } - - // Gone for good: not retired before StablePasses and StableFor, then retired and its entry removed. - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") - for i := 0; i < 6; i++ { - w.pass() - } - if len(w.a.removed) != 0 { - t.Fatalf("retired %v before the time was out", w.a.removed) - } - w.now = w.now.Add(11 * time.Minute) - w.pass() - w.pass() - if !reflect.DeepEqual(w.a.removed, []string{logins[1]}) || len(w.a.deleted) != 0 { - t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) - } - if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; kept { - t.Fatal("the entry stayed after the login was retired") - } -} - -// More than three at once waits for a person, as the harness's bound says. -func TestMoreThanTheBoundWaitsForAPerson(t *testing.T) { - bound, _ := bindings(5) - w := newMovedWorld(t, bound) - for i := 1; i <= 5; i++ { - w.bus.bind("ace", fmt.Sprintf("app%d", i), nil) - } - w.pass() - for i := 1; i <= 4; i++ { - w.bus.Delete(ctx, provider.AssignmentsBucket, fmt.Sprintf("ace.app%d", i)) - } - for i := 0; i < 6; i++ { - w.pass() - } - w.now = w.now.Add(11 * time.Minute) - w.pass() - w.pass() - if len(w.a.removed) != 0 || w.h.r.waiting == nil { - t.Fatalf("retired %v, waiting %v", w.a.removed, w.h.r.waiting) - } -} - -// A plan list that was never read is not nobody asking. -func TestNoPlansReadRetiresNothing(t *testing.T) { - bound, _ := bindings(1) - w := newMovedWorld(t, bound) - w.bus.bind("ace", "app1", nil) - w.pass() - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app1") - for i := 0; i < 8; i++ { - w.pass() - } - w.now = w.now.Add(20 * time.Minute) - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v with no plans read", w.a.removed) - } -} - -// A consumer whose credential is not in the file yet is waited for: nothing is made for it. -func TestAConsumerWithoutACredentialIsWaitedFor(t *testing.T) { - w := newMovedWorld(t, map[[2]string]string{}) - w.bus.bind("ace", "letta", map[string]any{"name": "letta"}) - outcomes := w.pass() - if len(outcomes) != 1 || outcomes[0].Status != provider.Waiting || len(w.a.created) != 0 { - t.Fatalf("%+v, created %v", outcomes, w.a.created) - } -} +// postgres's own part of the moved path (issue 849): the shared driving of it is proven in the SDK's harness package. // Where the file is not there, postgres takes the grant directory, as before; where it is, the helper. func TestTheGrantPathIsTakenWhereTheConsumersFileIsAbsent(t *testing.T) { @@ -326,33 +52,3 @@ func TestTheGrantPathLeavesWhenTheConsumersFileIsFilled(t *testing.T) { t.Fatal("did not leave when the file was filled") } } - -// A consumer whose module has a slug keeps the login its plan names, the slug first (issue 788): the adapter makes it -// under that login, and the retirement counts it as asked for, so a pass retires nothing. -func TestASluggedConsumerKeepsTheLoginItsPlanNames(t *testing.T) { - agent := provider.ConsumerIdentity("ace", "agent") - w := newMovedWorld(t, map[[2]string]string{{"ace", "claude-code"}: "pw-agent"}) - plan, _ := json.Marshal(map[string]any{"node": "ace", "module": "claude-code", "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", - "provider-module": "postgres", "as": agent}}}}) - if err := w.bus.Put(ctx, provider.AssignmentsBucket, "ace.claude-code", plan); err != nil { - t.Fatal(err) - } - for _, o := range w.pass() { - if o.Status != provider.Answered { - t.Fatalf("%+v", o) - } - } - if len(w.a.created) != 1 || w.a.created[0].As != agent { - t.Fatalf("made %+v, want one under %s", w.a.created, agent) - } - for i := 0; i < 8; i++ { - w.pass() - } - w.now = w.now.Add(15 * time.Minute) - w.pass() - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v: the plan's login was counted unasked", w.a.removed) - } -} diff --git a/modules/postgres/cmd/postgres-provider/provisioner.go b/modules/postgres/cmd/postgres-provider/provisioner.go index efaa95b0..a0d4ed06 100644 --- a/modules/postgres/cmd/postgres-provider/provisioner.go +++ b/modules/postgres/cmd/postgres-provider/provisioner.go @@ -14,6 +14,7 @@ package main import ( "context" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "time" ) @@ -23,7 +24,7 @@ type provisioner struct { announce func(event string, body map[string]string) } -func (a provisioner) Create(ctx context.Context, p Provision) error { +func (a provisioner) Create(ctx context.Context, p harness.Provision) error { // Read before anything runs: a malformed list is refused without touching the server. extensions, err := Extensions(p.Values) if err != nil { @@ -60,17 +61,19 @@ func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, wh } // Inventory is what this server holds that the mesh made (retire_pg.go). -func (a provisioner) Inventory(ctx context.Context) (Inventory, error) { return a.pg.Inventory(ctx) } +func (a provisioner) Inventory(ctx context.Context) (harness.Inventory, error) { + return a.pg.Inventory(ctx) +} // Delete drops a retired consumer's database and role, or a database set aside — a person's act. -func (a provisioner) Delete(ctx context.Context, r Retired) (int64, error) { +func (a provisioner) Delete(ctx context.Context, r harness.Retired) (int64, error) { return a.pg.DeleteRetired(ctx, r) } // Holds is asked every minute: whether the consumer can still log in as the mesh gave it, and finds // the extensions it asked for, so a login or extension lost behind the provisioner's back is made // again (novox/hq issue 120). -func (a provisioner) Holds(ctx context.Context, p Provision) (bool, error) { +func (a provisioner) Holds(ctx context.Context, p harness.Provision) (bool, error) { extensions, err := Extensions(p.Values) if err != nil { return false, err diff --git a/modules/postgres/cmd/postgres-provider/retire_pg.go b/modules/postgres/cmd/postgres-provider/retire_pg.go index 969dc4f5..e07a5640 100644 --- a/modules/postgres/cmd/postgres-provider/retire_pg.go +++ b/modules/postgres/cmd/postgres-provider/retire_pg.go @@ -26,6 +26,7 @@ import ( "context" "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "regexp" "strconv" "time" @@ -44,7 +45,7 @@ type roleMark struct { func readMark(comment string) (roleMark, bool) { var m roleMark - if comment == "" || json.Unmarshal([]byte(comment), &m) != nil || m.Mesh != KindConsumer { + if comment == "" || json.Unmarshal([]byte(comment), &m) != nil || m.Mesh != harness.KindConsumer { return roleMark{}, false } return m, true @@ -52,7 +53,7 @@ func readMark(comment string) (roleMark, bool) { // MarkStatement is the comment that marks a role as the mesh's consumer, active or retired. func MarkStatement(role string, m roleMark) string { - m.Mesh = KindConsumer + m.Mesh = harness.KindConsumer b, _ := json.Marshal(m) return fmt.Sprintf("COMMENT ON ROLE %s IS %s", Ident(role), Literal(string(b))) } @@ -97,8 +98,8 @@ ORDER BY r.rolname` const SetAsideStatement = `SELECT datname, pg_database_size(datname) FROM pg_database WHERE datname ~ '_deleted_[0-9]{8}$' ORDER BY datname` // Inventory is what this server holds that the mesh made. -func (c *Client) Inventory(ctx context.Context) (Inventory, error) { - inv := Inventory{Active: []string{}, Retired: []Retired{}, Sizes: map[string]int64{}} +func (c *Client) Inventory(ctx context.Context) (harness.Inventory, error) { + inv := harness.Inventory{Active: []string{}, Retired: []harness.Retired{}, Sizes: map[string]int64{}} r, err := c.Query(ctx, "", InventoryStatement) if err != nil { return inv, err @@ -117,8 +118,8 @@ func (c *Client) Inventory(ctx context.Context) (Inventory, error) { continue } at, _ := time.Parse(time.RFC3339, m.Retired) - inv.Retired = append(inv.Retired, Retired{Consumer: name, Node: m.Node, RetiredAt: at, Why: m.Why, - SizeBytes: sizeOf(size), Kind: KindConsumer}) + inv.Retired = append(inv.Retired, harness.Retired{Consumer: name, Node: m.Node, RetiredAt: at, Why: m.Why, + SizeBytes: sizeOf(size), Kind: harness.KindConsumer}) } r, err = c.Query(ctx, "", SetAsideStatement) if err != nil { @@ -131,7 +132,7 @@ func (c *Client) Inventory(ctx context.Context) (Inventory, error) { continue } at, _ := time.Parse("20060102", m[1]) - inv.Retired = append(inv.Retired, Retired{Consumer: name, RetiredAt: at, SizeBytes: sizeOf(cell(row, 1)), + inv.Retired = append(inv.Retired, harness.Retired{Consumer: name, RetiredAt: at, SizeBytes: sizeOf(cell(row, 1)), Why: "renamed aside — by postgres_retire_database, or by hand", Kind: KindSetAside}) } return inv, nil @@ -139,7 +140,7 @@ func (c *Client) Inventory(ctx context.Context) (Inventory, error) { // DeleteRetired drops what a retired entry names: a consumer's database and then its role, or one // database set aside. Answers the bytes freed. -func (c *Client) DeleteRetired(ctx context.Context, r Retired) (int64, error) { +func (c *Client) DeleteRetired(ctx context.Context, r harness.Retired) (int64, error) { if r.Kind == KindSetAside { if !setAside.MatchString(r.Consumer) { return 0, fmt.Errorf("%s is not a database set aside", r.Consumer) diff --git a/modules/postgres/cmd/postgres-provider/retire_pg_test.go b/modules/postgres/cmd/postgres-provider/retire_pg_test.go index a6c796aa..ee9a476a 100644 --- a/modules/postgres/cmd/postgres-provider/retire_pg_test.go +++ b/modules/postgres/cmd/postgres-provider/retire_pg_test.go @@ -4,6 +4,7 @@ package main // of it — the login refused, the data kept, a deletion dropping only its own — is live_test.go's. import ( + "git.novox.be/novox/mesh-sdk/go/provider/harness" "strings" "testing" "time" @@ -60,7 +61,7 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { r.RetiredAt.Format(time.RFC3339) != "2026-10-06T12:00:00Z" { t.Fatalf("%+v", r) } - if r := inv.Retired[1]; r.Consumer != "locked_before" || !r.RetiredAt.IsZero() || r.Kind != KindConsumer { + if r := inv.Retired[1]; r.Consumer != "locked_before" || !r.RetiredAt.IsZero() || r.Kind != harness.KindConsumer { t.Fatalf("found locked without a mark: %+v", r) } if r := inv.Retired[2]; r.Kind != KindSetAside || r.RetiredAt.Format("20060102") != "20261005" || r.SizeBytes != 7771827 { @@ -71,13 +72,13 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { func TestDeletingRefusesALoginThatCanConnect(t *testing.T) { f, c := newFake(admin) f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"t"}) - if _, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_baserow", Kind: KindConsumer}); err == nil { + if _, err := c.DeleteRetired(ctx, harness.Retired{Consumer: "mesh_ace_baserow", Kind: harness.KindConsumer}); err == nil { t.Fatal("deleted an active consumer") } noDrop(t, f.statements()) f, c = newFake(admin) - if _, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_baserow", Kind: KindSetAside}); err == nil { + if _, err := c.DeleteRetired(ctx, harness.Retired{Consumer: "mesh_ace_baserow", Kind: KindSetAside}); err == nil { t.Fatal("dropped a database not set aside") } noDrop(t, f.statements()) @@ -87,7 +88,7 @@ func TestDeletingDropsTheDatabaseThenTheRole(t *testing.T) { f, c := newFake(admin) f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"f"}) f.answer(`SELECT pg_database_size`, []string{"size"}, []string{"9000"}) - freed, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_letta", Kind: KindConsumer}) + freed, err := c.DeleteRetired(ctx, harness.Retired{Consumer: "mesh_ace_letta", Kind: harness.KindConsumer}) if err != nil || freed != 9000 { t.Fatal(freed, err) } @@ -100,7 +101,7 @@ func TestDeletingDropsTheDatabaseThenTheRole(t *testing.T) { f, c = newFake(admin) f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"f"}) f.answer(`SELECT datname FROM pg_database WHERE datdba`, []string{"datname"}, []string{"mesh_ace_letta_deleted_20261005"}) - if _, err := c.DeleteRetired(ctx, Retired{Consumer: "mesh_ace_letta", Kind: KindConsumer}); err == nil || + if _, err := c.DeleteRetired(ctx, harness.Retired{Consumer: "mesh_ace_letta", Kind: harness.KindConsumer}); err == nil || !strings.Contains(err.Error(), "still owns") || has(f.statements(), `DROP ROLE`) { t.Fatal(err, f.statements()) } diff --git a/modules/postgres/cmd/postgres-provider/retirement.go b/modules/postgres/cmd/postgres-provider/retirement.go deleted file mode 100644 index f48afde8..00000000 --- a/modules/postgres/cmd/postgres-provider/retirement.go +++ /dev/null @@ -1,623 +0,0 @@ -package main - -// What becomes of a consumer the mesh no longer asks for (novox/hq ADR 0230 — the operator's model, -// decided 2026-10-06; it replaces ADR 0229's withdrawal brake, which let one consumer go every hour). -// -// A consumer — a login, a client, a key, and the data behind it — is in one of three states: -// -// 1. ACTIVE. -// 2. RETIRED. The mesh stopped asking for it: its access is disabled, reversibly, and its login and -// data are marked "to delete" with when and why. Nothing is deleted. Asked for again, the ordinary -// create re-enables it at once, as it was. -// 3. DELETED, only through `cleanup delete`, a person's act, which this provider executes because it -// owns its backend. -// -// **Stable removals.** A consumer is retired only once the same set of consumers has gone unasked BOTH -// in StablePasses (5) consecutive passes that read the file AND for at least StableFor (10 minutes) -// since the first pass that saw it. Five passes alone are twenty-five seconds — shorter than a -// controller restart, a store reconnecting or a file half written. A pass that could not read the file -// is not a result and starts both again; so does a different set. Additions and changes are never -// delayed. -// -// **Too many is a person.** A stable set of more than RetireAtOnce (3), or — where more than one is -// held — of more than RetireFraction (half) of those held, retires nothing: it WAITS, said in the -// journal and announced `provisioner.retirement` `waiting` (again every SayAgainEvery), which the -// controller raises as an urgent condition, until `retire approve ` or `retire reject`. -// An unassignment the controller made itself is no exception: many changes at once means a person is -// at work, and a person confirms once. On 2026-10-04 one misread file withdrew seven consumers at once -// (issue 241); under this rule that is a question, not an act. -// -// **The backend remembers, not this process.** What is retired, since when and why is read from the -// backend's own mark (Adapter.Inventory), so a restart forgets nothing; a consumer the backend holds -// active that the mesh no longer asks for is retired by the same rules after a restart as before one. -// A consumer found disabled without the mark — withdrawn before this record — is ADOPTED as retired: -// marked, said, announced `adopted`, and its clock starts then. -// -// **A rejection lives as long as this process.** Rejected, the set is kept active and not asked about -// again while it stays the same set; a restart asks again — loudly, never silently. - -import ( - "context" - "errors" - "fmt" - "sort" - "strings" - "time" - - stdio "git.novox.be/novox/mesh-sdk/go" -) - -// EventRetirement is the one event a provider says about retirement, its `change` saying what -// happened. The controller derives the permission to emit it for every module that receives -// contributions, as it does the standing events (novox/hq ADR 0224, 0230). -const EventRetirement = "provisioner.retirement" - -// The changes EventRetirement carries. -const ( - ChangeWaiting = "waiting" // a stable set over the bound waits for a person - ChangeSettled = "settled" // a waiting or rejected set ended without being retired - ChangeApproved = "approved" // a person approved the waiting (or rejected) set - ChangeRejected = "rejected" // a person kept the waiting set active - ChangeRetired = "retired" // consumers disabled and marked to delete - ChangeReenabled = "reenabled" // a retired consumer asked for again, enabled as it was - ChangeDeleted = "deleted" // a retired consumer deleted, by a person - ChangeAdopted = "adopted" // found disabled without the mark, now retired on record -) - -// StableFor is the least time the same unasked set must hold before it is retired (novox/hq ADR 0230): -// longer than a controller restart, a store reconnecting or a file half written. -const StableFor = 10 * time.Minute - -// KindConsumer is a retired consumer. A backend may list other things set aside for deletion under a -// word of its own (postgres: a database renamed aside). -const KindConsumer = "consumer" - -// Retired is one thing a provider holds retired, as its backend's mark says. -type Retired struct { - Consumer string - // Node is the consumer's machine, where the mark records it. - Node string - // RetiredAt is when it was retired; zero for one found disabled without the mesh's mark. - RetiredAt time.Time - Why string - // SizeBytes is what deleting it would free; -1 when the backend cannot say. - SizeBytes int64 - Kind string -} - -// Inventory is what a backend holds that the mesh made. -type Inventory struct { - Active []string - Retired []Retired - // Sizes is what each active consumer keeps on the backend, in bytes, where the backend can say: what - // the controller tells an empty replacement of a consumer's data by (novox/hq ADR 0233). - Sizes map[string]int64 -} - -// retirement is the loop's memory of the sets it is counting, waiting on and was told to keep. -type retirement struct { - key string // the unasked set, joined - count int // consecutive passes that saw it - firstSeen time.Time - waiting *waitingSet - rejected *rejectedSet - retired map[string]retiredEntry - lastWant map[string]bool - seeded bool - seedSaid string - // noneSaid is whether this process has said that nothing waits: an earlier process may have left a set - // waiting that this one never knew (issue 937). - noneSaid bool -} - -type waitingSet struct { - set []string - since time.Time - saidAt time.Time - held int -} - -type rejectedSet struct { - set []string - by, why string - at time.Time -} - -type retiredEntry struct { - node string - at time.Time - why string -} - -// seed reads what the backend holds, once per process: an active consumer the mesh no longer asks -// for is held, so it is retired by the same rules as one this process made; one found disabled -// without the mark is adopted as retired. -func (h *Harness) seed(ctx context.Context, want map[string]bool) { - if h.r.seeded { - return - } - inv, err := h.Adapter.Inventory(ctx) - if err != nil { - if said := err.Error(); said != h.r.seedSaid { - h.say("cannot list what the backend holds (%v); a consumer the mesh stopped asking for while this "+ - "provider was down is not retired until it can", err) - h.r.seedSaid = said - } - return - } - h.r.seeded = true - for _, as := range inv.Active { - if _, held := h.applied[as]; !held && !want[as] { - // No hash: asked for again, it is applied again, which is harmless and marks it. - h.applied[as] = appliedEntry{} - h.say("%s: held by the backend and no longer asked for; retired once that holds for %d passes "+ - "and %s (novox/hq ADR 0230)", as, h.StablePasses, h.StableFor) - } - } - now := h.Now() - for _, r := range inv.Retired { - if r.Kind != "" && r.Kind != KindConsumer { - continue - } - if !r.RetiredAt.IsZero() { - h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: r.RetiredAt, why: r.Why} - continue - } - if want[r.Consumer] { - continue // asked for: this pass's create enables it - } - why := "found disabled without the mesh's mark — withdrawn before retirement was recorded " + - "(novox/hq ADR 0230); retired as found" - if err := h.Adapter.Retire(ctx, r.Consumer, nil, why, now); err != nil { - h.say("%s: found disabled and could not be marked retired, will try at the next start: %v", r.Consumer, err) - continue - } - h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: now, why: why} - h.say("%s: ADOPTED as retired — %s", r.Consumer, why) - h.announce(EventRetirement, h.retirementBody(ChangeAdopted, []map[string]any{ - {"consumer": r.Consumer, "node": r.Node, "retired_at": stamp(now), "why": why, "size_bytes": r.SizeBytes}, - }, map[string]any{"why": why})) - } -} - -// retireUnasked counts the passes that saw the same set no longer asked for and, once it is stable, -// retires it — or, past the bound, waits for a person. -func (h *Harness) retireUnasked(ctx context.Context, want map[string]bool) { - var unasked []string - for as := range h.applied { - if !want[as] { - unasked = append(unasked, as) - } - } - sort.Strings(unasked) - key := strings.Join(unasked, "\x00") - now := h.Now() - switch { - case len(unasked) == 0: - h.settle("every consumer held is asked for again") - h.r.key, h.r.count = "", 0 - return - case key != h.r.key: - if h.r.waiting != nil || h.r.rejected != nil { - h.settle("the set the mesh no longer asks for changed") - } - h.r.key, h.r.count, h.r.firstSeen = key, 1, now - h.say("no longer asked for: %s; retired once the same set holds for %d passes and %s", - strings.Join(unasked, ", "), h.StablePasses, h.StableFor) - default: - h.r.count++ - } - if h.r.rejected != nil || h.r.count < h.StablePasses || now.Sub(h.r.firstSeen) < h.StableFor { - return - } - if h.overTheBound(len(unasked), len(h.applied)) { - h.wait(unasked) - return - } - why := fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s, from %s", - h.r.count, now.Sub(h.r.firstSeen).Round(time.Second), stamp(h.r.firstSeen)) - h.retire(ctx, unasked, why, nil) -} - -// overTheBound says retiring n of the held consumers at once needs a person. -func (h *Harness) overTheBound(n, held int) bool { - if n == 0 { - return false - } - return n > h.RetireAtOnce || (held > 1 && float64(n) > h.RetireFraction*float64(held)) -} - -func (h *Harness) bound(held int) string { - return fmt.Sprintf("more than %d at once, or more than %.0f%% of the %d held", h.RetireAtOnce, - h.RetireFraction*100, held) -} - -// wait holds a stable set over the bound for a person, said once and announced again every -// SayAgainEvery so a controller that missed the first hears the next. -func (h *Harness) wait(set []string) { - now := h.Now() - w := h.r.waiting - if w == nil { - w = &waitingSet{set: set, since: now, held: len(h.applied)} - h.r.waiting = w - h.say("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d consumer(s) this provider "+ - "holds (%s), %s. Nothing is retired; each stays active until `retire approve %s ` or "+ - "`retire reject` (novox/hq ADR 0230)", len(set), w.held, strings.Join(set, ", "), h.bound(w.held), h.Node) - } else if now.Sub(w.saidAt) < h.SayAgainEvery { - return - } - w.saidAt = now - h.announce(EventRetirement, h.retirementBody(ChangeWaiting, h.named(set), map[string]any{ - "held": w.held, "bound": h.bound(w.held), "since": stamp(w.since), - })) -} - -// settle ends a waiting or rejected set that the mesh's own answer made moot. -func (h *Harness) settle(why string) { - var set []string - switch { - case h.r.waiting != nil: - set = h.r.waiting.set - case h.r.rejected != nil: - set = h.r.rejected.set - case h.r.noneSaid: - return - default: - // A restart forgets a set an earlier process waited on, and so would never say it ended: the - // controller's condition would ask a person to approve what nobody waits on any more (issue 937). - // The first time this process finds every consumer held asked for, it says so once, with no set. - h.r.noneSaid = true - h.say("nothing waits for a person: %s", why) - h.announce(EventRetirement, h.retirementBody(ChangeSettled, []map[string]any{}, map[string]any{"why": why})) - return - } - h.r.waiting, h.r.rejected = nil, nil - h.say("retirement of %s settled without retiring: %s", strings.Join(set, ", "), why) - h.announce(EventRetirement, h.retirementBody(ChangeSettled, h.named(set), map[string]any{"why": why})) -} - -// retire disables and marks each consumer of set; one that fails stays held and is tried again once -// its set is stable again. -func (h *Harness) retire(ctx context.Context, set []string, why string, extra map[string]any) []string { - now := h.Now() - held := len(h.applied) - var done []string - var said []map[string]any - for _, as := range set { - was, ok := h.applied[as] - if !ok { - continue - } - if err := h.Adapter.Retire(ctx, as, was.derived, why, now); err != nil { - h.say("%s: retiring failed, will try again: %v", as, err) - continue - } - delete(h.applied, as) - delete(h.lost, as) - delete(h.failing, as) - h.r.retired[as] = retiredEntry{node: was.node, at: now, why: why} - h.recovered(as, "retired") - h.say("%s: RETIRED — its access disabled and its data kept, marked to delete (%s). Asked for again "+ - "it is re-enabled as it was; it is deleted only by `cleanup delete` (novox/hq ADR 0230)", as, why) - done = append(done, as) - said = append(said, map[string]any{"consumer": as, "node": was.node, "retired_at": stamp(now), "why": why}) - } - h.r.key, h.r.count, h.r.waiting, h.r.rejected = "", 0, nil, nil - if len(done) > 0 { - body := map[string]any{"held": held, "why": why} - for k, v := range extra { - body[k] = v - } - h.announce(EventRetirement, h.retirementBody(ChangeRetired, said, body)) - } - return done -} - -// reenabled says a retired consumer was asked for again and its create enabled it. -func (h *Harness) reenabled(as, node string) { - e, was := h.r.retired[as] - if !was { - return - } - delete(h.r.retired, as) - h.say("%s: asked for again — re-enabled as it was, its mark to delete cleared (retired %s: %s)", as, - stamp(e.at), e.why) - h.announce(EventRetirement, h.retirementBody(ChangeReenabled, []map[string]any{ - {"consumer": as, "node": node, "retired_at": stamp(e.at), "why": e.why}, - }, nil)) -} - -// Approve retires exactly the set waiting (or the set rejected) — a person's confirmation. -func (h *Harness) Approve(ctx context.Context, consumers []string, why, by, via string) ([]string, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - if strings.TrimSpace(why) == "" { - return nil, errors.New("approve: say why") - } - set := sorted(consumers) - var held []string - switch { - case h.r.waiting != nil && same(set, h.r.waiting.set): - held = h.r.waiting.set - case h.r.rejected != nil && same(set, h.r.rejected.set): - held = h.r.rejected.set - default: - return nil, fmt.Errorf("approve: %s is not the set waiting here — %s", orNone(set), h.waitingWords()) - } - h.say("retirement of %s APPROVED by %s: %s", strings.Join(held, ", "), orSomebody(by), why) - h.announce(EventRetirement, h.retirementBody(ChangeApproved, h.named(held), - map[string]any{"why": why, "by": by, "via": via})) - reason := fmt.Sprintf("the mesh stopped asking for it; approved by %s: %s", orSomebody(by), why) - return h.retire(ctx, held, reason, map[string]any{"by": by, "via": via}), nil -} - -// Reject keeps the waiting set active; it is not asked about again while it stays the same set. -func (h *Harness) Reject(consumers []string, why, by, via string) ([]string, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - if strings.TrimSpace(why) == "" { - return nil, errors.New("reject: say why") - } - set := sorted(consumers) - if h.r.waiting == nil || !same(set, h.r.waiting.set) { - return nil, fmt.Errorf("reject: %s is not the set waiting here — %s", orNone(set), h.waitingWords()) - } - h.r.rejected = &rejectedSet{set: h.r.waiting.set, by: by, why: why, at: h.Now()} - h.r.waiting = nil - h.say("retirement of %s REJECTED by %s: %s. Kept active, and not asked about again while the mesh goes on "+ - "not asking for exactly these (until this provider restarts)", strings.Join(set, ", "), orSomebody(by), why) - h.announce(EventRetirement, h.retirementBody(ChangeRejected, h.named(set), - map[string]any{"why": why, "by": by, "via": via})) - return set, nil -} - -// DeleteRetired deletes one retired consumer, by a person's word: never an active one, never one the -// mesh asks for. -func (h *Harness) DeleteRetired(ctx context.Context, consumer, why, by, via string) (int64, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - if strings.TrimSpace(why) == "" { - return 0, errors.New("delete: say why") - } - if h.r.lastWant[consumer] { - return 0, fmt.Errorf("delete: the mesh asks for %s — it is not retired", consumer) - } - if _, held := h.applied[consumer]; held { - return 0, fmt.Errorf("delete: %s is active here — only a retired consumer is deleted", consumer) - } - inv, err := h.Adapter.Inventory(ctx) - if err != nil { - return 0, fmt.Errorf("delete: what the backend holds cannot be read, so nothing is deleted: %w", err) - } - var found *Retired - for i := range inv.Retired { - if inv.Retired[i].Consumer == consumer { - found = &inv.Retired[i] - } - } - if found == nil { - return 0, fmt.Errorf("delete: %s is not retired here — only a retired consumer is deleted", consumer) - } - freed, err := h.Adapter.Delete(ctx, *found) - if err != nil { - return 0, err - } - delete(h.r.retired, consumer) - h.say("%s: DELETED by %s: %s (retired %s: %s; %d bytes freed)", consumer, orSomebody(by), why, - stampOr(found.RetiredAt), found.Why, freed) - h.announce(EventRetirement, h.retirementBody(ChangeDeleted, []map[string]any{{ - "consumer": consumer, "node": found.Node, "retired_at": stampOr(found.RetiredAt), "why": found.Why, - "size_bytes": found.SizeBytes, "kind": kindOf(*found), - }}, map[string]any{"why": why, "by": by, "via": via, "freed_bytes": freed})) - return freed, nil -} - -// Retirement is what a person (and the controller's `cleanup list` and its probe) asks: what is -// held, waiting, rejected and retired here. -func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - inv, err := h.Adapter.Inventory(ctx) - if err != nil { - return nil, err - } - var held []string - for as := range h.applied { - held = append(held, as) - } - sort.Strings(held) - retired := []map[string]any{} - for _, r := range inv.Retired { - retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node, - "retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)}) - } - sizes := map[string]int64{} - for _, as := range held { - if size, ok := inv.Sizes[as]; ok && size >= 0 { - sizes[as] = size - } - } - out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), "held_sizes": sizes, - "stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil, - "retired": retired} - if w := h.r.waiting; w != nil { - out["waiting"] = map[string]any{"consumers": h.named(w.set), "since": stamp(w.since), "held": w.held} - } - if r := h.r.rejected; r != nil { - out["rejected"] = map[string]any{"consumers": h.named(r.set), "by": r.by, "why": r.why, "at": stamp(r.at)} - } - return out, nil -} - -func (h *Harness) waitingWords() string { - switch { - case h.r.waiting != nil: - return "waiting: " + strings.Join(h.r.waiting.set, ", ") - case h.r.rejected != nil: - return "nothing waits; rejected and kept: " + strings.Join(h.r.rejected.set, ", ") - } - return "nothing waits for a person here" -} - -// named is a set with each consumer's machine, as the events and the tools say it. -func (h *Harness) named(set []string) []map[string]any { - out := make([]map[string]any, 0, len(set)) - for _, as := range set { - out = append(out, map[string]any{"consumer": as, "node": h.applied[as].node}) - } - return out -} - -func (h *Harness) retirementBody(change string, consumers []map[string]any, extra map[string]any) map[string]any { - body := map[string]any{"provider": h.Resource, "provider-node": h.Node, "change": change, - "consumers": consumers, "at": stamp(h.Now())} - for k, v := range extra { - body[k] = v - } - return body -} - -// RetirementTools are the four tools every provider serves for retirement, the same names in every -// module: the controller's `retire` and `cleanup` verbs ask them on the provider's machine. -func RetirementTools(h *Harness) []stdio.Tool { - if h == nil { - return nil - } - ask := func() (context.Context, context.CancelFunc) { - return context.WithTimeout(context.Background(), 2*time.Minute) - } - who := map[string]any{ - "why": map[string]any{"type": "string", "description": "why — required, kept in the hand-act log"}, - "by": map[string]any{"type": "string", "description": "who decided"}, - "via": map[string]any{"type": "string", "description": "mesh-controller when asked through its verbs"}, - } - withSet := map[string]any{"consumers": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, - "description": "the set, exactly as provisioner_retirement names it"}} - for k, v := range who { - withSet[k] = v - } - withOne := map[string]any{ - "consumer": map[string]any{"type": "string", "description": "the retired consumer to delete"}, - "confirm": map[string]any{"type": "string", "description": "the consumer's name again, to say this is meant"}, - } - for k, v := range who { - withOne[k] = v - } - return []stdio.Tool{ - {Name: "provisioner_retirement", - Description: "What this provider holds, what waits for a person to approve its retirement, what was rejected, " + - "and every retired consumer with when, why and its size (novox/hq ADR 0230).", - Run: func(map[string]any) (any, error) { - ctx, cancel := ask() - defer cancel() - return h.Retirement(ctx) - }}, - {Name: "provisioner_retire_approve", - Description: "Retire exactly the set waiting for a person (or the set rejected earlier): access disabled, data " + - "kept and marked to delete. Use the controller's `retire approve`, which records the hand act.", - Input: withSet, - Run: func(args map[string]any) (any, error) { - ctx, cancel := ask() - defer cancel() - done, err := h.Approve(ctx, strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via")) - if err != nil { - return nil, err - } - return map[string]any{"retired": orEmptyList(done)}, nil - }}, - {Name: "provisioner_retire_reject", - Description: "Keep the set waiting for a person active. Use the controller's `retire reject`.", - Input: withSet, - Run: func(args map[string]any) (any, error) { - kept, err := h.Reject(strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via")) - if err != nil { - return nil, err - } - return map[string]any{"kept": kept}, nil - }}, - {Name: "provisioner_delete", - Description: "Delete one RETIRED consumer — its login and its data — for good. Refused for anything active or " + - "asked for. Use the controller's `cleanup delete`, which records the hand act.", - Input: withOne, - Run: func(args map[string]any) (any, error) { - consumer := argString(args, "consumer") - if consumer == "" || argString(args, "confirm") != consumer { - return nil, errors.New("delete: name the consumer, and repeat it in confirm") - } - ctx, cancel := ask() - defer cancel() - freed, err := h.DeleteRetired(ctx, consumer, argString(args, "why"), argString(args, "by"), argString(args, "via")) - if err != nil { - return nil, err - } - return map[string]any{"deleted": consumer, "freed_bytes": freed}, nil - }}, - } -} - -func strs(v any) []string { - list, _ := v.([]any) - out := []string{} - for _, x := range list { - if s, ok := x.(string); ok && s != "" { - out = append(out, s) - } - } - return out -} - -func sorted(list []string) []string { - out := append([]string(nil), list...) - sort.Strings(out) - return out -} - -func same(a, b []string) bool { - return strings.Join(sorted(a), "\x00") == strings.Join(sorted(b), "\x00") -} - -func orNone(set []string) string { - if len(set) == 0 { - return "an empty set" - } - return strings.Join(set, ", ") -} - -func orSomebody(by string) string { - if by == "" { - return "a person" - } - return by -} - -func orEmptyList(list []string) []string { - if list == nil { - return []string{} - } - return list -} - -func kindOf(r Retired) string { - if r.Kind == "" { - return KindConsumer - } - return r.Kind -} - -func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339) } - -func stampOr(t time.Time) string { - if t.IsZero() { - return "" - } - return stamp(t) -} - -func argString(args map[string]any, key string) string { - s, _ := args[key].(string) - return s -} diff --git a/modules/postgres/cmd/postgres-provider/retirement_test.go b/modules/postgres/cmd/postgres-provider/retirement_test.go deleted file mode 100644 index 407d4c2f..00000000 --- a/modules/postgres/cmd/postgres-provider/retirement_test.go +++ /dev/null @@ -1,571 +0,0 @@ -package main - -// Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is -// retired only after the same result in five consecutive passes, too many at once wait for a person, -// asked for again it is re-enabled, and only a person deletes. The same file in every Go provider. - -import ( - "context" - "errors" - "fmt" - "os" - "strings" - "testing" - "time" -) - -// recorder is a backend that remembers what it holds, active and retired, as a real one's mark does. -type recorder struct { - created []Provision - removed []string // retired, in order - deleted []string - held bool - failing error - holdsErr error - retErr error - inv Inventory - invErr error -} - -func (r *recorder) Create(_ context.Context, p Provision) error { - if r.failing != nil { - return r.failing - } - r.created = append(r.created, p) - r.inv.Retired = withoutRetired(r.inv.Retired, p.As) - if !listHas(r.inv.Active, p.As) { - r.inv.Active = append(r.inv.Active, p.As) - } - return nil -} - -func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error { - if r.retErr != nil { - return r.retErr - } - r.removed = append(r.removed, as) - r.inv.Active = without(r.inv.Active, as) - r.inv.Retired = append(withoutRetired(r.inv.Retired, as), - Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer}) - return nil -} - -func (r *recorder) Holds(context.Context, Provision) (bool, error) { - if r.holdsErr != nil { - return false, r.holdsErr - } - return r.held, nil -} - -func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr } - -func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) { - r.deleted = append(r.deleted, x.Consumer) - r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer) - return x.SizeBytes, nil -} - -func listHas(list []string, s string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} - -func without(list []string, s string) []string { - var out []string - for _, x := range list { - if x != s { - out = append(out, x) - } - } - return out -} - -func withoutRetired(list []Retired, s string) []Retired { - var out []Retired - for _, x := range list { - if x.Consumer != s { - out = append(out, x) - } - } - return out -} - -// holding gives the provider n consumers c1…cn and applies them. -func holding(w *world, n int) []map[string]any { - var given []map[string]any - for i := 1; i <= n; i++ { - given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"}) - } - w.give(given...) - w.h.Reconcile(ctx) - return given -} - -// pass is one reconcile five seconds after the last. -func (w *world) pass() { - w.now = w.now.Add(5 * time.Second) - w.h.Reconcile(ctx) -} - -func retirements(said []announced) []string { - var out []string - for _, a := range said { - if a.event == EventRetirement { - out = append(out, a.body["change"].(string)) - } - } - return out -} - -func noneWaits(a announced) bool { - c, _ := a.body["consumers"].([]map[string]any) - return a.body["change"] == ChangeSettled && len(c) == 0 -} - -func lastRetirement(t *testing.T, said []announced) map[string]any { - t.Helper() - for i := len(said) - 1; i >= 0; i-- { - if said[i].event == EventRetirement { - return said[i].body - } - } - t.Fatal("nothing about retirement was announced") - return nil -} - -func namesOf(body map[string]any) string { - var out []string - for _, c := range body["consumers"].([]map[string]any) { - out = append(out, c["consumer"].(string)) - } - return strings.Join(out, ",") -} - -// settle passes every five seconds until the same answer has held for StableFor, and one pass more. -func (w *world) settle() { w.passes(StableFor + 5*time.Second) } - -func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 1) - w.give() - for i := 0; i < 4; i++ { - w.pass() - } - w.give(given...) - w.pass() - if len(w.a.removed) != 0 || len(retirements(*said)) != 0 { - t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said)) - } -} - -// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold: -// five passes AND ten minutes of the same answer (novox/hq ADR 0230). -func TestFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) { - w, said := standingWorld(t) - holding(w, 1) - w.give() - for i := 0; i < 5; i++ { - w.pass() - } - if len(w.a.removed) != 0 || len(retirements(*said)) != 0 { - t.Fatalf("five passes in 25 s retired %v", w.a.removed) - } - w.passes(StableFor - 30*time.Second) - if len(w.a.removed) != 0 { - t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed) - } - w.passes(time.Minute) - if strings.Join(w.a.removed, ",") != "c1" { - t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed) - } - // Ten minutes in one slow pass are not five passes. - w2 := newWorld(t) - holding(w2, 1) - w2.give() - w2.pass() - w2.now = w2.now.Add(StableFor) - w2.pass() - if len(w2.a.removed) != 0 { - t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed) - } - w2.passes(15 * time.Second) - if len(w2.a.removed) != 1 { - t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed) - } -} - -func TestAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 3) - w.give(given[:2]...) - w.settle() - if strings.Join(w.a.removed, ",") != "c3" { - t.Fatalf("retired %v", w.a.removed) - } - body := lastRetirement(t, *said) - if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" || - !strings.Contains(body["why"].(string), "consecutive passes over 10m") { - t.Fatalf("%v", body) - } - if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" { - t.Fatalf("the backend does not hold it retired: %+v", w.a.inv) - } - // Asked for again: the ordinary create, on the first pass, and said. - created := len(w.a.created) - w.give(given...) - w.pass() - if len(w.a.created) != created+1 || w.a.created[created].As != "c3" { - t.Fatalf("not created again: %v", w.a.created) - } - if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" { - t.Fatalf("%v", body) - } - if len(w.a.inv.Retired) != 0 { - t.Fatalf("still marked retired: %+v", w.a.inv.Retired) - } -} - -func TestAnUnreadablePassStartsTheCountAgain(t *testing.T) { - w := newWorld(t) - holding(w, 1) - w.give() - w.passes(StableFor - time.Minute) - os.WriteFile(w.receives, []byte("{"), 0o600) - w.pass() - w.give() - w.passes(StableFor - time.Minute) - if len(w.a.removed) != 0 { - t.Fatalf("an unreadable pass counted: %v", w.a.removed) - } - w.passes(2 * time.Minute) - if len(w.a.removed) != 1 { - t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed) - } -} - -func TestADifferentSetStartsTheCountAgain(t *testing.T) { - w := newWorld(t) - given := holding(w, 5) - w.give(given[:4]...) - w.passes(StableFor - time.Minute) - w.give(given[:3]...) - w.passes(StableFor - time.Minute) - if len(w.a.removed) != 0 { - t.Fatalf("a changed set kept its count: %v", w.a.removed) - } - w.passes(2 * time.Minute) - if strings.Join(w.a.removed, ",") != "c4,c5" { - t.Fatalf("%v", w.a.removed) - } -} - -func TestAdditionsAndChangesAreNeverDelayed(t *testing.T) { - w := newWorld(t) - holding(w, 1) - w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"}) - w.pass() - if len(w.a.created) != 2 || w.a.created[1].As != "c2" { - t.Fatalf("an addition waited: %v", w.a.created) - } - w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"}) - w.pass() - if len(w.a.created) != 3 || w.a.created[2].As != "c1" { - t.Fatalf("a change waited: %v", w.a.created) - } -} - -func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) { - w, said := standingWorld(t) - holding(w, 4) - w.give() - w.passes(15 * time.Minute) - if len(w.a.removed) != 0 { - t.Fatalf("four of four were retired without a person: %v", w.a.removed) - } - if got := strings.Join(retirements(*said), ","); got != ChangeWaiting { - t.Fatalf("said %q, want one waiting", got) - } - body := lastRetirement(t, *said) - if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" { - t.Fatalf("%v", body) - } - if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") { - t.Fatal("the wait was not said") - } - // Said again every quarter of an hour while it waits. - w.passes(11 * time.Minute) - if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" { - t.Fatalf("%q", got) - } - - if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil { - t.Fatal("approved a set that is not the one waiting") - } - if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil { - t.Fatal("approved without a why") - } - done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller") - if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" { - t.Fatal(done, err, w.a.removed) - } - changes := retirements(*said) - if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" { - t.Fatalf("%v", changes) - } - if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" || - !strings.Contains(b["why"].(string), "the old machine is gone") { - t.Fatalf("%v", b) - } - // Nothing more waits. - w.passes(time.Minute) - if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 { - t.Fatalf("%v", r) - } -} - -func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 4) - w.give() - w.settle() - if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil { - t.Fatal("rejected a set that is not the one waiting") - } - kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller") - if err != nil || len(kept) != 4 { - t.Fatal(kept, err) - } - w.passes(time.Hour) - if len(w.a.removed) != 0 { - t.Fatalf("a rejected set was retired: %v", w.a.removed) - } - if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" { - t.Fatalf("asked again after a rejection: %q", got) - } - r, _ := w.h.Retirement(ctx) - if r["rejected"] == nil || r["waiting"] != nil { - t.Fatalf("%v", r) - } - // One of them asked for again: a different answer, so the rejection is settled and counting - // starts over — three of four is over the bound again, and waits again. - w.give(given[0]) - w.pass() - if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" { - t.Fatalf("%q", got) - } - w.settle() - if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" { - t.Fatalf("%q", got) - } - // A rejected set can still be approved later. - w2, _ := standingWorld(t) - holding(w2, 4) - w2.give() - w2.settle() - w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "") - if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 { - t.Fatal(done, err) - } -} - -func TestAWaitingSetAskedForAgainSettles(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 4) - w.give() - w.settle() - w.give(given...) - w.pass() - if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 { - t.Fatalf("%q %v", got, w.a.removed) - } - if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil { - t.Fatal("approved a set that no longer waits") - } -} - -func TestDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 5) - w.give(given[:3]...) - w.settle() - if strings.Join(w.a.removed, ",") != "c4,c5" { - t.Fatalf("%v", w.a.removed) - } - if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil { - t.Fatal("deleted an active consumer") - } - if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil { - t.Fatal("deleted without a why") - } - if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil { - t.Fatal("deleted something not retired") - } - freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller") - if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" { - t.Fatal(freed, err, w.a.deleted) - } - if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) { - t.Fatalf("%v", b) - } - r, _ := w.h.Retirement(ctx) - if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" { - t.Fatalf("%v", r) - } - // Retired and asked for again before anybody deleted it: refused, it is the mesh's again. - w.give(given[:4]...) - w.pass() - if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil { - t.Fatal("deleted a consumer the mesh asks for") - } -} - -func TestTheBound(t *testing.T) { - h := &Harness{} - h.init() - for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} { - if h.overTheBound(c.n, c.held) { - t.Errorf("%d of %d waits for a person", c.n, c.held) - } - } - for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} { - if !h.overTheBound(c.n, c.held) { - t.Errorf("%d of %d is retired without a person", c.n, c.held) - } - } -} - -func TestARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) { - w, said := standingWorld(t) - w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{ - {Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer}, - {Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"}, - }} - w.give(map[string]any{"as": "kept"}) - w.h.Reconcile(ctx) - if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" { - t.Fatalf("%v", b) - } - if strings.Join(w.a.removed, ",") != "locked-before" { - t.Fatalf("adopting did not mark it: %v", w.a.removed) - } - w.settle() - if strings.Join(w.a.removed, ",") != "locked-before,orphan" { - t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed) - } -} - -func TestABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) { - w := newWorld(t) - w.a.invErr = errors.New("connection refused") - holding(w, 1) - if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") { - t.Fatal(w.said) - } - w.a.invErr = nil - w.a.inv = Inventory{Active: []string{"c1", "orphan"}} - w.pass() - w.settle() - if strings.Join(w.a.removed, ",") != "orphan" { - t.Fatalf("%v", w.a.removed) - } -} - -func TestTheToolsAnswer(t *testing.T) { - w, _ := standingWorld(t) - holding(w, 4) - w.give() - w.settle() - tools := map[string]func(map[string]any) (any, error){} - for _, tool := range RetirementTools(w.h) { - tools[tool.Name] = tool.Run - } - if len(tools) != 4 { - t.Fatalf("%d tools", len(tools)) - } - got, err := tools["provisioner_retirement"](nil) - if err != nil || got.(map[string]any)["waiting"] == nil { - t.Fatal(got, err) - } - set := []any{"c1", "c2", "c3", "c4"} - if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil { - t.Fatal("approved without a why") - } - if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil { - t.Fatal(err) - } - if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil { - t.Fatal("deleted without confirm") - } - if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil { - t.Fatal(err) - } - if strings.Join(w.a.deleted, ",") != "c1" { - t.Fatal(w.a.deleted) - } -} - -func TestAFailingConsumerRetiredIsSaidRecovered(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 2) - w.a.held = false - w.a.failing = errors.New("boom") - w.passes(7 * time.Minute) - w.give(given[0]) - w.settle() - recovered := false - for _, a := range *said { - if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" { - recovered = true - } - } - if !recovered { - t.Fatalf("%v", *said) - } -} - -// everyWord is a world that keeps every word the provider announces. -func everyWord(t *testing.T) (*world, *[]announced) { - w := newWorld(t) - var said []announced - w.h.Announce = func(e string, b map[string]any) { said = append(said, announced{e, b}) } - w.h.Node = "anchor" - return w, &said -} - -// A restarted provider that finds every consumer it holds asked for says once that nothing waits, so the controller -// clears a waiting condition an earlier process raised and this one never knew (issue 937); it says it no more. -func TestARestartedProviderSaysOnceThatNothingWaits(t *testing.T) { - w, said := everyWord(t) - holding(w, 2) - w.pass() - w.pass() - var words []announced - for _, a := range *said { - if a.event == EventRetirement { - words = append(words, a) - } - } - if len(words) != 1 || !noneWaits(words[0]) || words[0].body["provider-node"] != w.h.Node { - t.Fatalf("said %+v, want one settled with no set", words) - } -} - -// A provider whose own set waits does not say that nothing waits. -func TestAProviderThatWaitsDoesNotSayNothingWaits(t *testing.T) { - w, said := everyWord(t) - w.a.inv.Active = []string{"c1", "c2", "c3", "c4", "c5"} - w.give(map[string]any{"as": "c1", "node": "anchor"}) - w.settle() - for _, a := range *said { - if a.event == EventRetirement && noneWaits(a) { - t.Fatalf("said nothing waits while %v waited", w.h.r.waiting) - } - } - if w.h.r.waiting == nil { - t.Fatal("nothing waited: the test does not hold the case") - } -} diff --git a/modules/postgres/cmd/postgres-provider/standing_test.go b/modules/postgres/cmd/postgres-provider/standing_test.go deleted file mode 100644 index a6b416f4..00000000 --- a/modules/postgres/cmd/postgres-provider/standing_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package main - -// A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224): not on the first -// failure, which may be a restart; after FailingAfter of failures with no success between; again -// every SayAgainEvery while it lasts; and recovered on the first success, or when the consumer goes. - -import ( - "errors" - "os" - "strings" - "testing" - "time" -) - -type announced struct { - event string - body map[string]any -} - -func standingWorld(t *testing.T) (*world, *[]announced) { - w := newWorld(t) - var said []announced - w.h.Announce = func(e string, b map[string]any) { - // The first success since start, and the first word that nothing waits (issue 937), are their own - // tests'; every other test reads past them. - if c, _ := b["consumers"].([]map[string]any); b["why"] != "first-success" && - !(e == EventRetirement && b["change"] == ChangeSettled && len(c) == 0) { - said = append(said, announced{e, b}) - } - } - w.h.Node = "anchor" - return w, &said -} - -// passes reconciles every five seconds for d, as Run would. -func (w *world) passes(d time.Duration) { - for end := w.now.Add(d); w.now.Before(end); w.now = w.now.Add(5 * time.Second) { - w.h.Reconcile(ctx) - } -} - -func events(said []announced) string { - var out []string - for _, a := range said { - out = append(out, a.event) - } - return strings.Join(out, ",") -} - -func TestAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass(t *testing.T) { - w, said := standingWorld(t) - w.a.failing = errors.New(`token request failed: 401 {"error":"invalid_grant","error_description":"Invalid user credentials"}`) - w.give(map[string]any{"as": "mesh_home_grafana", "node": "home-server"}) - - w.passes(4 * time.Minute) - if len(*said) != 0 { - t.Fatalf("announced before FailingAfter: %v", events(*said)) - } - w.passes(2 * time.Minute) - if events(*said) != EventFailing { - t.Fatalf("want one %s, got %q", EventFailing, events(*said)) - } - b := (*said)[0].body - if b["consumer"] != "mesh_home_grafana" || b["node"] != "home-server" || b["class"] != ClassCredentials || - b["provider"] != "postgres-database" || b["provider-node"] != "anchor" || b["attempts"].(int) < 60 { - t.Fatalf("%v", b) - } - - // Said again while it lasts, not every pass. - w.passes(14 * time.Minute) - if events(*said) != EventFailing { - t.Fatalf("repeated too soon: %q", events(*said)) - } - w.passes(2 * time.Minute) - if events(*said) != EventFailing+","+EventFailing { - t.Fatalf("not repeated: %q", events(*said)) - } - - // The first success takes it back. - w.a.failing = nil - w.passes(5 * time.Second) - if events(*said) != EventFailing+","+EventFailing+","+EventRecovered { - t.Fatalf("no recovery: %q", events(*said)) - } - if (*said)[2].body["consumer"] != "mesh_home_grafana" { - t.Fatal((*said)[2].body) - } -} - -func TestOneSuccessBetweenFailuresStartsTheRunAgain(t *testing.T) { - w, said := standingWorld(t) - w.a.failing = errors.New("connection refused") - w.give(map[string]any{"as": "a"}) - w.passes(4 * time.Minute) - w.a.failing = nil - w.passes(5 * time.Second) - w.give(map[string]any{"as": "a", "values": map[string]any{"name": "changed"}}) - w.a.failing = errors.New("connection refused") - w.passes(4 * time.Minute) - if len(*said) != 0 { - t.Fatalf("two runs of four minutes are not one of eight: %q", events(*said)) - } -} - -// The check that failed for a day on 2026-10-05: clients already made, every minute's check refused -// at the token. A check that cannot be asked is a failure too. -func TestACheckThatKeepsFailingIsAFailureToo(t *testing.T) { - w, said := standingWorld(t) - w.give(map[string]any{"as": "a"}) - w.h.Reconcile(ctx) - w.a.holdsErr = errors.New("401 invalid_grant") - w.passes(7 * time.Minute) - if events(*said) != EventFailing || (*said)[0].body["class"] != ClassCredentials { - t.Fatalf("%q %v", events(*said), *said) - } - w.a.holdsErr = nil - w.passes(time.Minute + 5*time.Second) - if events(*said) != EventFailing+","+EventRecovered { - t.Fatalf("%q", events(*said)) - } -} - -func TestAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) { - w, said := standingWorld(t) - w.give(map[string]any{"as": "a"}) - os.Remove(w.dir + "/a.secret") - w.passes(6 * time.Minute) - if events(*said) != EventFailing || (*said)[0].body["class"] != ClassSecret { - t.Fatalf("%q %v", events(*said), *said) - } -} - -func TestAConsumerNoLongerAskedForIsNoLongerFailing(t *testing.T) { - w, said := standingWorld(t) - w.a.failing = errors.New("boom") - w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}) - w.passes(6 * time.Minute) - if events(*said) != EventFailing+","+EventFailing { - t.Fatalf("%q", events(*said)) - } - w.give(map[string]any{"as": "a"}) - w.passes(5 * time.Second) - last := (*said)[len(*said)-1] - if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "no longer asked for" { - t.Fatalf("%v", *said) - } -} - -func TestAnErrorIsClassedByItsWords(t *testing.T) { - for text, want := range map[string]string{ - `Keycloak token request failed: 401 {"error":"invalid_grant"}`: ClassCredentials, - `FATAL: password authentication failed for user "postgres"`: ClassCredentials, - `dial tcp 127.0.0.1:5432: connect: connection refused`: ClassUnreachable, - `context deadline exceeded`: ClassUnreachable, - `extension "nope" is not available`: ClassRefused, - } { - if got := ClassOf(text); got != want { - t.Errorf("%s: %s, want %s", text, got, want) - } - } -} - -func TestAnAdapterThatClassesItsOwnErrorsIsBelieved(t *testing.T) { - w, said := standingWorld(t) - w.h.Adapter = classing{w.a} - w.a.failing = errors.New("anything") - w.give(map[string]any{"as": "a"}) - w.passes(6 * time.Minute) - if (*said)[0].body["class"] != "its-own" { - t.Fatal((*said)[0].body) - } -} - -type classing struct{ *recorder } - -func (classing) Class(error) string { return "its-own" } - -// A provider restarted after announcing a failure has forgotten it; its first success for each -// consumer is announced, so the controller clears what it kept rather than naming it for ever. -func TestTheFirstSuccessSinceStartIsAnnouncedOnce(t *testing.T) { - w := newWorld(t) - var said []announced - w.h.Announce = func(e string, b map[string]any) { - if e != EventRetirement { - said = append(said, announced{e, b}) - } - } - w.give(map[string]any{"as": "a"}) - w.passes(3 * time.Minute) - if events(said) != EventRecovered || said[0].body["why"] != "first-success" || said[0].body["consumer"] != "a" { - t.Fatalf("%v", said) - } -} From b0dc3323c936cacd5074311e2062bb5edd8d967b Mon Sep 17 00:00:00 2001 From: "fix-cat-pas@novox" Date: Fri, 16 Oct 2026 19:36:02 +0200 Subject: [PATCH 08/11] Run keycloak on the SDK's harness and moved path, dropping its copies (issue 849) --- .../cmd/keycloak-provider/admin_test.go | 3 +- .../cmd/keycloak-provider/early_test.go | 62 -- .../keycloak/cmd/keycloak-provider/harness.go | 574 ---------------- .../keycloak-provider/harness_same_test.go | 34 - .../cmd/keycloak-provider/harness_test.go | 156 ----- .../keycloak/cmd/keycloak-provider/main.go | 9 +- .../keycloak/cmd/keycloak-provider/moved.go | 267 +------- .../cmd/keycloak-provider/moved_test.go | 341 +--------- .../keycloak/cmd/keycloak-provider/oidc.go | 13 +- .../cmd/keycloak-provider/oidc_test.go | 7 +- .../cmd/keycloak-provider/provisioner.go | 13 +- .../cmd/keycloak-provider/retirement.go | 623 ------------------ .../cmd/keycloak-provider/retirement_test.go | 571 ---------------- .../cmd/keycloak-provider/standing_test.go | 193 ------ 14 files changed, 45 insertions(+), 2821 deletions(-) delete mode 100644 modules/keycloak/cmd/keycloak-provider/early_test.go delete mode 100644 modules/keycloak/cmd/keycloak-provider/harness.go delete mode 100644 modules/keycloak/cmd/keycloak-provider/harness_same_test.go delete mode 100644 modules/keycloak/cmd/keycloak-provider/harness_test.go delete mode 100644 modules/keycloak/cmd/keycloak-provider/retirement.go delete mode 100644 modules/keycloak/cmd/keycloak-provider/retirement_test.go delete mode 100644 modules/keycloak/cmd/keycloak-provider/standing_test.go diff --git a/modules/keycloak/cmd/keycloak-provider/admin_test.go b/modules/keycloak/cmd/keycloak-provider/admin_test.go index 30c9e250..09ae9886 100644 --- a/modules/keycloak/cmd/keycloak-provider/admin_test.go +++ b/modules/keycloak/cmd/keycloak-provider/admin_test.go @@ -8,6 +8,7 @@ package main import ( "context" "errors" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "strings" "testing" "time" @@ -198,7 +199,7 @@ func TestWhileTheAdminIsRefusedTheProvisionerDoesNotAskKeycloak(t *testing.T) { announce: func(string, map[string]any) {}, log: func(string, ...any) {}} logins := w.f.logins err := a.Create(ctx, grafana("s", nil)) - if !errors.Is(err, ErrRejected) || a.Class(err) != ClassCredentials { + if !errors.Is(err, ErrRejected) || a.Class(err) != harness.ClassCredentials { t.Fatal(err) } if _, err := a.Holds(ctx, grafana("s", nil)); !errors.Is(err, ErrRejected) { diff --git a/modules/keycloak/cmd/keycloak-provider/early_test.go b/modules/keycloak/cmd/keycloak-provider/early_test.go deleted file mode 100644 index 9a79a729..00000000 --- a/modules/keycloak/cmd/keycloak-provider/early_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -// The moved path started before its runtime answers asks again until it does (issue 943). - -import ( - "context" - "errors" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/provider" -) - -// earlyBus refuses its first watches as the runtime does before Serve runs, then answers as the bucket bus does. -type earlyBus struct { - *bucketBus - refuse int -} - -func (b *earlyBus) Watch(ctx context.Context, bucket string, handle func(string, []byte)) error { - if b.refuse > 0 { - b.refuse-- - return errors.New("before Serve: a bundle reaches the bus through the runtime that launched it") - } - return b.bucketBus.Watch(ctx, bucket, handle) -} - -func TestAMovedPathStartedTooEarlyAsksAgainUntilTheRuntimeAnswers(t *testing.T) { - w := newMovedWorld(t, map[[2]string]string{}) - w.run.FirstRetry = time.Millisecond - w.run.loaded = false - bus := &earlyBus{bucketBus: w.bus, refuse: 3} - p := provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: w.bus, - Vault: provider.ConsumersFile{Paths: []string{w.file}}, Make: w.run.Make, Gone: w.run.Gone, - Every: time.Hour, Logf: func(string, ...any) {}}) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- w.run.Run(ctx, p, bus) }() - deadline := time.After(time.Second) - for { - w.run.mu.Lock() - loaded := w.run.loaded - w.run.mu.Unlock() - if loaded { - break - } - select { - case err := <-done: - cancel() - t.Fatalf("Run stopped before the runtime answered: %v", err) - case <-deadline: - cancel() - t.Fatal("the plans were never watched") - case <-time.After(time.Millisecond): - } - } - cancel() - <-done - if bus.refuse != 0 { - t.Fatalf("%d refusals left", bus.refuse) - } -} diff --git a/modules/keycloak/cmd/keycloak-provider/harness.go b/modules/keycloak/cmd/keycloak-provider/harness.go deleted file mode 100644 index 5e348ce5..00000000 --- a/modules/keycloak/cmd/keycloak-provider/harness.go +++ /dev/null @@ -1,574 +0,0 @@ -package main - -// The reconcile loop every provider shares, as the TypeScript SDK's runProvisioner runs it -// (@novox/mesh-sdk/provisioner, 0.1.12). The Go SDK has no provisioner yet, so this module carries -// the loop itself, line for line in behaviour; when the Go SDK grows one, this file is what moves -// there (novox/hq ADR 0039: the loop is the SDK's, the adapter is the module's). -// -// Read the contributions the mesh delivered; bring each consumer's resource into being through the -// adapter, under the login and password the mesh minted; retire what the mesh no longer asks for. -// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).** -// -// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer -// whose create, check or secret has failed without one success in between for FailingAfter is -// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and -// again every SayAgainEvery while it lasts; the first success after that is `provisioner.recovered`. -// The controller keeps the newest per provider and consumer and `status` names it. On 2026-10-05 the -// identity provider failed every consumer 31,000 times in a day and said so only in its journal -// (novox/hq issue 179). -// -// **A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person -// (novox/hq ADR 0230, the operator's model of 2026-10-06).** Retiring disables its access — reversibly — -// and marks its login and data "to delete" with when and why; nothing is deleted. Asked for again, the -// ordinary create re-enables it as it was. It is retired only once the same set has gone unasked in -// StablePasses consecutive passes and for StableFor, and a set larger than the bound waits for a -// person. retirement.go. -// -// Carried, identical, by every Go provider until the Go SDK has the loop: postgres and keycloak. -// Each module's `harness_same_test.go` fails when its copy and the other's differ (this file and -// retirement.go). - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/url" - "os" - "strings" - "sync" - "time" -) - -// Provision is one consumer's resource to bring into being — everything the mesh derived and delivered. -type Provision struct { - // As is the login the mesh derived and gave the consumer to present. - As string - // Password is the one the mesh minted, read from the file the host unsealed. - Password string - // Values are what the consumer contributed (e.g. {"name": "letta", "extensions": ["vector"]}). - Values map[string]any - // Derived is what this provider's own definition derives for the consumer (novox/hq ADR 0201). - Derived map[string]any - // At is where the consumer is; Consumer is its node. - At string - Consumer string -} - -// Adapter is the per-service half. -type Adapter interface { - // Create brings the consumer into being under the mesh's login and password — and, for one that - // was retired, re-enables it as it was and clears its mark to delete. - Create(ctx context.Context, p Provision) error - // Retire disables the consumer's access, reversibly, and marks its login and data to delete with - // when and why. It deletes nothing (novox/hq ADR 0230). derived is what the mesh last derived, - // remembered here; nil for a consumer this process did not make. - Retire(ctx context.Context, as string, derived map[string]any, why string, at time.Time) error - // Holds says whether the backend still holds the consumer exactly as p says. Read-only. - Holds(ctx context.Context, p Provision) (bool, error) - // Inventory is what the backend holds that the mesh made: consumers active and retired, read from - // the backend itself so a restart forgets nothing. Read-only. - Inventory(ctx context.Context) (Inventory, error) - // Delete removes one retired consumer — its login and its data — for good. Only ever asked by a - // person, through `cleanup delete`; the harness has checked it is retired and not asked for. - Delete(ctx context.Context, r Retired) (freedBytes int64, err error) -} - -// Harness is the loop's settings and memory. -type Harness struct { - Resource string - Receives string - Adapter Adapter - Every time.Duration // 5s - VerifyEvery time.Duration // 60s - HoldsTimeout time.Duration // 30s - Log func(format string, args ...any) - Now func() time.Time - // Announce publishes one of the provider's standing events; nil announces nothing. Node is the - // machine this provider runs on, said in each. - Announce func(event string, body map[string]any) - Node string - // FailingAfter is how long a consumer fails without a success before it is announced (5m); - // SayAgainEvery is how often it is announced again while it lasts (15m), so a controller that - // missed the first hears the next, and a standing nobody repeats can be told from one that holds. - FailingAfter time.Duration - SayAgainEvery time.Duration - // StablePasses (5) is how many consecutive passes must see the same set no longer asked for, and - // StableFor (10m) how long it must have held since the first of them, before it is retired. RetireAtOnce (3) and RetireFraction (0.5) bound what is retired without a person: - // more consumers than RetireAtOnce, or — where more than one is held — a larger share of those held - // than RetireFraction, waits for `retire approve` (novox/hq ADR 0230). - StablePasses int - StableFor time.Duration - RetireAtOnce int - RetireFraction float64 - - // mu is held by a pass and by every tool a person asks, so the two never interleave. - mu sync.Mutex - verifiedAt time.Time - r retirement - applied map[string]appliedEntry - lost map[string]brake - waiting map[string]int - failing map[string]failure - trouble map[string]*standing - cleared map[string]bool - lastWarning string -} - -// standing is one consumer's unbroken run of failures: since when, how often, and the last error. -type standing struct { - node string - since time.Time - attempts int - class string - text string - saidAt time.Time -} - -// The events a provider's standing is announced as (novox/hq ADR 0224). The controller derives the -// permission to emit them for every module that receives contributions; no manifest lists them. -const ( - EventFailing = "provisioner.failing" - EventRecovered = "provisioner.recovered" -) - -// The classes of error a standing is announced with: what a person reading `status` needs to know -// before reading the journal. An adapter may say better (Classifier). -const ( - ClassCredentials = "credentials-rejected" - ClassUnreachable = "unreachable" - ClassSecret = "secret-unreadable" - ClassRefused = "refused" -) - -// Classifier is an adapter that can say what class an error of its own is. -type Classifier interface { - Class(err error) string -} - -type appliedEntry struct { - hash string - derived map[string]any - node string -} - -type brake struct { - times int - nextAt time.Time -} - -type failure struct { - text string - times int -} - -// The longest a consumer whose create keeps failing to satisfy holds waits between checks. -const maxBackoff = time.Hour - -// How many passes a secret may be unreadable before it stops being called a race (issue 225), and -// once said loudly, how often it is repeated. The same cadence quiets a create that keeps failing -// the same way. -const ( - patiently = 12 - loudlyEvery = 240 -) - -type contribution struct { - As string `json:"as"` - Secret string `json:"secret"` - Node string `json:"node"` - At string `json:"at"` - Values map[string]any `json:"values"` - Derived map[string]any `json:"derived"` -} - -func (h *Harness) init() { - if h.Every == 0 { - h.Every = 5 * time.Second - } - if h.VerifyEvery == 0 { - h.VerifyEvery = time.Minute - } - if h.HoldsTimeout == 0 { - h.HoldsTimeout = 30 * time.Second - } - if h.Now == nil { - h.Now = time.Now - } - if h.FailingAfter == 0 { - h.FailingAfter = 5 * time.Minute - } - if h.SayAgainEvery == 0 { - h.SayAgainEvery = 15 * time.Minute - } - if h.StablePasses == 0 { - h.StablePasses = 5 - } - if h.StableFor == 0 { - h.StableFor = StableFor - } - if h.RetireAtOnce == 0 { - h.RetireAtOnce = 3 - } - if h.RetireFraction == 0 { - h.RetireFraction = 0.5 - } - if h.Log == nil { - h.Log = func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) } - } - if h.applied == nil { - h.applied = map[string]appliedEntry{} - h.lost = map[string]brake{} - h.waiting = map[string]int{} - h.failing = map[string]failure{} - h.trouble = map[string]*standing{} - h.cleared = map[string]bool{} - h.r.retired = map[string]retiredEntry{} - } -} - -// Run reconciles until ctx ends. One consumer's failure never stops the others'. -func (h *Harness) Run(ctx context.Context) { - h.init() - for { - h.Reconcile(ctx) - select { - case <-ctx.Done(): - return - case <-time.After(h.Every): - } - } -} - -func (h *Harness) say(format string, args ...any) { - h.Log("[provisioner:"+h.Resource+"] "+format, args...) -} - -func (h *Harness) warn(why string) { - if why == h.lastWarning { - return - } - if why != "" { - h.say("%s; nothing applied or removed until it can be read", why) - } else { - h.say("contributions file readable again") - } - h.lastWarning = why -} - -// readContributions answers the consumers asked for, or nil when the file says nothing usable. -// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can retire anything. -func (h *Harness) readContributions() []contribution { - raw, err := os.ReadFile(h.Receives) - if err != nil { - h.warn(fmt.Sprintf("contributions file unreadable (%s): %v", h.Receives, err)) - return nil - } - var doc struct { - Requirement string `json:"requirement"` - Given json.RawMessage `json:"given"` - } - if err := json.Unmarshal(raw, &doc); err != nil { - h.warn(fmt.Sprintf("contributions file is not JSON (%s): %v", h.Receives, err)) - return nil - } - if doc.Requirement != "" && doc.Requirement != h.Resource { - h.warn(fmt.Sprintf("%s is for %s, not %s", h.Receives, doc.Requirement, h.Resource)) - return nil - } - var given []contribution - if len(doc.Given) == 0 || string(doc.Given) == "null" || json.Unmarshal(doc.Given, &given) != nil { - h.warn(fmt.Sprintf("%s has no given list", h.Receives)) - return nil - } - h.warn("") - out := []contribution{} - for _, g := range given { - // No `as` is not a credential grant: nothing to create for it. - if g.As != "" && g.Secret != "" { - out = append(out, g) - } - } - return out -} - -func hashOf(as, password string, values, derived map[string]any) string { - // Derived is in the hash: a provider that renames what it derives gave a different resource. - b, _ := json.Marshal([]any{as, password, orEmpty(values), orEmpty(derived)}) - return string(b) -} - -func orEmpty(m map[string]any) map[string]any { - if m == nil { - return map[string]any{} - } - return m -} - -// Reconcile is one pass. -func (h *Harness) Reconcile(ctx context.Context) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - given := h.readContributions() - if given == nil { - // Not a result: the passes that must agree before anything is retired start again. - h.r.key, h.r.count = "", 0 - return - } - want := map[string]bool{} - for _, g := range given { - want[g.As] = true - } - h.seed(ctx, want) - h.apply(ctx, given) - - // Retire what the mesh has stably stopped asking for — within the bound, or with a person. - h.retireUnasked(ctx, want) - h.r.lastWant = want - for as := range h.failing { - if !want[as] { - delete(h.failing, as) - } - } - // A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by - // anyone: said, so a standing the controller keeps for it is cleared rather than left naming a - // consumer that is gone. One still held is said recovered when it is retired. - for as := range h.trouble { - if _, held := h.applied[as]; !want[as] && !held { - h.recovered(as, "no longer asked for") - } - } -} - -// apply makes each consumer the grant file gives, and checks every VerifyEvery that the backend still holds it. The -// caller holds h.mu. One consumer's failure never stops the others'. -func (h *Harness) apply(ctx context.Context, given []contribution) { - verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery - if verifying { - h.verifiedAt = h.Now() - } - - for _, g := range given { - raw, err := os.ReadFile(g.Secret) - if err != nil { - // A secret the host has not written yet is a race on the first pass; past a minute it is - // a person's to look at, and said so (novox/hq issue 225). - n := h.waiting[g.As] + 1 - h.waiting[g.As] = n - if n <= patiently { - h.say("%s: secret not readable yet (%s): %v", g.As, g.Secret, err) - } else if n == patiently+1 || n%loudlyEvery == 0 { - h.say("%s: CANNOT READ the secret after %d attempts (%s): %v. This is not a race any more — "+ - "nothing has been provisioned for this consumer and nothing will be until somebody looks. "+ - "Check who owns the file and who this process runs as (novox/hq issue 225)", g.As, n, g.Secret, err) - } - h.failed(g.As, g.Node, ClassSecret, fmt.Sprintf("secret not readable (%s): %v", g.Secret, err)) - continue - } - delete(h.waiting, g.As) - password := strings.TrimSuffix(string(raw), "\n") - p := Provision{As: g.As, Password: password, Values: orEmpty(g.Values), Derived: orEmpty(g.Derived), At: g.At, Consumer: g.Node} - hash := hashOf(g.As, password, g.Values, g.Derived) - - reapplying := 0 - if was, ok := h.applied[g.As]; ok && was.hash == hash { - if !verifying { - continue - } - b, braked := h.lost[g.As] - if braked && h.Now().Before(b.nextAt) { - continue - } - hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) - held, err := h.Adapter.Holds(hctx, p) - timedOut := errors.Is(hctx.Err(), context.DeadlineExceeded) - cancel() - if err != nil { - // Unable to ask is not evidence of loss. A backend that timed out will time out for - // the next consumer too, so the rest of this pass is not asked. - text := scrub(err, password) - h.say("%s: could not check the backend, will ask again: %s", g.As, text) - h.failed(g.As, g.Node, h.classOf(err, text), text) - if timedOut { - verifying = false - } - continue - } - if held { - delete(h.lost, g.As) - h.succeeded(g.As) - continue - } - reapplying = b.times + 1 - if reapplying == 1 { - h.say("%s: the backend no longer holds it; applying again", g.As) - } else { - h.say("%s: still not held after being applied again (%d times in a row) — create does not "+ - "produce what holds checks; applying again", g.As, reapplying) - } - } - if err := h.Adapter.Create(ctx, p); err != nil { - text := scrub(err, password) - f := h.failing[g.As] - if f.text != text { - f = failure{text: text} - } - f.times++ - h.failing[g.As] = f - // Said each time it changes, and while it stays the same, as rarely as a lost secret. - if f.times == 1 || f.times%loudlyEvery == 0 { - h.say("%s: create failed, will retry: %s", g.As, text) - } - h.failed(g.As, g.Node, h.classOf(err, text), text) - if reapplying > 0 { - h.lost[g.As] = brake{times: reapplying - 1} - } - continue - } - if f, was := h.failing[g.As]; was { - h.say("%s: created, after %d failed attempt(s)", g.As, f.times) - delete(h.failing, g.As) - } - h.succeeded(g.As) - h.applied[g.As] = appliedEntry{hash: hash, derived: p.Derived, node: g.Node} - h.reenabled(g.As, g.Node) - if reapplying == 0 { - delete(h.lost, g.As) - } else { - wait := h.VerifyEvery << (reapplying - 1) - if wait > maxBackoff || wait <= 0 { - wait = maxBackoff - } - h.lost[g.As] = brake{times: reapplying, nextAt: h.Now().Add(wait)} - if reapplying > 1 { - h.say("%s: next check in %s", g.As, wait.Round(time.Second)) - } - } - } -} - -// failed counts one more failure in a consumer's unbroken run, and announces the run once it has -// lasted FailingAfter — then again every SayAgainEvery while it lasts. -func (h *Harness) failed(as, node, class, text string) { - now := h.Now() - s := h.trouble[as] - if s == nil { - s = &standing{since: now} - h.trouble[as] = s - } - s.node, s.class, s.text = node, class, text - s.attempts++ - if now.Sub(s.since) < h.FailingAfter { - return - } - if !s.saidAt.IsZero() && now.Sub(s.saidAt) < h.SayAgainEvery { - return - } - first := s.saidAt.IsZero() - s.saidAt = now - if first { - h.say("%s: FAILING for %s (%d attempts, %s): %s. Announced as %s; `status` names it until it "+ - "succeeds (novox/hq ADR 0224)", as, now.Sub(s.since).Round(time.Second), s.attempts, class, text, EventFailing) - } - h.announce(EventFailing, map[string]any{ - "provider": h.Resource, "provider-node": h.Node, - "consumer": as, "node": node, - "class": class, "error": clip(text), - "since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts, - }) -} - -// succeeded ends a consumer's run of failures; one that was announced is announced recovered. -// -// **And the first success for a consumer since this process started is announced too**, failing or -// not: a provider that announced a failure and was restarted has forgotten it, and without this the -// controller would name the consumer failing for ever after it recovered unheard. -func (h *Harness) succeeded(as string) { - if h.trouble[as] == nil && !h.cleared[as] { - h.cleared[as] = true - h.announce(EventRecovered, map[string]any{ - "provider": h.Resource, "provider-node": h.Node, "consumer": as, "why": "first-success", - }) - return - } - h.cleared[as] = true - h.recovered(as, "") -} - -func (h *Harness) recovered(as, why string) { - s := h.trouble[as] - if s == nil { - return - } - delete(h.trouble, as) - if s.saidAt.IsZero() { - return // never announced, so there is nothing to take back - } - if why == "" { - h.say("%s: recovered after %s and %d failed attempt(s)", as, h.Now().Sub(s.since).Round(time.Second), s.attempts) - } - body := map[string]any{ - "provider": h.Resource, "provider-node": h.Node, "consumer": as, "node": s.node, - "since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts, - } - if why != "" { - body["why"] = why - } - h.announce(EventRecovered, body) -} - -func (h *Harness) announce(event string, body map[string]any) { - if h.Announce != nil { - h.Announce(event, body) - } -} - -// classOf is an error's class: the adapter's word when it has one, else read from the text. -func (h *Harness) classOf(err error, text string) string { - if c, ok := h.Adapter.(Classifier); ok { - if class := c.Class(err); class != "" { - return class - } - } - return ClassOf(text) -} - -// ClassOf reads an error's class from its text — the words the backends the mesh runs use. -func ClassOf(text string) string { - t := strings.ToLower(text) - for _, w := range []string{"invalid_grant", "invalid user credentials", "password authentication failed", - "authentication failed", "unauthorized", " 401"} { - if strings.Contains(t, w) { - return ClassCredentials - } - } - for _, w := range []string{"connection refused", "no such host", "i/o timeout", "deadline exceeded", - "connection reset", "network is unreachable", "no route to host", "eof"} { - if strings.Contains(t, w) { - return ClassUnreachable - } - } - return ClassRefused -} - -// clip keeps an announced error to what belongs in a status line. -func clip(text string) string { - const most = 300 - if len(text) <= most { - return text - } - return text[:most] + "…" -} - -// scrub is an error's text with the consumer's password removed, raw and URL-encoded. -func scrub(err error, password string) string { - text := err.Error() - if password == "" { - return text - } - for _, form := range []string{password, url.QueryEscape(password), url.PathEscape(password)} { - text = strings.ReplaceAll(text, form, "***") - } - return text -} diff --git a/modules/keycloak/cmd/keycloak-provider/harness_same_test.go b/modules/keycloak/cmd/keycloak-provider/harness_same_test.go deleted file mode 100644 index df4b44ac..00000000 --- a/modules/keycloak/cmd/keycloak-provider/harness_same_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -// The provisioner loop is carried, identical, by every Go provider until the Go SDK has it -// (harness.go, retirement.go, and retirement_test.go which tests it). Two copies drift the moment one -// is fixed and the other is not — and the one left behind is the provider that fails a consumer -// without saying so (novox/hq ADR 0224), or retires one it should not (ADR 0230). This holds them to -// one text. Skipped where postgres is not beside this module, as in a build of this one alone. - -import ( - "bytes" - "errors" - "io/fs" - "os" - "testing" -) - -func TestTheHarnessIsTheSameAsPostgress(t *testing.T) { - for _, file := range []string{"harness.go", "retirement.go", "retirement_test.go"} { - theirs, err := os.ReadFile("../../../postgres/cmd/postgres-provider/" + file) - if errors.Is(err, fs.ErrNotExist) { - t.Skip("postgres is not beside this module") - } - if err != nil { - t.Fatal(err) - } - ours, err := os.ReadFile(file) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(ours, theirs) { - t.Fatalf("%s differs from postgres/cmd/postgres-provider/%s: change both, identically", file, file) - } - } -} diff --git a/modules/keycloak/cmd/keycloak-provider/harness_test.go b/modules/keycloak/cmd/keycloak-provider/harness_test.go deleted file mode 100644 index 2112371d..00000000 --- a/modules/keycloak/cmd/keycloak-provider/harness_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package main - -// The shared harness, as postgres tests it (harness.go is the same file in both modules). - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -type world struct { - t *testing.T - dir string - receives string - now time.Time - h *Harness - a *recorder - said []string -} - -func newWorld(t *testing.T) *world { - w := &world{t: t, dir: t.TempDir(), now: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), a: &recorder{held: true}} - w.receives = filepath.Join(w.dir, "mesh.json") - w.h = &Harness{Resource: "oidc-client", Receives: w.receives, Adapter: w.a, - Now: func() time.Time { return w.now }, - Log: func(f string, a ...any) { w.said = append(w.said, f) }} - return w -} - -func (w *world) give(given ...map[string]any) { - for _, g := range given { - secret := filepath.Join(w.dir, g["as"].(string)+".secret") - if err := os.WriteFile(secret, []byte("pw-"+g["as"].(string)+"\n"), 0o600); err != nil { - w.t.Fatal(err) - } - g["secret"] = secret - } - if given == nil { - given = []map[string]any{} - } - raw, _ := json.Marshal(map[string]any{"requirement": "oidc-client", "given": given}) - if err := os.WriteFile(w.receives, raw, 0o600); err != nil { - w.t.Fatal(err) - } -} - -func TestAConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}}) - w.h.Reconcile(ctx) - w.h.Reconcile(ctx) - if len(w.a.created) != 1 { - t.Fatalf("created %d times", len(w.a.created)) - } - p := w.a.created[0] - if p.As != "mesh_ace_letta" || p.Password != "pw-mesh_ace_letta" || p.Consumer != "ace" { - t.Fatalf("%+v", p) - } -} - -func TestAConsumerNoLongerAskedForIsRetiredOnceStable(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}) - w.h.Reconcile(ctx) - w.give(map[string]any{"as": "a"}) - w.settle() // five passes and ten minutes - if strings.Join(w.a.removed, ",") != "b" { - t.Fatal(w.a.removed) - } - // Only a file that says nobody asks retires the last one. - w.give() - w.settle() - if strings.Join(w.a.removed, ",") != "b,a" { - t.Fatal(w.a.removed) - } -} - -func TestNothingReadIsNotNobodyAsking(t *testing.T) { - for name, content := range map[string]string{ - "unreadable": "", - "not JSON": "{", - "no given": `{"requirement": "oidc-client"}`, - "another": `{"requirement": "mssql-database", "given": []}`, - } { - t.Run(name, func(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "a"}) - w.h.Reconcile(ctx) - if content == "" { - os.Remove(w.receives) - } else { - os.WriteFile(w.receives, []byte(content), 0o600) - } - w.h.Reconcile(ctx) - if len(w.a.removed) != 0 { - t.Fatalf("retired %v on a file it could not use", w.a.removed) - } - }) - } -} - -func TestALostConsumerIsAppliedAgainAndBraked(t *testing.T) { - w := newWorld(t) - w.give(map[string]any{"as": "a"}) - w.h.Reconcile(ctx) - w.a.held = false - w.now = w.now.Add(2 * time.Minute) - w.h.Reconcile(ctx) - if len(w.a.created) != 2 { - t.Fatalf("created %d times", len(w.a.created)) - } - // Still not held a minute later: applied again, then braked for two minutes. - w.now = w.now.Add(61 * time.Second) - w.h.Reconcile(ctx) - w.now = w.now.Add(61 * time.Second) - w.h.Reconcile(ctx) - if len(w.a.created) != 3 { - t.Fatalf("not braked: created %d times", len(w.a.created)) - } -} - -func TestAFailingCreateIsRetriedAndSaidOnce(t *testing.T) { - w := newWorld(t) - w.a.failing = &pgErr{"realm refused, password pw-a"} - w.give(map[string]any{"as": "a"}) - for i := 0; i < 5; i++ { - w.h.Reconcile(ctx) - } - n := 0 - for _, s := range w.said { - if strings.Contains(s, "create failed") { - n++ - } - } - if n != 1 { - t.Fatalf("said %d times", n) - } - w.a.failing = nil - w.h.Reconcile(ctx) - if len(w.a.created) != 1 { - t.Fatal("not retried") - } -} - -type pgErr struct{ s string } - -func (e *pgErr) Error() string { return e.s } - -func TestScrubRemovesThePassword(t *testing.T) { - if got := scrub(&pgErr{"bad pw a/b c and a%2Fb+c"}, "a/b c"); strings.Contains(got, "a/b c") || strings.Contains(got, "a%2Fb+c") { - t.Fatal(got) - } -} diff --git a/modules/keycloak/cmd/keycloak-provider/main.go b/modules/keycloak/cmd/keycloak-provider/main.go index d8f9334f..97633a51 100644 --- a/modules/keycloak/cmd/keycloak-provider/main.go +++ b/modules/keycloak/cmd/keycloak-provider/main.go @@ -8,6 +8,7 @@ package main import ( "context" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "os" "time" @@ -43,7 +44,7 @@ func main() { kc.onRejected = guard.Nudge go guard.Run(context.Background()) - var h *Harness + var h *harness.Harness if issuer, err := Issuer(os.Getenv); err != nil { say("%v — the provisioner cannot run without it", err) } else if realm, err := RealmOf(issuer); err != nil { @@ -52,7 +53,7 @@ func main() { // Handed to the provisioner (novox/hq ADR 0373): the provider helper answers the consumers its // machine's client fills the file with; the grant directory is not read. Retirement is the // harness's, never run, so its guards and its tools stand (issue 805). - h = &Harness{ + h = &harness.Harness{ Resource: ProvisionName, Adapter: provisioner{clients: OidcClients{KC: kc, Realm: realm}, guard: guard, announce: announce, log: logStderr}, Log: logStderr, @@ -72,7 +73,7 @@ func main() { } else if receives := os.Getenv("MESH_RECEIVES"); receives == "" { say("MESH_RECEIVES is not set — the provisioner cannot run without it") } else { - h = &Harness{ + h = &harness.Harness{ Resource: "oidc-client", Receives: receives, Adapter: provisioner{clients: OidcClients{KC: kc, Realm: realm}, guard: guard, announce: announce, log: logStderr}, @@ -94,7 +95,7 @@ func main() { }) } // The retirement tools beside keycloak's own (novox/hq ADR 0230). - if err := stdio.Serve("", append(Tools(kc, guard), RetirementTools(h)...)); err != nil { + if err := stdio.Serve("", append(Tools(kc, guard), harness.RetirementTools(h)...)); err != nil { say("%v", err) os.Exit(1) } diff --git a/modules/keycloak/cmd/keycloak-provider/moved.go b/modules/keycloak/cmd/keycloak-provider/moved.go index 7e63eba5..b3b6d1b1 100644 --- a/modules/keycloak/cmd/keycloak-provider/moved.go +++ b/modules/keycloak/cmd/keycloak-provider/moved.go @@ -6,17 +6,11 @@ package main // The helper learns who binds keycloak from the plans, composes each consumer's entry and reads its credential // through ConsumersFile, the file this machine's provisioner client fills. It leaves the rest to the two hooks // here: Make brings the consumer's OIDC client into being exactly as the harness's adapter does, and Gone leaves -// retirement to the harness's guards (retirement.go): the client is disabled and marked to delete, never removed. +// retirement to the harness's guards (the SDK's harness retirement): the client is disabled and marked to delete, never removed. import ( - "context" - "encoding/json" - "errors" - "fmt" - "sync" - "time" - "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provider/harness" ) // ProvisionName is what keycloak provides, as its manifest says. @@ -46,252 +40,13 @@ func valuesOf(e provider.Entry) map[string]any { return values } -// movedRun is keycloak on the provider helper, with the harness's own memory and retirement behind it: what is held, -// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's -// Make and Gone and the loop below drive the same state, so the guards are the harness's, not a copy. -type movedRun struct { - h *Harness - - // FirstRetry is the first wait before a watch asked too early is asked again (issue 943); a second when zero. - FirstRetry time.Duration - - mu sync.Mutex // guards assignments and loaded - assignments map[string][]byte - loaded bool - verified map[string]time.Time -} - -func newMovedRun(h *Harness) *movedRun { - h.mu.Lock() - h.init() - h.mu.Unlock() - return &movedRun{h: h, assignments: map[string][]byte{}, verified: map[string]time.Time{}} -} - -// Make brings one consumer's client into being under its client id (its plan's `as`) and the vault's credential as -// its secret, as the harness does: at once when it is new or changed, and otherwise only when Keycloak no longer -// holds it, which is asked every VerifyEvery. -func (m *movedRun) Make(ctx context.Context, c provider.Consumer) error { - h, e := m.h, c.Entry - if e.As == "" { - return fmt.Errorf("%s has no client id to make", e.Key()) - } - if len(c.Credential) == 0 { - return fmt.Errorf("%s has no credential to make its client with", e.Key()) - } - h.mu.Lock() - defer h.mu.Unlock() - as, password := e.As, string(c.Credential) - p := Provision{As: as, Password: password, Values: valuesOf(e), Consumer: e.Consumer} - hash := hashOf(as, password, p.Values, nil) - if was, ok := h.applied[as]; ok && was.hash == hash { - if h.Now().Sub(m.verified[as]) < h.VerifyEvery { - return nil - } - m.verified[as] = h.Now() - hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) - held, err := h.Adapter.Holds(hctx, p) - cancel() - if err != nil { - text := scrub(err, password) - h.failed(as, e.Consumer, h.classOf(err, text), text) - return errors.New(text) - } - if held { - h.succeeded(as) - return nil - } - h.say("%s: Keycloak no longer holds it; applying again", as) - } - if err := h.Adapter.Create(ctx, p); err != nil { - text := scrub(err, password) - h.failed(as, e.Consumer, h.classOf(err, text), text) - return errors.New(text) - } - h.succeeded(as) - h.applied[as] = appliedEntry{hash: hash, node: e.Consumer} - m.verified[as] = h.Now() - h.reenabled(as, e.Consumer) - return nil -} - -// Gone is the helper asking whether it may forget a consumer that no longer binds. **It never retires**: retiring is -// the harness's rule, the same unasked set over StablePasses passes and StableFor, at most RetireAtOnce at a time -// (retirement.go, ADR 0230), run by retire below. So a consumer missing from one read of the plans loses nothing: -// the helper is told no, and asks again at its next pass, until the client is retired, or asked for again. -func (m *movedRun) Gone(_ context.Context, e provider.Entry) error { - h := m.h - as := e.As - if as == "" { - return nil // refused: nothing was made for it - } - // The plans as they are now, not as the last retirement pass read them: with none read, nothing is let go. - want, read := m.asked(h.Node) - asked := want[as] - h.mu.Lock() - defer h.mu.Unlock() - _, held := h.applied[as] - _, retired := h.r.retired[as] - switch { - case !read: - case asked: - return nil // another holder of the same client id still binds: nothing to retire - case retired, !held && h.r.seeded: - return nil - } - return fmt.Errorf("%s is kept until its retirement is stable (%d passes and %s) and within the bound, or a "+ - "person approves it", as, h.StablePasses, h.StableFor) -} - -// plan is as much of an assignment as is read to learn who binds keycloak. -type plan struct { - Node string `json:"node"` - Module string `json:"module"` - Provisioning *struct { - Provisions []struct { - Provision string `json:"provision"` - Provider string `json:"provider"` - ProviderModule string `json:"provider-module"` - // As is who the consumer is at its provider's end, its slug first (issue 788). - As string `json:"as"` - } `json:"provisions"` - } `json:"provisioning"` -} - -// asked is the client ids the plans bind keycloak for, and whether they could all be read: a plan that is not the -// controller's, or no read yet, is not an answer, so nothing is counted unasked from it (novox/hq issue 241). -func (m *movedRun) asked(node string) (map[string]bool, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if !m.loaded || len(m.assignments) == 0 { - return nil, false - } - want := map[string]bool{} - for key, raw := range m.assignments { - var a plan - if err := json.Unmarshal(raw, &a); err != nil || a.Node+"."+a.Module != key { - return nil, false - } - if a.Provisioning == nil { - continue - } - for _, b := range a.Provisioning.Provisions { - if b.Provider == node && b.ProviderModule == "keycloak" && b.Provision == ProvisionName { - as := b.As - if as == "" { - as = provider.ConsumerIdentity(a.Node, a.Module) // a plan that names none - } - want[as] = true - } - } - } - return want, true -} - -// retire is one pass of the harness's retirement over what the plans bind now. -func (m *movedRun) retire(ctx context.Context) { - h := m.h - want, ok := m.asked(h.Node) - h.mu.Lock() - defer h.mu.Unlock() - if !ok { - // Not a result: the passes that must agree before anything is retired start again. - h.r.key, h.r.count = "", 0 - return - } - h.seed(ctx, want) - h.retireUnasked(ctx, want) - h.r.lastWant = want - for as := range h.failing { - if !want[as] { - delete(h.failing, as) - } - } - for as := range h.trouble { - if _, held := h.applied[as]; !want[as] && !held { - h.recovered(as, "no longer asked for") - } - } -} - -// watch keeps the plans as the bus holds them, from the first read on. -func (m *movedRun) watch(ctx context.Context, bus provider.Bus) error { - err := bus.Watch(ctx, provider.AssignmentsBucket, func(key string, value []byte) { - m.mu.Lock() - defer m.mu.Unlock() - if value == nil { - delete(m.assignments, key) - } else { - m.assignments[key] = append([]byte(nil), value...) - } - }) - if err == nil { - m.mu.Lock() - m.loaded = true - m.mu.Unlock() - } - return err -} - -// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends. -// -// The bus is reached through the runtime that launched this bundle, which answers only once Serve runs, and Run starts -// beside Serve: a watch asked too early is asked again, after a wait that doubles from FirstRetry up to a minute, -// until it is answered or ctx ends, so a start that loses the race never leaves the consumers unanswered (issue 943). -func (m *movedRun) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error { - wait := m.FirstRetry - if wait <= 0 { - wait = time.Second - } - for tries := 1; ; tries++ { - err := m.watch(ctx, bus) - if err == nil { - break - } - if tries == 4 || tries%20 == 0 { - m.h.say("the plans cannot be watched yet (%d tries), asking again: %v", tries, err) - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(wait): - } - wait = min(wait*2, time.Minute) - } - go func() { - for { - m.retire(ctx) - select { - case <-ctx.Done(): - return - case <-time.After(m.h.Every): - } - } - }() - return p.Run(ctx) -} - -// moved is keycloak handed to the provisioner: the helper answering its consumers, and the harness (never run) whose -// retirement and tools it shares. consumersFile is where its machine's client fills its consumers, issuer the one -// keycloak's own config gives (Issuer), served to every consumer. -func moved(consumersFile string, h *Harness, issuer string) (*movedRun, *provider.Provider, provider.Bus, error) { - if consumersFile == "" { - return nil, nil, nil, errors.New("no consumers' file") - } - run := newMovedRun(h) - bus := provider.StateBus("keycloak") - return run, provider.New(provider.Options{ - Node: h.Node, - Module: "keycloak", - Offers: map[string]provider.Offer{ProvisionName: { - Credential: provider.PerConsumer, - Serves: servedValues(issuer), - }}, - Bus: bus, - Vault: provider.ConsumersFile{Paths: []string{consumersFile}}, - Make: run.Make, - Gone: run.Gone, - Every: h.Every, - Logf: func(f string, a ...any) { h.say(f, a...) }, - }), bus, nil +// moved is keycloak handed to the provisioner: the SDK's moved path (harness.Moved) answering its consumers, and the +// harness (never run) whose retirement and tools it shares. What differs from postgres is only this declaration +// (issue 849). +func moved(consumersFile string, h *harness.Harness, issuer string) (*harness.Moved, *provider.Provider, provider.Bus, error) { + return harness.NewMoved(consumersFile, h, harness.MovedSpec{Module: "keycloak", Provision: ProvisionName, + Offer: provider.Offer{Credential: provider.PerConsumer, Serves: servedValues(issuer)}, + Login: func(e provider.Entry) string { return e.As }, + Values: valuesOf, + Noun: "client id", Made: "client", Backend: "Keycloak", WithGrants: false}) } diff --git a/modules/keycloak/cmd/keycloak-provider/moved_test.go b/modules/keycloak/cmd/keycloak-provider/moved_test.go index cfd33f39..7077324e 100644 --- a/modules/keycloak/cmd/keycloak-provider/moved_test.go +++ b/modules/keycloak/cmd/keycloak-provider/moved_test.go @@ -1,13 +1,9 @@ package main import ( - "context" - "encoding/json" - "fmt" "os" "path/filepath" "reflect" - "sync" "testing" "time" @@ -16,336 +12,17 @@ import ( const testIssuer = "https://id.example.org/realms/Novox" -// bucketBus is the helper's bus over in-memory buckets. -type bucketBus struct { - mu sync.Mutex - buckets map[string]map[string][]byte -} +// keycloak's own part of the moved path (issue 849): the shared driving of it is proven in the SDK's harness package. -func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { - b.mu.Lock() - defer b.mu.Unlock() - for k, v := range b.buckets[bucket] { - handle(k, v) +// What a consumer is told is what it contributes, with the names the controller composed for it over them (issue 792). +func TestAConsumersValuesAreWhatItContributesWithItsComposedNames(t *testing.T) { + got := valuesOf(provider.Entry{Contributes: map[string]any{"label": "x", "name": "old"}, + Names: map[string]string{"name": "x.example.org"}}) + if want := map[string]any{"label": "x", "name": "x.example.org"}; !reflect.DeepEqual(got, want) { + t.Fatalf("%v, want %v", got, want) } - return nil -} - -func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { - b.mu.Lock() - defer b.mu.Unlock() - if b.buckets[bucket] == nil { - b.buckets[bucket] = map[string][]byte{} - } - b.buckets[bucket][key] = append([]byte(nil), value...) - return nil -} - -func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { - b.mu.Lock() - defer b.mu.Unlock() - delete(b.buckets[bucket], key) - return nil -} - -// bind is one consumer's assignment, whose plan names this machine's keycloak as the provider and carries what the -// consumer contributes (issue 776). -func (b *bucketBus) bind(node, module string, contributes map[string]any) { - b.bindNamed(node, module, contributes, nil) -} - -// bindNamed is bind with the names the controller composed for the consumer (issue 792). -func (b *bucketBus) bindNamed(node, module string, contributes map[string]any, names map[string]string) { - plan, _ := json.Marshal(map[string]any{"node": node, "module": module, "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak", - "contributes": contributes, "names": names}}}}) - if err := b.Put(context.Background(), provider.AssignmentsBucket, node+"."+module, plan); err != nil { - panic(err) - } -} - -// consumersFile writes the file the machine's client fills: one credential per consumer. -func consumersFile(t *testing.T, bound map[[2]string]string) string { - var list []provider.ConsumerCredential - for k, password := range bound { - list = append(list, provider.ConsumerCredential{Consumer: k[0], Module: k[1], - Credential: provider.PairName("novox", ProvisionName, k[0], k[1], ""), Value: password}) - } - raw, _ := json.Marshal(list) - at := filepath.Join(t.TempDir(), "consumers.json") - if err := os.WriteFile(at, raw, 0o600); err != nil { - t.Fatal(err) - } - return at -} - -// movedWorld is keycloak on the helper over an in-memory bus, with a recorder for a server and a clock to move. -type movedWorld struct { - t *testing.T - a *recorder - bus *bucketBus - h *Harness - run *movedRun - p *provider.Provider - now time.Time - file string -} - -func newMovedWorld(t *testing.T, bound map[[2]string]string) *movedWorld { - w := &movedWorld{t: t, a: &recorder{}, bus: &bucketBus{buckets: map[string]map[string][]byte{}}, - now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)} - w.file = consumersFile(t, bound) - w.h = &Harness{Resource: ProvisionName, Adapter: w.a, Node: "novox", Now: func() time.Time { return w.now }, - Log: func(string, ...any) {}} - var bus provider.Bus - w.run, w.p, bus, _ = moved(w.file, w.h, testIssuer) - _ = bus - if err := w.run.watch(ctx, w.bus); err != nil { - t.Fatal(err) - } - return w -} - -// pass is one helper pass and one retirement pass, five seconds after the last. -func (w *movedWorld) pass() []provider.Outcome { - w.now = w.now.Add(5 * time.Second) - // the plans as the bus holds them now, as the watch would hand them - w.run.assignments = map[string][]byte{} - for k, v := range w.bus.buckets[provider.AssignmentsBucket] { - w.run.assignments[k] = v - } - // the helper reads its buckets from the in-memory bus - w.p = provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: w.bus, - Vault: provider.ConsumersFile{Paths: []string{w.file}}, - Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer, - Serves: servedValues(testIssuer)}}, - Make: w.run.Make, Gone: w.run.Gone, Logf: func(string, ...any) {}}) - out, err := w.p.Pass(ctx) - if err != nil { - w.t.Fatal(err) - } - w.run.retire(ctx) - return out -} - -func bindings(n int) (map[[2]string]string, []string) { - bound := map[[2]string]string{} - var logins []string - for i := 1; i <= n; i++ { - module := fmt.Sprintf("app%d", i) - bound[[2]string{"ace", module}] = "pw-" + module - logins = append(logins, provider.ConsumerIdentity("ace", module)) - } - return bound, logins -} - -// grafanaContributes and grafanaNames are what grafana's plan entry carries for keycloak: its contribution, and the -// names the controller composed for it (issues 776, 792). -var grafanaContributes = map[string]any{"label": "grafana", "endpoint": "web", "callback": "/login/generic_oauth"} -var grafanaNames = map[string]string{"name": "grafana.example.org", "internal-name": "grafana.home.internal"} - -// The consumers' file makes the same call on the adapter as the equivalent grant file does (the harness): the same -// client id, secret, and the values the redirect URIs are made from. -func TestTheConsumersFileMakesWhatTheGrantFileMakes(t *testing.T) { - grafana := provider.ConsumerIdentity("home", "grafana") - values := map[string]any{"name": grafanaNames["name"], "internal-name": grafanaNames["internal-name"]} - for k, v := range grafanaContributes { - values[k] = v - } - - // Today: the grant file, through the harness. - g := newWorld(t) - g.give(map[string]any{"as": grafana, "node": "home", "values": values}) - g.h.Reconcile(ctx) - - // The consumers' file, through the provider helper. - w := newMovedWorld(t, map[[2]string]string{{"home", "grafana"}: "pw-" + grafana}) - w.bus.bindNamed("home", "grafana", grafanaContributes, grafanaNames) - for _, o := range w.pass() { - if o.Status != provider.Answered { - t.Fatalf("%+v", o) - } - } - if len(w.a.created) != 1 || len(g.a.created) != 1 { - t.Fatalf("created %d through the helper, %d through the harness, want 1 each", len(w.a.created), len(g.a.created)) - } - c, want := w.a.created[0], g.a.created[0] - if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer || - !reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) { - t.Fatalf("made %+v, the grant file makes %+v", c, want) - } - if _, redirects, err := RedirectsOf(c.Values); err != nil || - !reflect.DeepEqual(redirects, []string{"https://grafana.example.org/login/generic_oauth", "https://grafana.home.internal/login/generic_oauth"}) { - t.Fatalf("redirects %v, %v", redirects, err) - } - // The entry tells grafana what keycloak serves, the issuer from its own config. - var entry provider.Entry - if err := json.Unmarshal(w.bus.buckets[provider.BucketOf("keycloak")]["home.grafana.oidc-client"], &entry); err != nil { - t.Fatal(err) - } - if entry.Values["issuer"] != testIssuer || entry.Values["token-path"] != "/protocol/openid-connect/token" || - entry.Values["authorization-path"] != "/protocol/openid-connect/auth" || entry.Values["userinfo-path"] != "/protocol/openid-connect/userinfo" { - t.Fatalf("served %v", entry.Values) - } - - // Unchanged, and held: nothing is made again, within a minute or after it. No longer held: made again. - w.a.held = true - w.pass() - w.now = w.now.Add(2 * time.Minute) - w.pass() - if len(w.a.created) != 1 { - t.Fatalf("an unchanged pass created again: %d", len(w.a.created)) - } - w.a.held = false - w.now = w.now.Add(2 * time.Minute) - w.pass() - if len(w.a.created) != 2 { - t.Fatalf("not made again: %d", len(w.a.created)) - } -} - -// Against a Keycloak, both paths leave the same client. -func TestBothPathsLeaveTheSameClientInKeycloak(t *testing.T) { - grafana := provider.ConsumerIdentity("home", "grafana") - values := map[string]any{"name": grafanaNames["name"], "internal-name": grafanaNames["internal-name"]} - for k, v := range grafanaContributes { - values[k] = v - } - made := func(through func(p provisioner)) Rep { - f, o := oidcWorld(t) - through(provisioner{clients: o, announce: func(string, map[string]any) {}, log: func(string, ...any) {}}) - return only(t, f, grafana) - } - viaGrant := made(func(p provisioner) { - g := newWorld(t) - g.h.Adapter = p - g.give(map[string]any{"as": grafana, "node": "home", "values": values}) - g.h.Reconcile(ctx) - }) - viaFile := made(func(p provisioner) { - w := newMovedWorld(t, map[[2]string]string{{"home", "grafana"}: "pw-" + grafana}) - w.h.Adapter = p - w.bus.bindNamed("home", "grafana", grafanaContributes, grafanaNames) - w.pass() - }) - // Keycloak's own ids differ between two runs; nothing the mesh sets does. - for _, c := range []Rep{viaGrant, viaFile} { - delete(c, "id") - raw, _ := json.Marshal(c) - var again Rep - _ = json.Unmarshal(raw, &again) - for k := range c { - delete(c, k) - } - for k, v := range again { - c[k] = v - } - if mappers, ok := c["protocolMappers"].([]any); ok { - for _, m := range mappers { - delete(m.(map[string]any), "id") - } - } - } - if !reflect.DeepEqual(viaGrant, viaFile) { - t.Fatalf("through the file %v, through the grant %v", viaFile, viaGrant) - } -} - -// A consumer that is no longer bound is kept until the harness's retirement is stable, then retired, never deleted. -func TestAConsumerMissingFromOneReadRetiresNothing(t *testing.T) { - bound, logins := bindings(2) - w := newMovedWorld(t, bound) - w.bus.bindNamed("ace", "app1", grafanaContributes, grafanaNames) - w.bus.bindNamed("ace", "app2", grafanaContributes, grafanaNames) - w.pass() - - saved := w.bus.buckets[provider.AssignmentsBucket]["ace.app2"] - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v on one read", w.a.removed) - } - w.bus.Put(ctx, provider.AssignmentsBucket, "ace.app2", saved) - for i := 0; i < 8; i++ { - w.pass() - } - w.now = w.now.Add(15 * time.Minute) - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v", w.a.removed) - } - - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") - for i := 0; i < 6; i++ { - w.pass() - } - if len(w.a.removed) != 0 { - t.Fatalf("retired %v before the time was out", w.a.removed) - } - w.now = w.now.Add(11 * time.Minute) - w.pass() - w.pass() - if !reflect.DeepEqual(w.a.removed, []string{logins[1]}) || len(w.a.deleted) != 0 { - t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) - } - if _, kept := w.bus.buckets[provider.BucketOf("keycloak")]["ace.app2.oidc-client"]; kept { - t.Fatal("the entry stayed after the client was retired") - } -} - -// A plan list that was never read is not nobody asking. -func TestNoPlansReadRetiresNothing(t *testing.T) { - bound, _ := bindings(1) - w := newMovedWorld(t, bound) - w.bus.bindNamed("ace", "app1", grafanaContributes, grafanaNames) - w.pass() - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app1") - for i := 0; i < 8; i++ { - w.pass() - } - w.now = w.now.Add(20 * time.Minute) - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v with no plans read", w.a.removed) - } -} - -// A consumer whose credential is not in the file yet is waited for: nothing is made for it. -func TestAConsumerWithoutACredentialIsWaitedFor(t *testing.T) { - w := newMovedWorld(t, map[[2]string]string{}) - w.bus.bindNamed("home", "grafana", grafanaContributes, grafanaNames) - outcomes := w.pass() - if len(outcomes) != 1 || outcomes[0].Status != provider.Waiting || len(w.a.created) != 0 { - t.Fatalf("%+v, created %v", outcomes, w.a.created) - } -} - -// A consumer whose plan names its client id (a slugged module) keeps it, and the retirement counts it as asked for. -func TestAConsumerNamedByItsPlanKeepsItsClientId(t *testing.T) { - agent := provider.ConsumerIdentity("home", "dashboard") - w := newMovedWorld(t, map[[2]string]string{{"home", "grafana-2"}: "pw"}) - plan, _ := json.Marshal(map[string]any{"node": "home", "module": "grafana-2", "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak", - "as": agent, "contributes": grafanaContributes, "names": grafanaNames}}}}) - if err := w.bus.Put(ctx, provider.AssignmentsBucket, "home.grafana-2", plan); err != nil { - t.Fatal(err) - } - for _, o := range w.pass() { - if o.Status != provider.Answered { - t.Fatalf("%+v", o) - } - } - if len(w.a.created) != 1 || w.a.created[0].As != agent { - t.Fatalf("made %+v, want one under %s", w.a.created, agent) - } - for i := 0; i < 8; i++ { - w.pass() - } - w.now = w.now.Add(15 * time.Minute) - w.pass() - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v: the plan's client id was counted unasked", w.a.removed) + if served := servedValues(testIssuer); served["issuer"] != testIssuer { + t.Fatalf("%v", served) } } diff --git a/modules/keycloak/cmd/keycloak-provider/oidc.go b/modules/keycloak/cmd/keycloak-provider/oidc.go index 030b55e6..c7815462 100644 --- a/modules/keycloak/cmd/keycloak-provider/oidc.go +++ b/modules/keycloak/cmd/keycloak-provider/oidc.go @@ -27,6 +27,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "net/url" "regexp" "sort" @@ -117,7 +118,7 @@ func contains(list []string, s string) bool { } // wanted is the fields the mesh owns on a client it made. Everything else is left as found. -func wanted(p Provision) (Rep, []string, error) { +func wanted(p harness.Provision) (Rep, []string, error) { root, redirects, err := RedirectsOf(p.Values) if err != nil { return nil, nil, err @@ -192,7 +193,7 @@ type OidcClients struct { // Ensure creates the consumer's client, or brings the mesh's existing one back to what the grant // says. Answers "created" or "updated". Idempotent. -func (o OidcClients) Ensure(ctx context.Context, p Provision) (string, error) { +func (o OidcClients) Ensure(ctx context.Context, p harness.Provision) (string, error) { want, _, err := wanted(p) if err != nil { return "", err @@ -270,7 +271,7 @@ func (o OidcClients) ensureMapper(ctx context.Context, id string) error { // Holds says whether Keycloak still holds this consumer's client exactly as the grant says: present, // the mesh's, enabled, confidential, with the mesh's secret and the redirects asked for. Reads only. -func (o OidcClients) Holds(ctx context.Context, p Provision) (bool, error) { +func (o OidcClients) Holds(ctx context.Context, p harness.Provision) (bool, error) { _, redirects, err := wanted(p) if err != nil { return false, err @@ -337,8 +338,8 @@ func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) ( // Inventory is every client in the realm the mesh made: enabled ones active, disabled ones retired — // with when and why from the mark, or none for one disabled before the mark existed. -func (o OidcClients) Inventory(ctx context.Context) (Inventory, error) { - inv := Inventory{Active: []string{}, Retired: []Retired{}} +func (o OidcClients) Inventory(ctx context.Context) (harness.Inventory, error) { + inv := harness.Inventory{Active: []string{}, Retired: []harness.Retired{}} clients, err := o.KC.ListClients(ctx, o.Realm) if err != nil { return inv, err @@ -356,7 +357,7 @@ func (o OidcClients) Inventory(ctx context.Context) (Inventory, error) { } at, _ := time.Parse(time.RFC3339, when) why, _ := a[MarkRetiredWhy].(string) - inv.Retired = append(inv.Retired, Retired{Consumer: id, RetiredAt: at, Why: why, SizeBytes: -1, Kind: KindConsumer}) + inv.Retired = append(inv.Retired, harness.Retired{Consumer: id, RetiredAt: at, Why: why, SizeBytes: -1, Kind: harness.KindConsumer}) } return inv, nil } diff --git a/modules/keycloak/cmd/keycloak-provider/oidc_test.go b/modules/keycloak/cmd/keycloak-provider/oidc_test.go index e8b9963b..45c95ae3 100644 --- a/modules/keycloak/cmd/keycloak-provider/oidc_test.go +++ b/modules/keycloak/cmd/keycloak-provider/oidc_test.go @@ -7,6 +7,7 @@ package main import ( "encoding/json" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "reflect" "strings" "testing" @@ -19,13 +20,13 @@ func oidcWorld(t *testing.T) (*fakeKeycloak, OidcClients) { } // grafana is a dashboard on the home server, as the mesh hands it to the provisioner. -func grafana(secret string, extra map[string]any) Provision { +func grafana(secret string, extra map[string]any) harness.Provision { values := map[string]any{"label": "grafana", "endpoint": "web", "port": 20010.0, "callback": "/login/generic_oauth", "name": "grafana.example.org", "internal-name": "grafana.home.internal"} for k, v := range extra { values[k] = v } - return Provision{As: "mesh_home_grafana", Password: secret, Consumer: "home", Values: values} + return harness.Provision{As: "mesh_home_grafana", Password: secret, Consumer: "home", Values: values} } func only(t *testing.T, f *fakeKeycloak, clientID string) Rep { @@ -42,7 +43,7 @@ func only(t *testing.T, f *fakeKeycloak, clientID string) Rep { return found[0] } -func mustHold(t *testing.T, o OidcClients, p Provision, want bool) { +func mustHold(t *testing.T, o OidcClients, p harness.Provision, want bool) { t.Helper() held, err := o.Holds(ctx, p) if err != nil || held != want { diff --git a/modules/keycloak/cmd/keycloak-provider/provisioner.go b/modules/keycloak/cmd/keycloak-provider/provisioner.go index 45840f73..e0428a4e 100644 --- a/modules/keycloak/cmd/keycloak-provider/provisioner.go +++ b/modules/keycloak/cmd/keycloak-provider/provisioner.go @@ -16,6 +16,7 @@ import ( "context" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/provider/harness" "os" "time" ) @@ -46,7 +47,7 @@ func (a provisioner) refused() error { return nil } -func (a provisioner) Create(ctx context.Context, p Provision) error { +func (a provisioner) Create(ctx context.Context, p harness.Provision) error { if err := a.refused(); err != nil { return err } @@ -81,16 +82,16 @@ func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, wh } // Inventory is every client in the realm the mesh made, active and retired. -func (a provisioner) Inventory(ctx context.Context) (Inventory, error) { +func (a provisioner) Inventory(ctx context.Context) (harness.Inventory, error) { if err := a.refused(); err != nil { - return Inventory{}, err + return harness.Inventory{}, err } return a.clients.Inventory(ctx) } // Delete removes a retired client, a person's act through `cleanup delete`. Nothing is freed that // Keycloak counts in bytes. -func (a provisioner) Delete(ctx context.Context, r Retired) (int64, error) { +func (a provisioner) Delete(ctx context.Context, r harness.Retired) (int64, error) { if err := a.refused(); err != nil { return 0, err } @@ -104,7 +105,7 @@ func (a provisioner) Delete(ctx context.Context, r Retired) (int64, error) { // Holds is asked every minute by the harness: whether Keycloak still holds this consumer's client // exactly as the mesh gave it, so one deleted or edited behind the mesh's back is made again // (novox/hq issue 120). -func (a provisioner) Holds(ctx context.Context, p Provision) (bool, error) { +func (a provisioner) Holds(ctx context.Context, p harness.Provision) (bool, error) { if err := a.refused(); err != nil { return false, err } @@ -114,7 +115,7 @@ func (a provisioner) Holds(ctx context.Context, p Provision) (bool, error) { // Class says a refused admin is a credentials problem, whatever the words around it. func (provisioner) Class(err error) string { if errors.Is(err, ErrRejected) { - return ClassCredentials + return harness.ClassCredentials } return "" } diff --git a/modules/keycloak/cmd/keycloak-provider/retirement.go b/modules/keycloak/cmd/keycloak-provider/retirement.go deleted file mode 100644 index f48afde8..00000000 --- a/modules/keycloak/cmd/keycloak-provider/retirement.go +++ /dev/null @@ -1,623 +0,0 @@ -package main - -// What becomes of a consumer the mesh no longer asks for (novox/hq ADR 0230 — the operator's model, -// decided 2026-10-06; it replaces ADR 0229's withdrawal brake, which let one consumer go every hour). -// -// A consumer — a login, a client, a key, and the data behind it — is in one of three states: -// -// 1. ACTIVE. -// 2. RETIRED. The mesh stopped asking for it: its access is disabled, reversibly, and its login and -// data are marked "to delete" with when and why. Nothing is deleted. Asked for again, the ordinary -// create re-enables it at once, as it was. -// 3. DELETED, only through `cleanup delete`, a person's act, which this provider executes because it -// owns its backend. -// -// **Stable removals.** A consumer is retired only once the same set of consumers has gone unasked BOTH -// in StablePasses (5) consecutive passes that read the file AND for at least StableFor (10 minutes) -// since the first pass that saw it. Five passes alone are twenty-five seconds — shorter than a -// controller restart, a store reconnecting or a file half written. A pass that could not read the file -// is not a result and starts both again; so does a different set. Additions and changes are never -// delayed. -// -// **Too many is a person.** A stable set of more than RetireAtOnce (3), or — where more than one is -// held — of more than RetireFraction (half) of those held, retires nothing: it WAITS, said in the -// journal and announced `provisioner.retirement` `waiting` (again every SayAgainEvery), which the -// controller raises as an urgent condition, until `retire approve ` or `retire reject`. -// An unassignment the controller made itself is no exception: many changes at once means a person is -// at work, and a person confirms once. On 2026-10-04 one misread file withdrew seven consumers at once -// (issue 241); under this rule that is a question, not an act. -// -// **The backend remembers, not this process.** What is retired, since when and why is read from the -// backend's own mark (Adapter.Inventory), so a restart forgets nothing; a consumer the backend holds -// active that the mesh no longer asks for is retired by the same rules after a restart as before one. -// A consumer found disabled without the mark — withdrawn before this record — is ADOPTED as retired: -// marked, said, announced `adopted`, and its clock starts then. -// -// **A rejection lives as long as this process.** Rejected, the set is kept active and not asked about -// again while it stays the same set; a restart asks again — loudly, never silently. - -import ( - "context" - "errors" - "fmt" - "sort" - "strings" - "time" - - stdio "git.novox.be/novox/mesh-sdk/go" -) - -// EventRetirement is the one event a provider says about retirement, its `change` saying what -// happened. The controller derives the permission to emit it for every module that receives -// contributions, as it does the standing events (novox/hq ADR 0224, 0230). -const EventRetirement = "provisioner.retirement" - -// The changes EventRetirement carries. -const ( - ChangeWaiting = "waiting" // a stable set over the bound waits for a person - ChangeSettled = "settled" // a waiting or rejected set ended without being retired - ChangeApproved = "approved" // a person approved the waiting (or rejected) set - ChangeRejected = "rejected" // a person kept the waiting set active - ChangeRetired = "retired" // consumers disabled and marked to delete - ChangeReenabled = "reenabled" // a retired consumer asked for again, enabled as it was - ChangeDeleted = "deleted" // a retired consumer deleted, by a person - ChangeAdopted = "adopted" // found disabled without the mark, now retired on record -) - -// StableFor is the least time the same unasked set must hold before it is retired (novox/hq ADR 0230): -// longer than a controller restart, a store reconnecting or a file half written. -const StableFor = 10 * time.Minute - -// KindConsumer is a retired consumer. A backend may list other things set aside for deletion under a -// word of its own (postgres: a database renamed aside). -const KindConsumer = "consumer" - -// Retired is one thing a provider holds retired, as its backend's mark says. -type Retired struct { - Consumer string - // Node is the consumer's machine, where the mark records it. - Node string - // RetiredAt is when it was retired; zero for one found disabled without the mesh's mark. - RetiredAt time.Time - Why string - // SizeBytes is what deleting it would free; -1 when the backend cannot say. - SizeBytes int64 - Kind string -} - -// Inventory is what a backend holds that the mesh made. -type Inventory struct { - Active []string - Retired []Retired - // Sizes is what each active consumer keeps on the backend, in bytes, where the backend can say: what - // the controller tells an empty replacement of a consumer's data by (novox/hq ADR 0233). - Sizes map[string]int64 -} - -// retirement is the loop's memory of the sets it is counting, waiting on and was told to keep. -type retirement struct { - key string // the unasked set, joined - count int // consecutive passes that saw it - firstSeen time.Time - waiting *waitingSet - rejected *rejectedSet - retired map[string]retiredEntry - lastWant map[string]bool - seeded bool - seedSaid string - // noneSaid is whether this process has said that nothing waits: an earlier process may have left a set - // waiting that this one never knew (issue 937). - noneSaid bool -} - -type waitingSet struct { - set []string - since time.Time - saidAt time.Time - held int -} - -type rejectedSet struct { - set []string - by, why string - at time.Time -} - -type retiredEntry struct { - node string - at time.Time - why string -} - -// seed reads what the backend holds, once per process: an active consumer the mesh no longer asks -// for is held, so it is retired by the same rules as one this process made; one found disabled -// without the mark is adopted as retired. -func (h *Harness) seed(ctx context.Context, want map[string]bool) { - if h.r.seeded { - return - } - inv, err := h.Adapter.Inventory(ctx) - if err != nil { - if said := err.Error(); said != h.r.seedSaid { - h.say("cannot list what the backend holds (%v); a consumer the mesh stopped asking for while this "+ - "provider was down is not retired until it can", err) - h.r.seedSaid = said - } - return - } - h.r.seeded = true - for _, as := range inv.Active { - if _, held := h.applied[as]; !held && !want[as] { - // No hash: asked for again, it is applied again, which is harmless and marks it. - h.applied[as] = appliedEntry{} - h.say("%s: held by the backend and no longer asked for; retired once that holds for %d passes "+ - "and %s (novox/hq ADR 0230)", as, h.StablePasses, h.StableFor) - } - } - now := h.Now() - for _, r := range inv.Retired { - if r.Kind != "" && r.Kind != KindConsumer { - continue - } - if !r.RetiredAt.IsZero() { - h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: r.RetiredAt, why: r.Why} - continue - } - if want[r.Consumer] { - continue // asked for: this pass's create enables it - } - why := "found disabled without the mesh's mark — withdrawn before retirement was recorded " + - "(novox/hq ADR 0230); retired as found" - if err := h.Adapter.Retire(ctx, r.Consumer, nil, why, now); err != nil { - h.say("%s: found disabled and could not be marked retired, will try at the next start: %v", r.Consumer, err) - continue - } - h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: now, why: why} - h.say("%s: ADOPTED as retired — %s", r.Consumer, why) - h.announce(EventRetirement, h.retirementBody(ChangeAdopted, []map[string]any{ - {"consumer": r.Consumer, "node": r.Node, "retired_at": stamp(now), "why": why, "size_bytes": r.SizeBytes}, - }, map[string]any{"why": why})) - } -} - -// retireUnasked counts the passes that saw the same set no longer asked for and, once it is stable, -// retires it — or, past the bound, waits for a person. -func (h *Harness) retireUnasked(ctx context.Context, want map[string]bool) { - var unasked []string - for as := range h.applied { - if !want[as] { - unasked = append(unasked, as) - } - } - sort.Strings(unasked) - key := strings.Join(unasked, "\x00") - now := h.Now() - switch { - case len(unasked) == 0: - h.settle("every consumer held is asked for again") - h.r.key, h.r.count = "", 0 - return - case key != h.r.key: - if h.r.waiting != nil || h.r.rejected != nil { - h.settle("the set the mesh no longer asks for changed") - } - h.r.key, h.r.count, h.r.firstSeen = key, 1, now - h.say("no longer asked for: %s; retired once the same set holds for %d passes and %s", - strings.Join(unasked, ", "), h.StablePasses, h.StableFor) - default: - h.r.count++ - } - if h.r.rejected != nil || h.r.count < h.StablePasses || now.Sub(h.r.firstSeen) < h.StableFor { - return - } - if h.overTheBound(len(unasked), len(h.applied)) { - h.wait(unasked) - return - } - why := fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s, from %s", - h.r.count, now.Sub(h.r.firstSeen).Round(time.Second), stamp(h.r.firstSeen)) - h.retire(ctx, unasked, why, nil) -} - -// overTheBound says retiring n of the held consumers at once needs a person. -func (h *Harness) overTheBound(n, held int) bool { - if n == 0 { - return false - } - return n > h.RetireAtOnce || (held > 1 && float64(n) > h.RetireFraction*float64(held)) -} - -func (h *Harness) bound(held int) string { - return fmt.Sprintf("more than %d at once, or more than %.0f%% of the %d held", h.RetireAtOnce, - h.RetireFraction*100, held) -} - -// wait holds a stable set over the bound for a person, said once and announced again every -// SayAgainEvery so a controller that missed the first hears the next. -func (h *Harness) wait(set []string) { - now := h.Now() - w := h.r.waiting - if w == nil { - w = &waitingSet{set: set, since: now, held: len(h.applied)} - h.r.waiting = w - h.say("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d consumer(s) this provider "+ - "holds (%s), %s. Nothing is retired; each stays active until `retire approve %s ` or "+ - "`retire reject` (novox/hq ADR 0230)", len(set), w.held, strings.Join(set, ", "), h.bound(w.held), h.Node) - } else if now.Sub(w.saidAt) < h.SayAgainEvery { - return - } - w.saidAt = now - h.announce(EventRetirement, h.retirementBody(ChangeWaiting, h.named(set), map[string]any{ - "held": w.held, "bound": h.bound(w.held), "since": stamp(w.since), - })) -} - -// settle ends a waiting or rejected set that the mesh's own answer made moot. -func (h *Harness) settle(why string) { - var set []string - switch { - case h.r.waiting != nil: - set = h.r.waiting.set - case h.r.rejected != nil: - set = h.r.rejected.set - case h.r.noneSaid: - return - default: - // A restart forgets a set an earlier process waited on, and so would never say it ended: the - // controller's condition would ask a person to approve what nobody waits on any more (issue 937). - // The first time this process finds every consumer held asked for, it says so once, with no set. - h.r.noneSaid = true - h.say("nothing waits for a person: %s", why) - h.announce(EventRetirement, h.retirementBody(ChangeSettled, []map[string]any{}, map[string]any{"why": why})) - return - } - h.r.waiting, h.r.rejected = nil, nil - h.say("retirement of %s settled without retiring: %s", strings.Join(set, ", "), why) - h.announce(EventRetirement, h.retirementBody(ChangeSettled, h.named(set), map[string]any{"why": why})) -} - -// retire disables and marks each consumer of set; one that fails stays held and is tried again once -// its set is stable again. -func (h *Harness) retire(ctx context.Context, set []string, why string, extra map[string]any) []string { - now := h.Now() - held := len(h.applied) - var done []string - var said []map[string]any - for _, as := range set { - was, ok := h.applied[as] - if !ok { - continue - } - if err := h.Adapter.Retire(ctx, as, was.derived, why, now); err != nil { - h.say("%s: retiring failed, will try again: %v", as, err) - continue - } - delete(h.applied, as) - delete(h.lost, as) - delete(h.failing, as) - h.r.retired[as] = retiredEntry{node: was.node, at: now, why: why} - h.recovered(as, "retired") - h.say("%s: RETIRED — its access disabled and its data kept, marked to delete (%s). Asked for again "+ - "it is re-enabled as it was; it is deleted only by `cleanup delete` (novox/hq ADR 0230)", as, why) - done = append(done, as) - said = append(said, map[string]any{"consumer": as, "node": was.node, "retired_at": stamp(now), "why": why}) - } - h.r.key, h.r.count, h.r.waiting, h.r.rejected = "", 0, nil, nil - if len(done) > 0 { - body := map[string]any{"held": held, "why": why} - for k, v := range extra { - body[k] = v - } - h.announce(EventRetirement, h.retirementBody(ChangeRetired, said, body)) - } - return done -} - -// reenabled says a retired consumer was asked for again and its create enabled it. -func (h *Harness) reenabled(as, node string) { - e, was := h.r.retired[as] - if !was { - return - } - delete(h.r.retired, as) - h.say("%s: asked for again — re-enabled as it was, its mark to delete cleared (retired %s: %s)", as, - stamp(e.at), e.why) - h.announce(EventRetirement, h.retirementBody(ChangeReenabled, []map[string]any{ - {"consumer": as, "node": node, "retired_at": stamp(e.at), "why": e.why}, - }, nil)) -} - -// Approve retires exactly the set waiting (or the set rejected) — a person's confirmation. -func (h *Harness) Approve(ctx context.Context, consumers []string, why, by, via string) ([]string, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - if strings.TrimSpace(why) == "" { - return nil, errors.New("approve: say why") - } - set := sorted(consumers) - var held []string - switch { - case h.r.waiting != nil && same(set, h.r.waiting.set): - held = h.r.waiting.set - case h.r.rejected != nil && same(set, h.r.rejected.set): - held = h.r.rejected.set - default: - return nil, fmt.Errorf("approve: %s is not the set waiting here — %s", orNone(set), h.waitingWords()) - } - h.say("retirement of %s APPROVED by %s: %s", strings.Join(held, ", "), orSomebody(by), why) - h.announce(EventRetirement, h.retirementBody(ChangeApproved, h.named(held), - map[string]any{"why": why, "by": by, "via": via})) - reason := fmt.Sprintf("the mesh stopped asking for it; approved by %s: %s", orSomebody(by), why) - return h.retire(ctx, held, reason, map[string]any{"by": by, "via": via}), nil -} - -// Reject keeps the waiting set active; it is not asked about again while it stays the same set. -func (h *Harness) Reject(consumers []string, why, by, via string) ([]string, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - if strings.TrimSpace(why) == "" { - return nil, errors.New("reject: say why") - } - set := sorted(consumers) - if h.r.waiting == nil || !same(set, h.r.waiting.set) { - return nil, fmt.Errorf("reject: %s is not the set waiting here — %s", orNone(set), h.waitingWords()) - } - h.r.rejected = &rejectedSet{set: h.r.waiting.set, by: by, why: why, at: h.Now()} - h.r.waiting = nil - h.say("retirement of %s REJECTED by %s: %s. Kept active, and not asked about again while the mesh goes on "+ - "not asking for exactly these (until this provider restarts)", strings.Join(set, ", "), orSomebody(by), why) - h.announce(EventRetirement, h.retirementBody(ChangeRejected, h.named(set), - map[string]any{"why": why, "by": by, "via": via})) - return set, nil -} - -// DeleteRetired deletes one retired consumer, by a person's word: never an active one, never one the -// mesh asks for. -func (h *Harness) DeleteRetired(ctx context.Context, consumer, why, by, via string) (int64, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - if strings.TrimSpace(why) == "" { - return 0, errors.New("delete: say why") - } - if h.r.lastWant[consumer] { - return 0, fmt.Errorf("delete: the mesh asks for %s — it is not retired", consumer) - } - if _, held := h.applied[consumer]; held { - return 0, fmt.Errorf("delete: %s is active here — only a retired consumer is deleted", consumer) - } - inv, err := h.Adapter.Inventory(ctx) - if err != nil { - return 0, fmt.Errorf("delete: what the backend holds cannot be read, so nothing is deleted: %w", err) - } - var found *Retired - for i := range inv.Retired { - if inv.Retired[i].Consumer == consumer { - found = &inv.Retired[i] - } - } - if found == nil { - return 0, fmt.Errorf("delete: %s is not retired here — only a retired consumer is deleted", consumer) - } - freed, err := h.Adapter.Delete(ctx, *found) - if err != nil { - return 0, err - } - delete(h.r.retired, consumer) - h.say("%s: DELETED by %s: %s (retired %s: %s; %d bytes freed)", consumer, orSomebody(by), why, - stampOr(found.RetiredAt), found.Why, freed) - h.announce(EventRetirement, h.retirementBody(ChangeDeleted, []map[string]any{{ - "consumer": consumer, "node": found.Node, "retired_at": stampOr(found.RetiredAt), "why": found.Why, - "size_bytes": found.SizeBytes, "kind": kindOf(*found), - }}, map[string]any{"why": why, "by": by, "via": via, "freed_bytes": freed})) - return freed, nil -} - -// Retirement is what a person (and the controller's `cleanup list` and its probe) asks: what is -// held, waiting, rejected and retired here. -func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.init() - inv, err := h.Adapter.Inventory(ctx) - if err != nil { - return nil, err - } - var held []string - for as := range h.applied { - held = append(held, as) - } - sort.Strings(held) - retired := []map[string]any{} - for _, r := range inv.Retired { - retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node, - "retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)}) - } - sizes := map[string]int64{} - for _, as := range held { - if size, ok := inv.Sizes[as]; ok && size >= 0 { - sizes[as] = size - } - } - out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), "held_sizes": sizes, - "stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil, - "retired": retired} - if w := h.r.waiting; w != nil { - out["waiting"] = map[string]any{"consumers": h.named(w.set), "since": stamp(w.since), "held": w.held} - } - if r := h.r.rejected; r != nil { - out["rejected"] = map[string]any{"consumers": h.named(r.set), "by": r.by, "why": r.why, "at": stamp(r.at)} - } - return out, nil -} - -func (h *Harness) waitingWords() string { - switch { - case h.r.waiting != nil: - return "waiting: " + strings.Join(h.r.waiting.set, ", ") - case h.r.rejected != nil: - return "nothing waits; rejected and kept: " + strings.Join(h.r.rejected.set, ", ") - } - return "nothing waits for a person here" -} - -// named is a set with each consumer's machine, as the events and the tools say it. -func (h *Harness) named(set []string) []map[string]any { - out := make([]map[string]any, 0, len(set)) - for _, as := range set { - out = append(out, map[string]any{"consumer": as, "node": h.applied[as].node}) - } - return out -} - -func (h *Harness) retirementBody(change string, consumers []map[string]any, extra map[string]any) map[string]any { - body := map[string]any{"provider": h.Resource, "provider-node": h.Node, "change": change, - "consumers": consumers, "at": stamp(h.Now())} - for k, v := range extra { - body[k] = v - } - return body -} - -// RetirementTools are the four tools every provider serves for retirement, the same names in every -// module: the controller's `retire` and `cleanup` verbs ask them on the provider's machine. -func RetirementTools(h *Harness) []stdio.Tool { - if h == nil { - return nil - } - ask := func() (context.Context, context.CancelFunc) { - return context.WithTimeout(context.Background(), 2*time.Minute) - } - who := map[string]any{ - "why": map[string]any{"type": "string", "description": "why — required, kept in the hand-act log"}, - "by": map[string]any{"type": "string", "description": "who decided"}, - "via": map[string]any{"type": "string", "description": "mesh-controller when asked through its verbs"}, - } - withSet := map[string]any{"consumers": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, - "description": "the set, exactly as provisioner_retirement names it"}} - for k, v := range who { - withSet[k] = v - } - withOne := map[string]any{ - "consumer": map[string]any{"type": "string", "description": "the retired consumer to delete"}, - "confirm": map[string]any{"type": "string", "description": "the consumer's name again, to say this is meant"}, - } - for k, v := range who { - withOne[k] = v - } - return []stdio.Tool{ - {Name: "provisioner_retirement", - Description: "What this provider holds, what waits for a person to approve its retirement, what was rejected, " + - "and every retired consumer with when, why and its size (novox/hq ADR 0230).", - Run: func(map[string]any) (any, error) { - ctx, cancel := ask() - defer cancel() - return h.Retirement(ctx) - }}, - {Name: "provisioner_retire_approve", - Description: "Retire exactly the set waiting for a person (or the set rejected earlier): access disabled, data " + - "kept and marked to delete. Use the controller's `retire approve`, which records the hand act.", - Input: withSet, - Run: func(args map[string]any) (any, error) { - ctx, cancel := ask() - defer cancel() - done, err := h.Approve(ctx, strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via")) - if err != nil { - return nil, err - } - return map[string]any{"retired": orEmptyList(done)}, nil - }}, - {Name: "provisioner_retire_reject", - Description: "Keep the set waiting for a person active. Use the controller's `retire reject`.", - Input: withSet, - Run: func(args map[string]any) (any, error) { - kept, err := h.Reject(strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via")) - if err != nil { - return nil, err - } - return map[string]any{"kept": kept}, nil - }}, - {Name: "provisioner_delete", - Description: "Delete one RETIRED consumer — its login and its data — for good. Refused for anything active or " + - "asked for. Use the controller's `cleanup delete`, which records the hand act.", - Input: withOne, - Run: func(args map[string]any) (any, error) { - consumer := argString(args, "consumer") - if consumer == "" || argString(args, "confirm") != consumer { - return nil, errors.New("delete: name the consumer, and repeat it in confirm") - } - ctx, cancel := ask() - defer cancel() - freed, err := h.DeleteRetired(ctx, consumer, argString(args, "why"), argString(args, "by"), argString(args, "via")) - if err != nil { - return nil, err - } - return map[string]any{"deleted": consumer, "freed_bytes": freed}, nil - }}, - } -} - -func strs(v any) []string { - list, _ := v.([]any) - out := []string{} - for _, x := range list { - if s, ok := x.(string); ok && s != "" { - out = append(out, s) - } - } - return out -} - -func sorted(list []string) []string { - out := append([]string(nil), list...) - sort.Strings(out) - return out -} - -func same(a, b []string) bool { - return strings.Join(sorted(a), "\x00") == strings.Join(sorted(b), "\x00") -} - -func orNone(set []string) string { - if len(set) == 0 { - return "an empty set" - } - return strings.Join(set, ", ") -} - -func orSomebody(by string) string { - if by == "" { - return "a person" - } - return by -} - -func orEmptyList(list []string) []string { - if list == nil { - return []string{} - } - return list -} - -func kindOf(r Retired) string { - if r.Kind == "" { - return KindConsumer - } - return r.Kind -} - -func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339) } - -func stampOr(t time.Time) string { - if t.IsZero() { - return "" - } - return stamp(t) -} - -func argString(args map[string]any, key string) string { - s, _ := args[key].(string) - return s -} diff --git a/modules/keycloak/cmd/keycloak-provider/retirement_test.go b/modules/keycloak/cmd/keycloak-provider/retirement_test.go deleted file mode 100644 index 407d4c2f..00000000 --- a/modules/keycloak/cmd/keycloak-provider/retirement_test.go +++ /dev/null @@ -1,571 +0,0 @@ -package main - -// Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is -// retired only after the same result in five consecutive passes, too many at once wait for a person, -// asked for again it is re-enabled, and only a person deletes. The same file in every Go provider. - -import ( - "context" - "errors" - "fmt" - "os" - "strings" - "testing" - "time" -) - -// recorder is a backend that remembers what it holds, active and retired, as a real one's mark does. -type recorder struct { - created []Provision - removed []string // retired, in order - deleted []string - held bool - failing error - holdsErr error - retErr error - inv Inventory - invErr error -} - -func (r *recorder) Create(_ context.Context, p Provision) error { - if r.failing != nil { - return r.failing - } - r.created = append(r.created, p) - r.inv.Retired = withoutRetired(r.inv.Retired, p.As) - if !listHas(r.inv.Active, p.As) { - r.inv.Active = append(r.inv.Active, p.As) - } - return nil -} - -func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error { - if r.retErr != nil { - return r.retErr - } - r.removed = append(r.removed, as) - r.inv.Active = without(r.inv.Active, as) - r.inv.Retired = append(withoutRetired(r.inv.Retired, as), - Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer}) - return nil -} - -func (r *recorder) Holds(context.Context, Provision) (bool, error) { - if r.holdsErr != nil { - return false, r.holdsErr - } - return r.held, nil -} - -func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr } - -func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) { - r.deleted = append(r.deleted, x.Consumer) - r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer) - return x.SizeBytes, nil -} - -func listHas(list []string, s string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} - -func without(list []string, s string) []string { - var out []string - for _, x := range list { - if x != s { - out = append(out, x) - } - } - return out -} - -func withoutRetired(list []Retired, s string) []Retired { - var out []Retired - for _, x := range list { - if x.Consumer != s { - out = append(out, x) - } - } - return out -} - -// holding gives the provider n consumers c1…cn and applies them. -func holding(w *world, n int) []map[string]any { - var given []map[string]any - for i := 1; i <= n; i++ { - given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"}) - } - w.give(given...) - w.h.Reconcile(ctx) - return given -} - -// pass is one reconcile five seconds after the last. -func (w *world) pass() { - w.now = w.now.Add(5 * time.Second) - w.h.Reconcile(ctx) -} - -func retirements(said []announced) []string { - var out []string - for _, a := range said { - if a.event == EventRetirement { - out = append(out, a.body["change"].(string)) - } - } - return out -} - -func noneWaits(a announced) bool { - c, _ := a.body["consumers"].([]map[string]any) - return a.body["change"] == ChangeSettled && len(c) == 0 -} - -func lastRetirement(t *testing.T, said []announced) map[string]any { - t.Helper() - for i := len(said) - 1; i >= 0; i-- { - if said[i].event == EventRetirement { - return said[i].body - } - } - t.Fatal("nothing about retirement was announced") - return nil -} - -func namesOf(body map[string]any) string { - var out []string - for _, c := range body["consumers"].([]map[string]any) { - out = append(out, c["consumer"].(string)) - } - return strings.Join(out, ",") -} - -// settle passes every five seconds until the same answer has held for StableFor, and one pass more. -func (w *world) settle() { w.passes(StableFor + 5*time.Second) } - -func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 1) - w.give() - for i := 0; i < 4; i++ { - w.pass() - } - w.give(given...) - w.pass() - if len(w.a.removed) != 0 || len(retirements(*said)) != 0 { - t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said)) - } -} - -// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold: -// five passes AND ten minutes of the same answer (novox/hq ADR 0230). -func TestFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) { - w, said := standingWorld(t) - holding(w, 1) - w.give() - for i := 0; i < 5; i++ { - w.pass() - } - if len(w.a.removed) != 0 || len(retirements(*said)) != 0 { - t.Fatalf("five passes in 25 s retired %v", w.a.removed) - } - w.passes(StableFor - 30*time.Second) - if len(w.a.removed) != 0 { - t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed) - } - w.passes(time.Minute) - if strings.Join(w.a.removed, ",") != "c1" { - t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed) - } - // Ten minutes in one slow pass are not five passes. - w2 := newWorld(t) - holding(w2, 1) - w2.give() - w2.pass() - w2.now = w2.now.Add(StableFor) - w2.pass() - if len(w2.a.removed) != 0 { - t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed) - } - w2.passes(15 * time.Second) - if len(w2.a.removed) != 1 { - t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed) - } -} - -func TestAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 3) - w.give(given[:2]...) - w.settle() - if strings.Join(w.a.removed, ",") != "c3" { - t.Fatalf("retired %v", w.a.removed) - } - body := lastRetirement(t, *said) - if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" || - !strings.Contains(body["why"].(string), "consecutive passes over 10m") { - t.Fatalf("%v", body) - } - if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" { - t.Fatalf("the backend does not hold it retired: %+v", w.a.inv) - } - // Asked for again: the ordinary create, on the first pass, and said. - created := len(w.a.created) - w.give(given...) - w.pass() - if len(w.a.created) != created+1 || w.a.created[created].As != "c3" { - t.Fatalf("not created again: %v", w.a.created) - } - if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" { - t.Fatalf("%v", body) - } - if len(w.a.inv.Retired) != 0 { - t.Fatalf("still marked retired: %+v", w.a.inv.Retired) - } -} - -func TestAnUnreadablePassStartsTheCountAgain(t *testing.T) { - w := newWorld(t) - holding(w, 1) - w.give() - w.passes(StableFor - time.Minute) - os.WriteFile(w.receives, []byte("{"), 0o600) - w.pass() - w.give() - w.passes(StableFor - time.Minute) - if len(w.a.removed) != 0 { - t.Fatalf("an unreadable pass counted: %v", w.a.removed) - } - w.passes(2 * time.Minute) - if len(w.a.removed) != 1 { - t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed) - } -} - -func TestADifferentSetStartsTheCountAgain(t *testing.T) { - w := newWorld(t) - given := holding(w, 5) - w.give(given[:4]...) - w.passes(StableFor - time.Minute) - w.give(given[:3]...) - w.passes(StableFor - time.Minute) - if len(w.a.removed) != 0 { - t.Fatalf("a changed set kept its count: %v", w.a.removed) - } - w.passes(2 * time.Minute) - if strings.Join(w.a.removed, ",") != "c4,c5" { - t.Fatalf("%v", w.a.removed) - } -} - -func TestAdditionsAndChangesAreNeverDelayed(t *testing.T) { - w := newWorld(t) - holding(w, 1) - w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"}) - w.pass() - if len(w.a.created) != 2 || w.a.created[1].As != "c2" { - t.Fatalf("an addition waited: %v", w.a.created) - } - w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"}) - w.pass() - if len(w.a.created) != 3 || w.a.created[2].As != "c1" { - t.Fatalf("a change waited: %v", w.a.created) - } -} - -func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) { - w, said := standingWorld(t) - holding(w, 4) - w.give() - w.passes(15 * time.Minute) - if len(w.a.removed) != 0 { - t.Fatalf("four of four were retired without a person: %v", w.a.removed) - } - if got := strings.Join(retirements(*said), ","); got != ChangeWaiting { - t.Fatalf("said %q, want one waiting", got) - } - body := lastRetirement(t, *said) - if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" { - t.Fatalf("%v", body) - } - if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") { - t.Fatal("the wait was not said") - } - // Said again every quarter of an hour while it waits. - w.passes(11 * time.Minute) - if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" { - t.Fatalf("%q", got) - } - - if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil { - t.Fatal("approved a set that is not the one waiting") - } - if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil { - t.Fatal("approved without a why") - } - done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller") - if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" { - t.Fatal(done, err, w.a.removed) - } - changes := retirements(*said) - if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" { - t.Fatalf("%v", changes) - } - if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" || - !strings.Contains(b["why"].(string), "the old machine is gone") { - t.Fatalf("%v", b) - } - // Nothing more waits. - w.passes(time.Minute) - if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 { - t.Fatalf("%v", r) - } -} - -func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 4) - w.give() - w.settle() - if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil { - t.Fatal("rejected a set that is not the one waiting") - } - kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller") - if err != nil || len(kept) != 4 { - t.Fatal(kept, err) - } - w.passes(time.Hour) - if len(w.a.removed) != 0 { - t.Fatalf("a rejected set was retired: %v", w.a.removed) - } - if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" { - t.Fatalf("asked again after a rejection: %q", got) - } - r, _ := w.h.Retirement(ctx) - if r["rejected"] == nil || r["waiting"] != nil { - t.Fatalf("%v", r) - } - // One of them asked for again: a different answer, so the rejection is settled and counting - // starts over — three of four is over the bound again, and waits again. - w.give(given[0]) - w.pass() - if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" { - t.Fatalf("%q", got) - } - w.settle() - if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" { - t.Fatalf("%q", got) - } - // A rejected set can still be approved later. - w2, _ := standingWorld(t) - holding(w2, 4) - w2.give() - w2.settle() - w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "") - if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 { - t.Fatal(done, err) - } -} - -func TestAWaitingSetAskedForAgainSettles(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 4) - w.give() - w.settle() - w.give(given...) - w.pass() - if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 { - t.Fatalf("%q %v", got, w.a.removed) - } - if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil { - t.Fatal("approved a set that no longer waits") - } -} - -func TestDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 5) - w.give(given[:3]...) - w.settle() - if strings.Join(w.a.removed, ",") != "c4,c5" { - t.Fatalf("%v", w.a.removed) - } - if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil { - t.Fatal("deleted an active consumer") - } - if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil { - t.Fatal("deleted without a why") - } - if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil { - t.Fatal("deleted something not retired") - } - freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller") - if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" { - t.Fatal(freed, err, w.a.deleted) - } - if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) { - t.Fatalf("%v", b) - } - r, _ := w.h.Retirement(ctx) - if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" { - t.Fatalf("%v", r) - } - // Retired and asked for again before anybody deleted it: refused, it is the mesh's again. - w.give(given[:4]...) - w.pass() - if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil { - t.Fatal("deleted a consumer the mesh asks for") - } -} - -func TestTheBound(t *testing.T) { - h := &Harness{} - h.init() - for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} { - if h.overTheBound(c.n, c.held) { - t.Errorf("%d of %d waits for a person", c.n, c.held) - } - } - for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} { - if !h.overTheBound(c.n, c.held) { - t.Errorf("%d of %d is retired without a person", c.n, c.held) - } - } -} - -func TestARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) { - w, said := standingWorld(t) - w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{ - {Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer}, - {Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"}, - }} - w.give(map[string]any{"as": "kept"}) - w.h.Reconcile(ctx) - if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" { - t.Fatalf("%v", b) - } - if strings.Join(w.a.removed, ",") != "locked-before" { - t.Fatalf("adopting did not mark it: %v", w.a.removed) - } - w.settle() - if strings.Join(w.a.removed, ",") != "locked-before,orphan" { - t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed) - } -} - -func TestABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) { - w := newWorld(t) - w.a.invErr = errors.New("connection refused") - holding(w, 1) - if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") { - t.Fatal(w.said) - } - w.a.invErr = nil - w.a.inv = Inventory{Active: []string{"c1", "orphan"}} - w.pass() - w.settle() - if strings.Join(w.a.removed, ",") != "orphan" { - t.Fatalf("%v", w.a.removed) - } -} - -func TestTheToolsAnswer(t *testing.T) { - w, _ := standingWorld(t) - holding(w, 4) - w.give() - w.settle() - tools := map[string]func(map[string]any) (any, error){} - for _, tool := range RetirementTools(w.h) { - tools[tool.Name] = tool.Run - } - if len(tools) != 4 { - t.Fatalf("%d tools", len(tools)) - } - got, err := tools["provisioner_retirement"](nil) - if err != nil || got.(map[string]any)["waiting"] == nil { - t.Fatal(got, err) - } - set := []any{"c1", "c2", "c3", "c4"} - if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil { - t.Fatal("approved without a why") - } - if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil { - t.Fatal(err) - } - if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil { - t.Fatal("deleted without confirm") - } - if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil { - t.Fatal(err) - } - if strings.Join(w.a.deleted, ",") != "c1" { - t.Fatal(w.a.deleted) - } -} - -func TestAFailingConsumerRetiredIsSaidRecovered(t *testing.T) { - w, said := standingWorld(t) - given := holding(w, 2) - w.a.held = false - w.a.failing = errors.New("boom") - w.passes(7 * time.Minute) - w.give(given[0]) - w.settle() - recovered := false - for _, a := range *said { - if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" { - recovered = true - } - } - if !recovered { - t.Fatalf("%v", *said) - } -} - -// everyWord is a world that keeps every word the provider announces. -func everyWord(t *testing.T) (*world, *[]announced) { - w := newWorld(t) - var said []announced - w.h.Announce = func(e string, b map[string]any) { said = append(said, announced{e, b}) } - w.h.Node = "anchor" - return w, &said -} - -// A restarted provider that finds every consumer it holds asked for says once that nothing waits, so the controller -// clears a waiting condition an earlier process raised and this one never knew (issue 937); it says it no more. -func TestARestartedProviderSaysOnceThatNothingWaits(t *testing.T) { - w, said := everyWord(t) - holding(w, 2) - w.pass() - w.pass() - var words []announced - for _, a := range *said { - if a.event == EventRetirement { - words = append(words, a) - } - } - if len(words) != 1 || !noneWaits(words[0]) || words[0].body["provider-node"] != w.h.Node { - t.Fatalf("said %+v, want one settled with no set", words) - } -} - -// A provider whose own set waits does not say that nothing waits. -func TestAProviderThatWaitsDoesNotSayNothingWaits(t *testing.T) { - w, said := everyWord(t) - w.a.inv.Active = []string{"c1", "c2", "c3", "c4", "c5"} - w.give(map[string]any{"as": "c1", "node": "anchor"}) - w.settle() - for _, a := range *said { - if a.event == EventRetirement && noneWaits(a) { - t.Fatalf("said nothing waits while %v waited", w.h.r.waiting) - } - } - if w.h.r.waiting == nil { - t.Fatal("nothing waited: the test does not hold the case") - } -} diff --git a/modules/keycloak/cmd/keycloak-provider/standing_test.go b/modules/keycloak/cmd/keycloak-provider/standing_test.go deleted file mode 100644 index 870bb95a..00000000 --- a/modules/keycloak/cmd/keycloak-provider/standing_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package main - -// A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224): not on the first -// failure, which may be a restart; after FailingAfter of failures with no success between; again -// every SayAgainEvery while it lasts; and recovered on the first success, or when the consumer goes. - -import ( - "errors" - "os" - "strings" - "testing" - "time" -) - -type announced struct { - event string - body map[string]any -} - -func standingWorld(t *testing.T) (*world, *[]announced) { - w := newWorld(t) - var said []announced - w.h.Announce = func(e string, b map[string]any) { - // The first success since start, and the first word that nothing waits (issue 937), are their own - // tests'; every other test reads past them. - if c, _ := b["consumers"].([]map[string]any); b["why"] != "first-success" && - !(e == EventRetirement && b["change"] == ChangeSettled && len(c) == 0) { - said = append(said, announced{e, b}) - } - } - w.h.Node = "anchor" - return w, &said -} - -// passes reconciles every five seconds for d, as Run would. -func (w *world) passes(d time.Duration) { - for end := w.now.Add(d); w.now.Before(end); w.now = w.now.Add(5 * time.Second) { - w.h.Reconcile(ctx) - } -} - -func events(said []announced) string { - var out []string - for _, a := range said { - out = append(out, a.event) - } - return strings.Join(out, ",") -} - -func TestAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass(t *testing.T) { - w, said := standingWorld(t) - w.a.failing = errors.New(`token request failed: 401 {"error":"invalid_grant","error_description":"Invalid user credentials"}`) - w.give(map[string]any{"as": "mesh_home_grafana", "node": "home-server"}) - - w.passes(4 * time.Minute) - if len(*said) != 0 { - t.Fatalf("announced before FailingAfter: %v", events(*said)) - } - w.passes(2 * time.Minute) - if events(*said) != EventFailing { - t.Fatalf("want one %s, got %q", EventFailing, events(*said)) - } - b := (*said)[0].body - if b["consumer"] != "mesh_home_grafana" || b["node"] != "home-server" || b["class"] != ClassCredentials || - b["provider"] != "oidc-client" || b["provider-node"] != "anchor" || b["attempts"].(int) < 60 { - t.Fatalf("%v", b) - } - - // Said again while it lasts, not every pass. - w.passes(14 * time.Minute) - if events(*said) != EventFailing { - t.Fatalf("repeated too soon: %q", events(*said)) - } - w.passes(2 * time.Minute) - if events(*said) != EventFailing+","+EventFailing { - t.Fatalf("not repeated: %q", events(*said)) - } - - // The first success takes it back. - w.a.failing = nil - w.passes(5 * time.Second) - if events(*said) != EventFailing+","+EventFailing+","+EventRecovered { - t.Fatalf("no recovery: %q", events(*said)) - } - if (*said)[2].body["consumer"] != "mesh_home_grafana" { - t.Fatal((*said)[2].body) - } -} - -func TestOneSuccessBetweenFailuresStartsTheRunAgain(t *testing.T) { - w, said := standingWorld(t) - w.a.failing = errors.New("connection refused") - w.give(map[string]any{"as": "a"}) - w.passes(4 * time.Minute) - w.a.failing = nil - w.passes(5 * time.Second) - w.give(map[string]any{"as": "a", "values": map[string]any{"name": "changed"}}) - w.a.failing = errors.New("connection refused") - w.passes(4 * time.Minute) - if len(*said) != 0 { - t.Fatalf("two runs of four minutes are not one of eight: %q", events(*said)) - } -} - -// The check that failed for a day on 2026-10-05: clients already made, every minute's check refused -// at the token. A check that cannot be asked is a failure too. -func TestACheckThatKeepsFailingIsAFailureToo(t *testing.T) { - w, said := standingWorld(t) - w.give(map[string]any{"as": "a"}) - w.h.Reconcile(ctx) - w.a.holdsErr = errors.New("401 invalid_grant") - w.passes(7 * time.Minute) - if events(*said) != EventFailing || (*said)[0].body["class"] != ClassCredentials { - t.Fatalf("%q %v", events(*said), *said) - } - w.a.holdsErr = nil - w.passes(time.Minute + 5*time.Second) - if events(*said) != EventFailing+","+EventRecovered { - t.Fatalf("%q", events(*said)) - } -} - -func TestAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) { - w, said := standingWorld(t) - w.give(map[string]any{"as": "a"}) - os.Remove(w.dir + "/a.secret") - w.passes(6 * time.Minute) - if events(*said) != EventFailing || (*said)[0].body["class"] != ClassSecret { - t.Fatalf("%q %v", events(*said), *said) - } -} - -func TestAConsumerNoLongerAskedForIsNoLongerFailing(t *testing.T) { - w, said := standingWorld(t) - w.a.failing = errors.New("boom") - w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}) - w.passes(6 * time.Minute) - if events(*said) != EventFailing+","+EventFailing { - t.Fatalf("%q", events(*said)) - } - w.give(map[string]any{"as": "a"}) - w.passes(5 * time.Second) - last := (*said)[len(*said)-1] - if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "no longer asked for" { - t.Fatalf("%v", *said) - } -} - -func TestAnErrorIsClassedByItsWords(t *testing.T) { - for text, want := range map[string]string{ - `Keycloak token request failed: 401 {"error":"invalid_grant"}`: ClassCredentials, - `FATAL: password authentication failed for user "postgres"`: ClassCredentials, - `dial tcp 127.0.0.1:5432: connect: connection refused`: ClassUnreachable, - `context deadline exceeded`: ClassUnreachable, - `extension "nope" is not available`: ClassRefused, - } { - if got := ClassOf(text); got != want { - t.Errorf("%s: %s, want %s", text, got, want) - } - } -} - -func TestAnAdapterThatClassesItsOwnErrorsIsBelieved(t *testing.T) { - w, said := standingWorld(t) - w.h.Adapter = classing{w.a} - w.a.failing = errors.New("anything") - w.give(map[string]any{"as": "a"}) - w.passes(6 * time.Minute) - if (*said)[0].body["class"] != "its-own" { - t.Fatal((*said)[0].body) - } -} - -type classing struct{ *recorder } - -func (classing) Class(error) string { return "its-own" } - -// A provider restarted after announcing a failure has forgotten it; its first success for each -// consumer is announced, so the controller clears what it kept rather than naming it for ever. -func TestTheFirstSuccessSinceStartIsAnnouncedOnce(t *testing.T) { - w := newWorld(t) - var said []announced - w.h.Announce = func(e string, b map[string]any) { - if e != EventRetirement { - said = append(said, announced{e, b}) - } - } - w.give(map[string]any{"as": "a"}) - w.passes(3 * time.Minute) - if events(said) != EventRecovered || said[0].body["why"] != "first-success" || said[0].body["consumer"] != "a" { - t.Fatalf("%v", said) - } -} From 302eab14311c324ad9bec616838ba57e278aa230 Mon Sep 17 00:00:00 2001 From: "fix-cat-pas@novox" Date: Fri, 16 Oct 2026 19:37:54 +0200 Subject: [PATCH 09/11] Restore the test that both keycloak paths leave the same client, over the SDK's moved path (issue 849) --- .../cmd/keycloak-provider/moved_test.go | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/modules/keycloak/cmd/keycloak-provider/moved_test.go b/modules/keycloak/cmd/keycloak-provider/moved_test.go index 7077324e..f69aa363 100644 --- a/modules/keycloak/cmd/keycloak-provider/moved_test.go +++ b/modules/keycloak/cmd/keycloak-provider/moved_test.go @@ -1,13 +1,17 @@ package main import ( + "context" + "encoding/json" "os" "path/filepath" "reflect" + "sync" "testing" "time" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provider/harness" ) const testIssuer = "https://id.example.org/realms/Novox" @@ -69,3 +73,121 @@ func TestTheGrantPathLeavesWhenTheConsumersFileIsFilled(t *testing.T) { t.Fatal("did not leave when the file was filled") } } + +// bucketBus is the helper's bus over in-memory buckets. +type bucketBus struct { + mu sync.Mutex + buckets map[string]map[string][]byte +} + +func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { + b.mu.Lock() + defer b.mu.Unlock() + for k, v := range b.buckets[bucket] { + handle(k, v) + } + return nil +} + +func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.buckets[bucket] == nil { + b.buckets[bucket] = map[string][]byte{} + } + b.buckets[bucket][key] = append([]byte(nil), value...) + return nil +} + +func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.buckets[bucket], key) + return nil +} + +// grafanaContributes and grafanaNames are what grafana's plan entry carries for keycloak: its contribution, and the +// names the controller composed for it (issues 776, 792). +var grafanaContributes = map[string]any{"label": "grafana", "endpoint": "web", "callback": "/login/generic_oauth"} +var grafanaNames = map[string]string{"name": "grafana.example.org", "internal-name": "grafana.home.internal"} + +// Against a Keycloak, both paths leave the same client: the grant file through the harness, and the consumers' file +// through the SDK's moved path with keycloak's spec (issue 849). +func TestBothPathsLeaveTheSameClientInKeycloak(t *testing.T) { + grafana := provider.ConsumerIdentity("home", "grafana") + values := map[string]any{"name": grafanaNames["name"], "internal-name": grafanaNames["internal-name"]} + for k, v := range grafanaContributes { + values[k] = v + } + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + made := func(through func(p provisioner)) Rep { + f, o := oidcWorld(t) + through(provisioner{clients: o, announce: func(string, map[string]any) {}, log: func(string, ...any) {}}) + return only(t, f, grafana) + } + viaGrant := made(func(p provisioner) { + dir := t.TempDir() + secret := filepath.Join(dir, grafana+".secret") + if err := os.WriteFile(secret, []byte("pw-"+grafana+"\n"), 0o600); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(map[string]any{"requirement": ProvisionName, "given": []map[string]any{ + {"as": grafana, "node": "home", "values": values, "secret": secret}}}) + receives := filepath.Join(dir, "mesh.json") + if err := os.WriteFile(receives, raw, 0o600); err != nil { + t.Fatal(err) + } + h := &harness.Harness{Resource: ProvisionName, Receives: receives, Adapter: p, + Now: func() time.Time { return now }, Log: func(string, ...any) {}} + h.Reconcile(ctx) + }) + viaFile := made(func(p provisioner) { + list, _ := json.Marshal([]provider.ConsumerCredential{{Consumer: "home", Module: "grafana", + Credential: provider.PairName("novox", ProvisionName, "home", "grafana", ""), Value: "pw-" + grafana}}) + file := filepath.Join(t.TempDir(), "consumers.json") + if err := os.WriteFile(file, list, 0o600); err != nil { + t.Fatal(err) + } + h := &harness.Harness{Resource: ProvisionName, Adapter: p, Node: "novox", + Now: func() time.Time { return now }, Log: func(string, ...any) {}} + run, _, _, err := moved(file, h, testIssuer) + if err != nil { + t.Fatal(err) + } + bus := &bucketBus{buckets: map[string]map[string][]byte{}} + plan, _ := json.Marshal(map[string]any{"node": "home", "module": "grafana", "provisioning": map[string]any{ + "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": "keycloak", "contributes": grafanaContributes, "names": grafanaNames}}}}) + _ = bus.Put(ctx, provider.AssignmentsBucket, "home.grafana", plan) + pr := provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: bus, + Vault: provider.ConsumersFile{Paths: []string{file}}, + Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer, + Serves: servedValues(testIssuer)}}, + Make: run.Make, Gone: run.Gone, Logf: func(string, ...any) {}}) + out, err := pr.Pass(ctx) + if err != nil || len(out) != 1 || out[0].Status != provider.Answered { + t.Fatalf("%+v %v", out, err) + } + }) + // Keycloak's own ids differ between two runs; nothing the mesh sets does. + for _, c := range []Rep{viaGrant, viaFile} { + delete(c, "id") + raw, _ := json.Marshal(c) + var again Rep + _ = json.Unmarshal(raw, &again) + for k := range c { + delete(c, k) + } + for k, v := range again { + c[k] = v + } + if mappers, ok := c["protocolMappers"].([]any); ok { + for _, m := range mappers { + delete(m.(map[string]any), "id") + } + } + } + if !reflect.DeepEqual(viaGrant, viaFile) { + t.Fatalf("through the file %v, through the grant %v", viaFile, viaGrant) + } +} From dcedd8d2b23081d02d308ece05df1cfe4f337e6d Mon Sep 17 00:00:00 2001 From: "fix-cat-pas@novox" Date: Fri, 16 Oct 2026 19:44:19 +0200 Subject: [PATCH 10/11] Give the route plan entries their credential-form, and read the entry's vault-name in the route providers' tests (issues 923, 945) --- .../cmd/mesh-provisioner-client/route_names_test.go | 4 ++-- modules/route-adapter/cmd/route-provider/provider_test.go | 2 +- modules/route-proxy/cmd/route-provider/provider_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go index 42b27ecf..76922ed1 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go @@ -96,12 +96,12 @@ func routeRun(t *testing.T, module, id, path, templatePath, entryKey, firstEntry } // gitea's plan entry for its `web` route, as the controller writes it for a node whose public domain is example.org. -const giteaRoutePlanEntry = `{"provision":"route","provider":"novox","provider-module":"route-proxy",` + +const giteaRoutePlanEntry = `{"provision":"route","provider":"novox","provider-module":"route-proxy","credential-form":"pair",` + `"at":"novox.internal","from":"novox","as":"mesh_g14_gitea",` + `"names":{"name-web":"git.example.org","internal-name-web":"git.novox.internal",` + `"name-internal-api-refused":"git.example.org","internal-name-internal-api-refused":"git.novox.internal"}}` -const keycloakRoutePlanEntry = `{"provision":"route","provider":"novox","provider-module":"route-proxy",` + +const keycloakRoutePlanEntry = `{"provision":"route","provider":"novox","provider-module":"route-proxy","credential-form":"pair",` + `"at":"novox.internal","from":"novox","as":"mesh_g14_keycloak",` + `"names":{"name":"keycloak.example.org","internal-name":"keycloak.novox.internal"}}` diff --git a/modules/route-adapter/cmd/route-provider/provider_test.go b/modules/route-adapter/cmd/route-provider/provider_test.go index 10c30933..abb464e1 100644 --- a/modules/route-adapter/cmd/route-provider/provider_test.go +++ b/modules/route-adapter/cmd/route-provider/provider_test.go @@ -66,7 +66,7 @@ func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { t.Fatal(err) } if e.Values["name-web"] != "git.example.org" || e.Values["name"] != "git.example.org" || - e.Values["internal-name"] != "git.novox.internal" || e.Credential != "" || e.User != "" || e.Refused != "" { + e.Values["internal-name"] != "git.novox.internal" || e.VaultName != "" || e.User != "" || e.Refused != "" { t.Fatalf("the entry is %+v", e) } } diff --git a/modules/route-proxy/cmd/route-provider/provider_test.go b/modules/route-proxy/cmd/route-provider/provider_test.go index 6c580b97..d7ccaffc 100644 --- a/modules/route-proxy/cmd/route-provider/provider_test.go +++ b/modules/route-proxy/cmd/route-provider/provider_test.go @@ -69,7 +69,7 @@ func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { t.Fatal(err) } if e.Values["name-web"] != "git.example.org" || e.Values["name"] != "git.example.org" || - e.Values["internal-name"] != "git.novox.internal" || e.Credential != "" || e.User != "" || e.Refused != "" { + e.Values["internal-name"] != "git.novox.internal" || e.VaultName != "" || e.User != "" || e.Refused != "" { t.Fatalf("the entry is %+v", e) } } From d2992f81d17f290b6df5c1bec5c107f6df041890 Mon Sep 17 00:00:00 2001 From: lead-57 Date: Fri, 16 Oct 2026 20:59:14 +0200 Subject: [PATCH 11/11] Take mesh-sdk a4e7f48 in the modules that use its provider helper (issues 825, 849, 945) postgres, keycloak, mesh-provisioner-client, route-proxy and route-adapter now build against the feature's mesh-sdk: the harness package, the vault-name key and the helper's fixes. --- modules/keycloak/go.mod | 2 +- modules/keycloak/go.sum | 4 ++-- modules/mesh-provisioner-client/go.mod | 2 +- modules/mesh-provisioner-client/go.sum | 4 ++-- modules/postgres/go.mod | 2 +- modules/postgres/go.sum | 4 ++-- modules/route-adapter/go.mod | 2 +- modules/route-adapter/go.sum | 4 ++-- modules/route-proxy/go.mod | 2 +- modules/route-proxy/go.sum | 4 ++-- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/keycloak/go.mod b/modules/keycloak/go.mod index f8dcc045..893fc271 100644 --- a/modules/keycloak/go.mod +++ b/modules/keycloak/go.mod @@ -2,4 +2,4 @@ module keycloak go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 diff --git a/modules/keycloak/go.sum b/modules/keycloak/go.sum index 9a531b0e..63b4da41 100644 --- a/modules/keycloak/go.sum +++ b/modules/keycloak/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/mesh-provisioner-client/go.mod b/modules/mesh-provisioner-client/go.mod index 39b43e4a..aaf68d4c 100644 --- a/modules/mesh-provisioner-client/go.mod +++ b/modules/mesh-provisioner-client/go.mod @@ -3,7 +3,7 @@ module mesh-provisioner-client go 1.25.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.51.0 ) diff --git a/modules/mesh-provisioner-client/go.sum b/modules/mesh-provisioner-client/go.sum index da3d1f5d..bf0db5c6 100644 --- a/modules/mesh-provisioner-client/go.sum +++ b/modules/mesh-provisioner-client/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 h1:I9rm8HhYRJ0+mzQQz+VAH0kgv2YJWkvctEc67Yd0PX8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/postgres/go.mod b/modules/postgres/go.mod index 820d0e67..1b761fe0 100644 --- a/modules/postgres/go.mod +++ b/modules/postgres/go.mod @@ -3,7 +3,7 @@ module postgres go 1.25.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 github.com/jackc/pgx/v5 v5.11.0 ) diff --git a/modules/postgres/go.sum b/modules/postgres/go.sum index 929f7d28..1764370b 100644 --- a/modules/postgres/go.sum +++ b/modules/postgres/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/route-adapter/go.mod b/modules/route-adapter/go.mod index 792f6ec3..961c03e3 100644 --- a/modules/route-adapter/go.mod +++ b/modules/route-adapter/go.mod @@ -2,4 +2,4 @@ module route-adapter go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 diff --git a/modules/route-adapter/go.sum b/modules/route-adapter/go.sum index 9a531b0e..63b4da41 100644 --- a/modules/route-adapter/go.sum +++ b/modules/route-adapter/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/route-proxy/go.mod b/modules/route-proxy/go.mod index ade1dc0a..54bca71c 100644 --- a/modules/route-proxy/go.mod +++ b/modules/route-proxy/go.mod @@ -2,4 +2,4 @@ module route-proxy go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 diff --git a/modules/route-proxy/go.sum b/modules/route-proxy/go.sum index 9a531b0e..63b4da41 100644 --- a/modules/route-proxy/go.sum +++ b/modules/route-proxy/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=