From a3106a4b642b7e2f9e8c4fd37364b1e82fd31418 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 14:09:06 +0100 Subject: [PATCH] Provisioner client: the key and plan layouts, the provider and its harness, moved from mesh-sdk; the provider's 31 tests are 6 tables, the plan stays typed in the client (ADR 0634, ADR 0635; issue 1966) --- .../clients/meshprovisionerclient/client.go | 310 ++++++ .../meshprovisionerclient/client_test.go | 121 +++ .../go/clients/meshprovisionerclient/go.mod | 5 + .../provider/consumers.go | 72 ++ .../provider/harness/ctx_test.go | 5 + .../provider/harness/early_test.go | 22 + .../provider/harness/grants_moved_test.go | 124 +++ .../provider/harness/harness.go | 572 +++++++++++ .../provider/harness/harness_test.go | 154 +++ .../provider/harness/moved.go | 312 ++++++ .../provider/harness/moved_test.go | 464 +++++++++ .../provider/harness/retirement.go | 626 ++++++++++++ .../provider/harness/retirement_test.go | 575 +++++++++++ .../provider/harness/standing_test.go | 193 ++++ .../provider/harness/tables_test.go | 114 +++ .../provider/provider.go | 858 +++++++++++++++++ .../provider/provider_test.go | 294 ++++++ .../provider/statebus_test.go | 54 ++ .../provider/tables_test.go | 909 ++++++++++++++++++ 19 files changed, 5784 insertions(+) create mode 100644 provisioning/go/clients/meshprovisionerclient/client.go create mode 100644 provisioning/go/clients/meshprovisionerclient/client_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/go.mod create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/consumers.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/ctx_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/early_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/grants_moved_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/harness.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/harness_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/moved.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/moved_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/retirement.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/retirement_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/standing_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/harness/tables_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/provider.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/provider_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/statebus_test.go create mode 100644 provisioning/go/clients/meshprovisionerclient/provider/tables_test.go diff --git a/provisioning/go/clients/meshprovisionerclient/client.go b/provisioning/go/clients/meshprovisionerclient/client.go new file mode 100644 index 0000000..32a0ed6 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/client.go @@ -0,0 +1,310 @@ +// Package meshprovisionerclient is the contract of the seat mesh-provisioner's records (novox/hq ADR 0570 clauses 1, 2 and 5; issue +// 1954): every provision one key of the state `provisions`, holding a reference and never a value, and every handed +// module's provisioning plan one key of the state `plans`. The module mesh-provisioner alone writes both; the vault, +// each node's provisioner client and every provider read them through `reads`. +package meshprovisionerclient + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The seat, its holder and its two states, and the names a reader declares in its `reads`. +const ( + Seat = "mesh-provisioner" + Module = "mesh-provisioner" + ProvisionsState = "provisions" + PlansState = "plans" + ProvisionsRead = Module + "." + ProvisionsState + PlansRead = Module + "." + PlansState + // ProvisionsBucket and PlansBucket are the states' buckets. + ProvisionsBucket = Module + "_" + ProvisionsState + PlansBucket = Module + "_" + PlansState +) + +// The states of a provision (clause 1). +const ( + // Asked is a provision its consumer needs that its provider has not answered yet, or whose dependencies are not + // all provided (clause 3). + Asked = "asked" + // Provided is a provision the provider said it made: set only from the provider's outcome (clause 3). + Provided = "provided" + // Waits is a provision only a person can give, naming what is awaited (clause 7). + Waits = "waits" + // Failed is a provision the provider could not make, with why. + Failed = "failed" + // Retiring is a provision no want needs, asked of the operator before its credential goes (clause 5). + Retiring = "retiring" +) + +// States is every state a provision may be in. +var States = []string{Asked, Provided, Waits, Failed, Retiring} + +// Version is the version of the records' layout. +const Version = 1 + +// token is one part of a key: what a bucket's key takes, no dot. +var token = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// Key is a provision's key: `..`, and `.` where the consumer keeps several values of +// one provision (ADR 0094). The same key the provider's own entry for that consumer has (provider.Entry.Key). +func Key(node, consumer, provision, local string) (string, error) { + parts := []string{node, consumer, provision} + if local != "" { + parts = append(parts, local) + } + for _, p := range parts { + if !token.MatchString(p) { + return "", fmt.Errorf("provisions: %q cannot be part of a provision's key", p) + } + } + return strings.Join(parts, "."), nil +} + +// ParseKey is the inverse of Key. +func ParseKey(key string) (node, consumer, provision, local string, err error) { + parts := strings.Split(key, ".") + if len(parts) < 3 || len(parts) > 4 { + return "", "", "", "", fmt.Errorf("provisions: %q is not ..[.]", key) + } + for _, p := range parts { + if !token.MatchString(p) { + return "", "", "", "", fmt.Errorf("provisions: %q is not ..[.]", key) + } + } + if len(parts) == 4 { + local = parts[3] + } + return parts[0], parts[1], parts[2], local, nil +} + +// PlanKey is a plan's key in `plans`: `.` (clause 2). +func PlanKey(node, module string) (string, error) { + if !token.MatchString(node) || !token.MatchString(module) { + return "", fmt.Errorf("provisions: %s.%s is not a plan's key, .", node, module) + } + return node + "." + module, nil +} + +// The forms of a provision's credential, as a plan's entry names them (catalogue.PlanProvision.CredentialForm): the +// catalogue's one list (catalogue.CredentialForms, issue 774), under the names this package has used. +const ( + FormPair = catalogue.CredentialFormPair + FormShared = catalogue.CredentialFormShared + FormIssued = catalogue.CredentialFormIssued + FormGiven = catalogue.CredentialFormGiven + // FormNone is an offer whose consumers present no credential (catalogue.CredentialFormNone): the vault mints nothing, + // and the provision depends on no credential. + FormNone = catalogue.CredentialFormNone +) + +// VaultName is the vault's name of a provision's credential in form, as its verb `own` names it and the provider asks +// it: `pair////[/]` for a login of the consumer's own (ADR 0158 +// §11), `provision//` for the provider's one shared value, and `provision////` for a holder's own issued or given value. Empty for a form with no +// credential. +func VaultName(form string, p Provision) string { + switch form { + case FormPair: + n := "pair/" + p.Provider.Node + "/" + p.Provision + "/" + p.Consumer.Node + "/" + p.Consumer.Module + if p.Local != "" { + n += "/" + p.Local + } + return n + case FormShared: + return "provision/" + p.Provider.Node + "/" + p.Provision + case FormIssued, FormGiven: + return "provision/" + p.Provider.Node + "/" + p.Provision + "/" + p.Consumer.Node + "/" + p.Consumer.Module + } + return "" +} + +// VaultModule is the provider of every provision's credential (novox/hq ADR 0570 clauses 1, 6). +const VaultModule = "mesh-vault" + +// CredentialKey is the key of a provision's credential, itself a provision whose provider is the vault (clause 6; the +// key form the lead settled on 2026-10-29): `..secret-[.]` for a value of the +// consumer's own (pair, issued, given), and `..secret-` for the provider's one +// shared value. False for a form with no vault value, and for a provision the vault itself provides. +func CredentialKey(form string, consumer, provider End, provision, local string) (string, bool) { + if provider.Module == VaultModule { + return "", false // the vault's own offer is the value: it has no credential beside it (the lead, 2026-10-29) + } + name := catalogue.CredentialProvisionPrefix + provision + switch form { + case FormPair, FormIssued, FormGiven: + k, err := Key(consumer.Node, consumer.Module, name, local) + return k, err == nil + case FormShared: + k, err := Key(provider.Node, provider.Module, name, "") + return k, err == nil + } + return "", false +} + +// DependsOn is what a provision waits for before its provider makes it (clause 3): its credential's key, where its form +// has a vault value, and the key of every provision its provider module itself requires on its own node, read from +// the provider's plan there (nil when the provider is not handed and has none). Sorted, none twice. +func DependsOn(consumer End, entry catalogue.PlanProvision, providerPlan *catalogue.Plan) []string { + seen := map[string]bool{} + var out []string + add := func(k string) { + if k != "" && !seen[k] { + seen[k] = true + out = append(out, k) + } + } + provider := End{Module: entry.ProviderModule, Node: entry.Provider} + if k, ok := CredentialKey(entry.CredentialForm, consumer, provider, entry.Provision, entry.Local); ok { + add(k) + } + if providerPlan != nil { + for _, own := range providerPlan.Provisions { + if k, err := Key(provider.Node, provider.Module, own.Provision, own.Local); err == nil { + add(k) + } + } + } + sort.Strings(out) + return out +} + +// End is one end of a provision: a module on a node. +type End struct { + Module string `json:"module"` + Node string `json:"node"` +} + +// Reference is a value the vault keeps, by its id and the generation of the value: never the value, sealed or not +// (ADR 0349 §7). The holder writes the generation the vault says in its own entry (clause 6). ID is the vault's name of +// the credential exactly as its verb `own` names it (VaultName): the consuming node's engine checks both the name and +// the generation of what it is handed. +type Reference struct { + ID string `json:"id"` + Generation int `json:"generation"` +} + +// Warrant is the operator's answer to a retiring provision's ask, as the holder writes it into the key (clause 5). +type Warrant struct { + Ask string `json:"ask"` + Option string `json:"option"` + By string `json:"by"` + At time.Time `json:"at"` +} + +// Provision is one key of `provisions`. +type Provision struct { + V int `json:"v"` + Provision string `json:"provision"` + Local string `json:"local,omitempty"` + Consumer End `json:"consumer"` + Provider End `json:"provider"` + // DependsOn are the keys of the provisions this one waits for (clause 3): a database user whose password is the + // vault's. + DependsOn []string `json:"depends-on,omitempty"` + State string `json:"state"` + // Why is the reason for waits, failed and retiring, and for asked when a dependency holds it. + Why string `json:"why,omitempty"` + Reference *Reference `json:"reference,omitempty"` + // Ask is a retiring provision's ask of the operator; Warrant the answer, once given. + Ask string `json:"ask,omitempty"` + Warrant *Warrant `json:"warrant,omitempty"` + // Plan is the plan entry the key was composed from, at the consumer's wanted build or its newest (issue 1126): the + // provider binds the consumer from it, so a requirement the newest build adds is provided before the want that + // waits for it moves, though the plan the seat keeps at the wanted build names it not yet. + Plan *catalogue.PlanProvision `json:"plan,omitempty"` + At time.Time `json:"at"` +} + +// Key is the provision's key. +func (p Provision) Key() (string, error) { + return Key(p.Consumer.Node, p.Consumer.Module, p.Provision, p.Local) +} + +func (p Provision) check() error { + if _, err := p.Key(); err != nil { + return err + } + if p.Provider.Module == "" || p.Provider.Node == "" { + return fmt.Errorf("provisions: %s names no provider", p.Provision) + } + known := false + for _, s := range States { + known = known || s == p.State + } + switch { + case !known: + return fmt.Errorf("provisions: a provision's state is one of %s, not %q", strings.Join(States, ", "), p.State) + case (p.State == Waits || p.State == Failed || p.State == Retiring) && strings.TrimSpace(p.Why) == "": + return fmt.Errorf("provisions: a %s provision says why", p.State) + case p.Warrant != nil && p.State != Retiring: + return fmt.Errorf("provisions: only a retiring provision carries the operator's warrant") + } + for _, d := range p.DependsOn { + if _, _, _, _, err := ParseKey(d); err != nil { + return err + } + } + return nil +} + +// Encode is p as JSON at Version; it refuses what a holder may not write. +func Encode(p Provision) ([]byte, error) { + p.V = Version + if err := p.check(); err != nil { + return nil, err + } + return json.Marshal(p) +} + +// Decode reads a key's value. **A field the layout does not name is read past and dropped** (issue 1126 review 1): a +// reader on an older mesh-sdk must not refuse a key a newer writer adds a field to. No value rides in a provision all +// the same (clause 1, ADR 0201 §13): Encode writes the layout alone, so what a reader does not know is never kept. +func Decode(data []byte) (Provision, error) { + var p Provision + dec := json.NewDecoder(bytes.NewReader(data)) + if err := dec.Decode(&p); err != nil { + return Provision{}, fmt.Errorf("provisions: a provision is not readable: %w", err) + } + if p.V != Version { + return Provision{}, fmt.Errorf("provisions: a provision of version %d, and this reads %d", p.V, Version) + } + return p, p.check() +} + +// Plan is one key of `plans`: a module's provisioning plan on a node, never a value (clause 2). +type Plan struct { + V int `json:"v"` + Node string `json:"node"` + Module string `json:"module"` + Plan catalogue.Plan `json:"plan"` +} + +// EncodePlan is p as JSON at Version. +func EncodePlan(p Plan) ([]byte, error) { + p.V = Version + if _, err := PlanKey(p.Node, p.Module); err != nil { + return nil, err + } + return json.Marshal(p) +} + +// DecodePlan reads a plan's key. +func DecodePlan(data []byte) (Plan, error) { + var p Plan + if err := json.Unmarshal(data, &p); err != nil { + return Plan{}, fmt.Errorf("provisions: a plan is not readable: %w", err) + } + if p.V != Version { + return Plan{}, fmt.Errorf("provisions: a plan of version %d, and this reads %d", p.V, Version) + } + _, err := PlanKey(p.Node, p.Module) + return p, err +} diff --git a/provisioning/go/clients/meshprovisionerclient/client_test.go b/provisioning/go/clients/meshprovisionerclient/client_test.go new file mode 100644 index 0000000..8461091 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/client_test.go @@ -0,0 +1,121 @@ +package meshprovisionerclient + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// ADR 0570 clauses 1, 2 and 5 (issue 1954 L6-P2): a provision's key is its consumer's node and module, the provision and +// its local name, the provider's entry's key; its value round-trips, names its state and why, and refuses any field the +// layout does not name, so no value rides in it; a plan's key is its node and module. +func TestAProvisionIsAReferenceUnderItsConsumersKey(t *testing.T) { + at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + db := Provision{Provision: "postgres-database", Consumer: End{Module: "keycloak", Node: "novox"}, + Provider: End{Module: "postgres", Node: "novox"}, DependsOn: []string{"novox.keycloak.secret.db"}, State: Asked, + Reference: &Reference{ID: "provision/postgres/postgres-database", Generation: 2}, At: at} + for _, c := range []struct { + name string + edit func(*Provision) + key string + refused string + }{ + {name: "asked, depending on the vault's", key: "novox.keycloak.postgres-database"}, + {name: "under a local name", edit: func(p *Provision) { p.Local = "replica" }, key: "novox.keycloak.postgres-database.replica"}, + {name: "retiring with the operator's warrant", edit: func(p *Provision) { + p.State, p.Why, p.Ask = Retiring, "keycloak left novox", "a-1" + p.Warrant = &Warrant{Ask: "a-1", Option: "yes", By: "the operator", At: at} + }, key: "novox.keycloak.postgres-database"}, + {name: "an unknown state", edit: func(p *Provision) { p.State = "done" }, refused: "one of"}, + {name: "failed without why", edit: func(p *Provision) { p.State = Failed }, refused: "says why"}, + {name: "a warrant on a provided one", edit: func(p *Provision) { p.State, p.Warrant = Provided, &Warrant{Ask: "a"} }, refused: "only a retiring"}, + {name: "no provider", edit: func(p *Provision) { p.Provider = End{} }, refused: "no provider"}, + {name: "a dotted consumer", edit: func(p *Provision) { p.Consumer.Module = "a.b" }, refused: "cannot be part"}, + {name: "a dependency that is no key", edit: func(p *Provision) { p.DependsOn = []string{"x"} }, refused: "is not"}, + } { + p := db + if c.edit != nil { + c.edit(&p) + } + raw, err := Encode(p) + if c.refused != "" { + if err == nil || !strings.Contains(err.Error(), c.refused) { + t.Errorf("%s: %v, want refused saying %q", c.name, err, c.refused) + } + continue + } + back, err := Decode(raw) + key, kerr := back.Key() + if err != nil || kerr != nil || key != c.key || back.V != Version || back.Reference.Generation != 2 { + t.Errorf("%s: read back as %+v under %q, %v %v", c.name, back, key, err, kerr) + } + if node, consumer, provision, local, err := ParseKey(key); err != nil || node != "novox" || consumer != "keycloak" || + provision != "postgres-database" || local != p.Local { + t.Errorf("%s: %s parses as %s %s %s %s, %v", c.name, key, node, consumer, provision, local, err) + } + } + // A field a later layout adds is read past, never kept (issue 1126 review 1: a reader on an older mesh-sdk must not + // refuse a key a newer writer adds a field to). A value still never rides in a provision: Encode writes the layout + // alone, so what a reader does not know is dropped, and encoding it again carries none of it. + later, err := Decode([]byte(`{"v":1,"provision":"p","consumer":{"module":"c","node":"n"},"provider":{"module":"p","node":"n"},` + + `"state":"provided","at":"2026-10-29T22:00:00Z","a-later-field":{"x":1},"password":"hunter2"}`)) + if err != nil || later.Provision != "p" || later.State != Provided { + t.Errorf("a provision with a field of a later layout: %+v, %v", later, err) + } + if again, _ := Encode(later); strings.Contains(string(again), "hunter2") || strings.Contains(string(again), "a-later-field") { + t.Errorf("a field the layout does not name was kept: %s", again) + } + plan := Plan{Node: "novox", Module: "restic", Plan: catalogue.Plan{}} + raw, err := EncodePlan(plan) + if err != nil { + t.Fatal(err) + } + if back, err := DecodePlan(raw); err != nil || back.Node != "novox" || back.Module != "restic" { + t.Errorf("a plan read back as %+v, %v", back, err) + } + if k, err := PlanKey("novox", "restic"); k != "novox.restic" || err != nil { + t.Errorf("a plan's key is %q, %v", k, err) + } + // A reference's id is the vault's name as its verb own names it, in each form (the host agent, 2026-10-29). + for form, want := range map[string]string{FormPair: "pair/novox/postgres-database/novox/keycloak", FormShared: "provision/novox/postgres-database", + FormIssued: "provision/novox/postgres-database/novox/keycloak", FormNone: ""} { + if got := VaultName(form, db); got != want { + t.Errorf("the %s name is %q", form, got) + } + } + if got := VaultName(FormPair, Provision{Provision: "p", Local: "r", Provider: End{Node: "a"}, Consumer: End{Node: "b", Module: "m"}}); got != "pair/a/p/b/m/r" { + t.Errorf("a local name's pair is %q", got) + } + // A provision waits for its credential, the vault's provision `secret-`, and for what its provider itself + // requires on its node (clause 3; the key form the lead settled on 2026-10-29). + keycloak, postgres := End{Module: "keycloak", Node: "novox"}, End{Module: "postgres", Node: "ace"} + providerPlan := &catalogue.Plan{Provisions: []catalogue.PlanProvision{{Provision: "backup-target", Provider: "ace", ProviderModule: "restic"}}} + for _, c := range []struct { + form string + plan *catalogue.Plan + want string + }{ + {FormPair, nil, "novox.keycloak.secret-postgres-database"}, + {FormShared, nil, "ace.postgres.secret-postgres-database"}, + {FormPair, providerPlan, "ace.postgres.backup-target,novox.keycloak.secret-postgres-database"}, + {"", nil, ""}, + // An offer with no credential (the lead, 2026-10-29): nothing minted, nothing to wait for. + {FormNone, nil, ""}, + } { + entry := catalogue.PlanProvision{Provision: "postgres-database", Provider: postgres.Node, ProviderModule: postgres.Module, + CredentialForm: c.form} + if got := strings.Join(DependsOn(keycloak, entry, c.plan), ","); got != c.want { + t.Errorf("%q depends on %q, want %q", c.form, got, c.want) + } + } + // The vault's own offer is the value: no secret-secret beside it. + vaultEntry := catalogue.PlanProvision{Provision: "secret", Provider: "novox", ProviderModule: VaultModule, CredentialForm: FormPair} + if got := DependsOn(keycloak, vaultEntry, nil); len(got) != 0 { + t.Errorf("the vault's secret depends on %v", got) + } + if ProvisionsBucket != "mesh-provisioner_provisions" || PlansRead != "mesh-provisioner.plans" { + t.Errorf("the states are %s and %s", ProvisionsBucket, PlansRead) + } +} diff --git a/provisioning/go/clients/meshprovisionerclient/go.mod b/provisioning/go/clients/meshprovisionerclient/go.mod new file mode 100644 index 0000000..b6fdcd2 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/go.mod @@ -0,0 +1,5 @@ +module git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104174118-f94abb95aaef diff --git a/provisioning/go/clients/meshprovisionerclient/provider/consumers.go b/provisioning/go/clients/meshprovisionerclient/provider/consumers.go new file mode 100644 index 0000000..326d0a9 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/consumers.go @@ -0,0 +1,72 @@ +package provider + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" +) + +// The provider's consumers' credentials, from the file its machine's provisioner client fills (novox/hq ADR 0373, +// issue 679). +// +// **The provider never asks the vault.** A provider in the tool runner has no account of its own, and the vault +// hands a secret only to a node's provisioner client or to a module under an account of its own (ADR 0349 clause +// 11, ADR 0370 clause 2). So the provider's template names `${consumers:}`, and its machine's client +// fills that file with one entry per consumer bound to it for that provision, each credential asked of the vault in +// the client's own name; the file is the provider's alone (mode 0600, owned). ConsumersFile is that file as the +// helper's Vault: it reads the file on each ask, so a consumer that binds or a credential the vault remakes reaches +// the provider at its next pass. + +// ConsumerCredential is one entry of the file: the consumer, the vault's name of its credential, and its value. +type ConsumerCredential struct { + // Consumer and Module are the consumer's machine and module, Local its local name for the provision, if any. + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + // Credential is the vault's name of the consumer's credential, PairName's. + Credential string `json:"credential"` + // Value is the credential itself. + Value string `json:"value"` +} + +// ConsumersFile answers a credential from the files the provider machine's client fills: one per provision whose +// template names `${consumers:}`, each a JSON list of ConsumerCredential. +type ConsumersFile struct { + Paths []string +} + +// Value is the credential named name, from the first file carrying it. A file not there yet is passed over; one not +// in the client's form is an error, never read as no consumers. +func (f ConsumersFile) Value(_ context.Context, name string) ([]byte, error) { + var missing []string + for _, at := range f.Paths { + raw, err := os.ReadFile(at) + if errors.Is(err, fs.ErrNotExist) { + missing = append(missing, at) + continue + } + if err != nil { + return nil, fmt.Errorf("the consumers' file %s: %w", at, err) + } + var entries []ConsumerCredential + if err := json.Unmarshal(raw, &entries); err != nil { + return nil, fmt.Errorf("the consumers' file %s is not the provisioner client's list of consumers", at) + } + for _, e := range entries { + if e.Credential == name { + if e.Value == "" { + return nil, fmt.Errorf("the consumers' file %s carries %s with no value", at, name) + } + return []byte(e.Value), nil + } + } + } + if len(missing) == len(f.Paths) { + return nil, fmt.Errorf("%s: no consumers' file is filled yet (%v)", name, f.Paths) + } + return nil, fmt.Errorf("%s is in no consumers' file: the vault holds none for it yet, or its machine's client "+ + "has not filled it", name) +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/ctx_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/ctx_test.go new file mode 100644 index 0000000..736ef25 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/ctx_test.go @@ -0,0 +1,5 @@ +package harness + +import "context" + +var ctx = context.Background() diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/early_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/early_test.go new file mode 100644 index 0000000..4598603 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/early_test.go @@ -0,0 +1,22 @@ +package harness + +// The moved path started before its runtime answers asks again until it does (issue 943). + +import ( + "context" + "errors" +) + +// earlyBus refuses its first watches as the runtime does before Serve runs, then answers as the bucket bus does. +type earlyBus struct { + *bucketBus + refuse int +} + +func (b *earlyBus) Watch(ctx context.Context, bucket string, handle func(string, []byte)) error { + if b.refuse > 0 { + b.refuse-- + return errors.New("before Serve: a bundle reaches the bus through the runtime that launched it") + } + return b.bucketBus.Watch(ctx, bucket, handle) +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/grants_moved_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/grants_moved_test.go new file mode 100644 index 0000000..d8206dd --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/grants_moved_test.go @@ -0,0 +1,124 @@ +package harness + +// postgres handed to the provisioner still serves the consumers the controller's grant file gives (issue 937): a +// consumer not moved has no plan naming postgres, only its grant, until R1 (issue 682). + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +// granting gives the moved world a grant file, as the controller writes it for the consumers not moved. +func (w *movedWorld) granting(given ...map[string]any) { + if w.h.Receives == "" { + w.h.Receives = filepath.Join(w.t.TempDir(), "mesh.json") + } + dir := filepath.Dir(w.h.Receives) + for _, g := range given { + secret := filepath.Join(dir, g["as"].(string)+".secret") + if err := os.WriteFile(secret, []byte("pw-"+g["as"].(string)+"\n"), 0o600); err != nil { + w.t.Fatal(err) + } + g["secret"] = secret + } + if given == nil { + given = []map[string]any{} + } + raw, _ := json.Marshal(map[string]any{"requirement": ProvisionName, "given": given}) + if err := os.WriteFile(w.h.Receives, raw, 0o600); err != nil { + w.t.Fatal(err) + } +} + +// settled is enough passes and time for a retirement to be stable. +func (w *movedWorld) settled() { + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(15 * time.Minute) + w.pass() + w.pass() +} + +// The logins the grant file gives are asked for: made as the grant path makes them, never offered for retirement, +// beside a moved consumer the helper makes from its plan. +func testAConsumerOnlyTheGrantFileGivesIsAskedForAndMade(t *testing.T) { + bound, logins := bindings(1) + w := newMovedWorld(t, bound) + w.a.inv.Active = []string{"mesh_ace_baserow", "mesh_ace_letta", logins[0]} + w.bus.bind("ace", "app1", nil) + w.granting( + map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}}, + map[string]any{"as": "mesh_ace_letta", "node": "ace", + "values": map[string]any{"name": "letta", "extensions": []any{"vector"}}}, + ) + w.settled() + if len(w.a.removed) != 0 || w.h.r.waiting != nil { + t.Fatalf("retired %v, waiting %+v: the grant file's logins were counted unasked", w.a.removed, w.h.r.waiting) + } + made := map[string]Provision{} + for _, c := range w.a.created { + made[c.As] = c + } + letta := made["mesh_ace_letta"] + if letta.Password != "pw-mesh_ace_letta" || letta.Consumer != "ace" || + !reflect.DeepEqual(letta.Values, map[string]any{"name": "letta", "extensions": []any{"vector"}}) { + t.Fatalf("made %+v for letta, as its grant gives it", letta) + } + if _, ok := made["mesh_ace_baserow"]; !ok { + t.Fatalf("baserow not made: %+v", w.a.created) + } + if _, ok := made[logins[0]]; !ok { + t.Fatalf("the moved consumer not made: %+v", w.a.created) + } +} + +// A login both a plan and the grant file name is the plan's: made once, with the plan's credential. +func testALoginAPlanNamesIsNotMadeFromTheGrantFile(t *testing.T) { + bound, logins := bindings(1) + w := newMovedWorld(t, bound) + w.bus.bind("ace", "app1", nil) + w.granting(map[string]any{"as": logins[0], "node": "ace", "values": map[string]any{"name": "app1"}}) + w.pass() + w.pass() + for _, c := range w.a.created { + if c.As == logins[0] && c.Password != "pw-app1" { + t.Fatalf("made %s with the grant's password %q, not the plan's", c.As, c.Password) + } + } +} + +// An unreadable grant file is not nobody asking (issue 241): nothing is retired while it cannot be read. +func testAnUnreadableGrantFileRetiresNothing(t *testing.T) { + bound, logins := bindings(1) + w := newMovedWorld(t, bound) + w.a.inv.Active = []string{"mesh_ace_baserow", logins[0]} + w.bus.bind("ace", "app1", nil) + w.h.Receives = filepath.Join(t.TempDir(), "absent.json") + w.settled() + if len(w.a.removed) != 0 || w.h.r.waiting != nil { + t.Fatalf("retired %v, waiting %+v with the grant file unreadable", w.a.removed, w.h.r.waiting) + } +} + +// A login the grant file stops giving is not retired by the provider itself (novox/hq ADR 0570 clause 5): every +// retirement waits for the operator's yes on a provision's key. +func testALoginTheGrantFileStopsGivingIsRetired(t *testing.T) { + bound, _ := bindings(1) + w := newMovedWorld(t, bound) + w.bus.bind("ace", "app1", nil) + w.granting( + map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}}, + map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}}, + ) + w.pass() + w.granting(map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}}) + w.settled() + if len(w.a.removed) != 0 || len(w.a.deleted) != 0 { + t.Fatalf("retired %v, deleted %v, by itself", w.a.removed, w.a.deleted) + } +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/harness.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/harness.go new file mode 100644 index 0000000..9e9dedc --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/harness.go @@ -0,0 +1,572 @@ +// Package harness is the reconcile loop and the retirement of a provider that keeps a resource per consumer; the +// module supplies an Adapter. +package harness + +// The reconcile loop every provider shares, as the TypeScript SDK's runProvisioner runs it +// (@novox/mesh-sdk/provisioner, 0.1.12), line for line in behaviour. It moved here from the postgres and keycloak +// providers, which each carried an identical copy (novox/hq ADR 0039: the loop is the SDK's, the adapter is the +// module's; issue 849). +// +// Read the contributions the mesh delivered; bring each consumer's resource into being through the +// adapter, under the login and password the mesh minted; retire what the mesh no longer asks for. +// **A provider creates the credential the mesh minted, and seals nothing (novox/hq ADR 0048).** +// +// **A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224).** A consumer +// whose create, check or secret has failed without one success in between for FailingAfter is +// announced as `provisioner.failing` — naming the consumer, its machine and the class of error — and +// again every SayAgainEvery while it lasts; the first success after that is `provisioner.recovered`. +// The controller keeps the newest per provider and consumer and `status` names it. On 2026-10-05 the +// identity provider failed every consumer 31,000 times in a day and said so only in its journal +// (novox/hq issue 179). +// +// **A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person +// (novox/hq ADR 0230, the operator's model of 2026-10-06).** Retiring disables its access — reversibly — +// and marks its login and data "to delete" with when and why; nothing is deleted. Asked for again, the +// ordinary create re-enables it as it was. It is retired only once the same set has gone unasked in +// StablePasses consecutive passes and for StableFor, and a set larger than the bound waits for a +// person. retirement.go. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "strings" + "sync" + "time" +) + +// Provision is one consumer's resource to bring into being — everything the mesh derived and delivered. +type Provision struct { + // As is the login the mesh derived and gave the consumer to present. + As string + // Password is the one the mesh minted, read from the file the host unsealed. + Password string + // Values are what the consumer contributed (e.g. {"name": "letta", "extensions": ["vector"]}). + Values map[string]any + // Derived is what this provider's own definition derives for the consumer (novox/hq ADR 0201). + Derived map[string]any + // At is where the consumer is; Consumer is its node. + At string + Consumer string +} + +// Adapter is the per-service half. +type Adapter interface { + // Create brings the consumer into being under the mesh's login and password — and, for one that + // was retired, re-enables it as it was and clears its mark to delete. + Create(ctx context.Context, p Provision) error + // Retire disables the consumer's access, reversibly, and marks its login and data to delete with + // when and why. It deletes nothing (novox/hq ADR 0230). derived is what the mesh last derived, + // remembered here; nil for a consumer this process did not make. + Retire(ctx context.Context, as string, derived map[string]any, why string, at time.Time) error + // Holds says whether the backend still holds the consumer exactly as p says. Read-only. + Holds(ctx context.Context, p Provision) (bool, error) + // Inventory is what the backend holds that the mesh made: consumers active and retired, read from + // the backend itself so a restart forgets nothing. Read-only. + Inventory(ctx context.Context) (Inventory, error) + // Delete removes one retired consumer — its login and its data — for good. Only ever asked by a + // person, through `cleanup delete`; the harness has checked it is retired and not asked for. + Delete(ctx context.Context, r Retired) (freedBytes int64, err error) +} + +// Harness is the loop's settings and memory. +type Harness struct { + Resource string + Receives string + Adapter Adapter + Every time.Duration // 5s + VerifyEvery time.Duration // 60s + HoldsTimeout time.Duration // 30s + Log func(format string, args ...any) + Now func() time.Time + // Announce publishes one of the provider's standing events; nil announces nothing. Node is the + // machine this provider runs on, said in each. + Announce func(event string, body map[string]any) + Node string + // FailingAfter is how long a consumer fails without a success before it is announced (5m); + // SayAgainEvery is how often it is announced again while it lasts (15m), so a controller that + // missed the first hears the next, and a standing nobody repeats can be told from one that holds. + FailingAfter time.Duration + SayAgainEvery time.Duration + // StablePasses (5) is how many consecutive passes must see the same set no longer asked for, and + // StableFor (10m) how long it must have held since the first of them, before it is retired. RetireAtOnce (3) and RetireFraction (0.5) bound what is retired without a person: + // more consumers than RetireAtOnce, or — where more than one is held — a larger share of those held + // than RetireFraction, waits for `retire approve` (novox/hq ADR 0230). + StablePasses int + StableFor time.Duration + RetireAtOnce int + RetireFraction float64 + + // mu is held by a pass and by every tool a person asks, so the two never interleave. + mu sync.Mutex + verifiedAt time.Time + r retirement + applied map[string]appliedEntry + lost map[string]brake + waiting map[string]int + failing map[string]failure + trouble map[string]*standing + cleared map[string]bool + lastWarning string +} + +// standing is one consumer's unbroken run of failures: since when, how often, and the last error. +type standing struct { + node string + since time.Time + attempts int + class string + text string + saidAt time.Time +} + +// The events a provider's standing is announced as (novox/hq ADR 0224). The controller derives the +// permission to emit them for every module that receives contributions; no manifest lists them. +const ( + EventFailing = "provisioner.failing" + EventRecovered = "provisioner.recovered" +) + +// The classes of error a standing is announced with: what a person reading `status` needs to know +// before reading the journal. An adapter may say better (Classifier). +const ( + ClassCredentials = "credentials-rejected" + ClassUnreachable = "unreachable" + ClassSecret = "secret-unreadable" + ClassRefused = "refused" +) + +// Classifier is an adapter that can say what class an error of its own is. +type Classifier interface { + Class(err error) string +} + +type appliedEntry struct { + hash string + derived map[string]any + node string +} + +type brake struct { + times int + nextAt time.Time +} + +type failure struct { + text string + times int +} + +// The longest a consumer whose create keeps failing to satisfy holds waits between checks. +const maxBackoff = time.Hour + +// How many passes a secret may be unreadable before it stops being called a race (issue 225), and +// once said loudly, how often it is repeated. The same cadence quiets a create that keeps failing +// the same way. +const ( + patiently = 12 + loudlyEvery = 240 +) + +type contribution struct { + As string `json:"as"` + Secret string `json:"secret"` + Node string `json:"node"` + At string `json:"at"` + Values map[string]any `json:"values"` + Derived map[string]any `json:"derived"` +} + +func (h *Harness) init() { + if h.Every == 0 { + h.Every = 5 * time.Second + } + if h.VerifyEvery == 0 { + h.VerifyEvery = time.Minute + } + if h.HoldsTimeout == 0 { + h.HoldsTimeout = 30 * time.Second + } + if h.Now == nil { + h.Now = time.Now + } + if h.FailingAfter == 0 { + h.FailingAfter = 5 * time.Minute + } + if h.SayAgainEvery == 0 { + h.SayAgainEvery = 15 * time.Minute + } + if h.StablePasses == 0 { + h.StablePasses = 5 + } + if h.StableFor == 0 { + h.StableFor = StableFor + } + if h.RetireAtOnce == 0 { + h.RetireAtOnce = 3 + } + if h.RetireFraction == 0 { + h.RetireFraction = 0.5 + } + if h.Log == nil { + h.Log = func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) } + } + if h.applied == nil { + h.applied = map[string]appliedEntry{} + h.lost = map[string]brake{} + h.waiting = map[string]int{} + h.failing = map[string]failure{} + h.trouble = map[string]*standing{} + h.cleared = map[string]bool{} + h.r.retired = map[string]retiredEntry{} + } +} + +// Run reconciles until ctx ends. One consumer's failure never stops the others'. +func (h *Harness) Run(ctx context.Context) { + h.init() + for { + h.Reconcile(ctx) + select { + case <-ctx.Done(): + return + case <-time.After(h.Every): + } + } +} + +func (h *Harness) say(format string, args ...any) { + h.Log("[provisioner:"+h.Resource+"] "+format, args...) +} + +func (h *Harness) warn(why string) { + if why == h.lastWarning { + return + } + if why != "" { + h.say("%s; nothing applied or removed until it can be read", why) + } else { + h.say("contributions file readable again") + } + h.lastWarning = why +} + +// readContributions answers the consumers asked for, or nil when the file says nothing usable. +// **Nothing read is not nobody asking** (novox/hq issue 241): only a file that was read can retire anything. +func (h *Harness) readContributions() []contribution { + raw, err := os.ReadFile(h.Receives) + if err != nil { + h.warn(fmt.Sprintf("contributions file unreadable (%s): %v", h.Receives, err)) + return nil + } + var doc struct { + Requirement string `json:"requirement"` + Given json.RawMessage `json:"given"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + h.warn(fmt.Sprintf("contributions file is not JSON (%s): %v", h.Receives, err)) + return nil + } + if doc.Requirement != "" && doc.Requirement != h.Resource { + h.warn(fmt.Sprintf("%s is for %s, not %s", h.Receives, doc.Requirement, h.Resource)) + return nil + } + var given []contribution + if len(doc.Given) == 0 || string(doc.Given) == "null" || json.Unmarshal(doc.Given, &given) != nil { + h.warn(fmt.Sprintf("%s has no given list", h.Receives)) + return nil + } + h.warn("") + out := []contribution{} + for _, g := range given { + // No `as` is not a credential grant: nothing to create for it. + if g.As != "" && g.Secret != "" { + out = append(out, g) + } + } + return out +} + +func hashOf(as, password string, values, derived map[string]any) string { + // Derived is in the hash: a provider that renames what it derives gave a different resource. + b, _ := json.Marshal([]any{as, password, orEmpty(values), orEmpty(derived)}) + return string(b) +} + +func orEmpty(m map[string]any) map[string]any { + if m == nil { + return map[string]any{} + } + return m +} + +// Reconcile is one pass. +func (h *Harness) Reconcile(ctx context.Context) { + h.mu.Lock() + defer h.mu.Unlock() + h.init() + given := h.readContributions() + if given == nil { + // Not a result: the passes that must agree before anything is retired start again. + h.r.key, h.r.count = "", 0 + return + } + want := map[string]bool{} + for _, g := range given { + want[g.As] = true + } + h.seed(ctx, want) + h.apply(ctx, given) + + // Retire what the mesh has stably stopped asking for — within the bound, or with a person. + h.retireUnasked(ctx, want) + h.r.lastWant = want + for as := range h.failing { + if !want[as] { + delete(h.failing, as) + } + } + // A consumer the mesh stopped asking for that this provider holds nothing for is no longer failed by + // anyone: said, so a standing the controller keeps for it is cleared rather than left naming a + // consumer that is gone. One still held is said recovered when it is retired. + for as := range h.trouble { + if _, held := h.applied[as]; !want[as] && !held { + h.recovered(as, "no longer asked for") + } + } +} + +// apply makes each consumer the grant file gives, and checks every VerifyEvery that the backend still holds it. The +// caller holds h.mu. One consumer's failure never stops the others'. +func (h *Harness) apply(ctx context.Context, given []contribution) { + verifying := h.Now().Sub(h.verifiedAt) >= h.VerifyEvery + if verifying { + h.verifiedAt = h.Now() + } + + for _, g := range given { + raw, err := os.ReadFile(g.Secret) + if err != nil { + // A secret the host has not written yet is a race on the first pass; past a minute it is + // a person's to look at, and said so (novox/hq issue 225). + n := h.waiting[g.As] + 1 + h.waiting[g.As] = n + if n <= patiently { + h.say("%s: secret not readable yet (%s): %v", g.As, g.Secret, err) + } else if n == patiently+1 || n%loudlyEvery == 0 { + h.say("%s: CANNOT READ the secret after %d attempts (%s): %v. This is not a race any more — "+ + "nothing has been provisioned for this consumer and nothing will be until somebody looks. "+ + "Check who owns the file and who this process runs as (novox/hq issue 225)", g.As, n, g.Secret, err) + } + h.failed(g.As, g.Node, ClassSecret, fmt.Sprintf("secret not readable (%s): %v", g.Secret, err)) + continue + } + delete(h.waiting, g.As) + password := strings.TrimSuffix(string(raw), "\n") + p := Provision{As: g.As, Password: password, Values: orEmpty(g.Values), Derived: orEmpty(g.Derived), At: g.At, Consumer: g.Node} + hash := hashOf(g.As, password, g.Values, g.Derived) + + reapplying := 0 + if was, ok := h.applied[g.As]; ok && was.hash == hash { + if !verifying { + continue + } + b, braked := h.lost[g.As] + if braked && h.Now().Before(b.nextAt) { + continue + } + hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) + held, err := h.Adapter.Holds(hctx, p) + timedOut := errors.Is(hctx.Err(), context.DeadlineExceeded) + cancel() + if err != nil { + // Unable to ask is not evidence of loss. A backend that timed out will time out for + // the next consumer too, so the rest of this pass is not asked. + text := scrub(err, password) + h.say("%s: could not check the backend, will ask again: %s", g.As, text) + h.failed(g.As, g.Node, h.classOf(err, text), text) + if timedOut { + verifying = false + } + continue + } + if held { + delete(h.lost, g.As) + h.succeeded(g.As) + continue + } + reapplying = b.times + 1 + if reapplying == 1 { + h.say("%s: the backend no longer holds it; applying again", g.As) + } else { + h.say("%s: still not held after being applied again (%d times in a row) — create does not "+ + "produce what holds checks; applying again", g.As, reapplying) + } + } + if err := h.Adapter.Create(ctx, p); err != nil { + text := scrub(err, password) + f := h.failing[g.As] + if f.text != text { + f = failure{text: text} + } + f.times++ + h.failing[g.As] = f + // Said each time it changes, and while it stays the same, as rarely as a lost secret. + if f.times == 1 || f.times%loudlyEvery == 0 { + h.say("%s: create failed, will retry: %s", g.As, text) + } + h.failed(g.As, g.Node, h.classOf(err, text), text) + if reapplying > 0 { + h.lost[g.As] = brake{times: reapplying - 1} + } + continue + } + if f, was := h.failing[g.As]; was { + h.say("%s: created, after %d failed attempt(s)", g.As, f.times) + delete(h.failing, g.As) + } + h.succeeded(g.As) + h.applied[g.As] = appliedEntry{hash: hash, derived: p.Derived, node: g.Node} + h.reenabled(g.As, g.Node) + if reapplying == 0 { + delete(h.lost, g.As) + } else { + wait := h.VerifyEvery << (reapplying - 1) + if wait > maxBackoff || wait <= 0 { + wait = maxBackoff + } + h.lost[g.As] = brake{times: reapplying, nextAt: h.Now().Add(wait)} + if reapplying > 1 { + h.say("%s: next check in %s", g.As, wait.Round(time.Second)) + } + } + } +} + +// failed counts one more failure in a consumer's unbroken run, and announces the run once it has +// lasted FailingAfter — then again every SayAgainEvery while it lasts. +func (h *Harness) failed(as, node, class, text string) { + now := h.Now() + s := h.trouble[as] + if s == nil { + s = &standing{since: now} + h.trouble[as] = s + } + s.node, s.class, s.text = node, class, text + s.attempts++ + if now.Sub(s.since) < h.FailingAfter { + return + } + if !s.saidAt.IsZero() && now.Sub(s.saidAt) < h.SayAgainEvery { + return + } + first := s.saidAt.IsZero() + s.saidAt = now + if first { + h.say("%s: FAILING for %s (%d attempts, %s): %s. Announced as %s; `status` names it until it "+ + "succeeds (novox/hq ADR 0224)", as, now.Sub(s.since).Round(time.Second), s.attempts, class, text, EventFailing) + } + h.announce(EventFailing, map[string]any{ + "provider": h.Resource, "provider-node": h.Node, + "consumer": as, "node": node, + "class": class, "error": clip(text), + "since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts, + }) +} + +// succeeded ends a consumer's run of failures; one that was announced is announced recovered. +// +// **And the first success for a consumer since this process started is announced too**, failing or +// not: a provider that announced a failure and was restarted has forgotten it, and without this the +// controller would name the consumer failing for ever after it recovered unheard. +func (h *Harness) succeeded(as string) { + if h.trouble[as] == nil && !h.cleared[as] { + h.cleared[as] = true + h.announce(EventRecovered, map[string]any{ + "provider": h.Resource, "provider-node": h.Node, "consumer": as, "why": "first-success", + }) + return + } + h.cleared[as] = true + h.recovered(as, "") +} + +func (h *Harness) recovered(as, why string) { + s := h.trouble[as] + if s == nil { + return + } + delete(h.trouble, as) + if s.saidAt.IsZero() { + return // never announced, so there is nothing to take back + } + if why == "" { + h.say("%s: recovered after %s and %d failed attempt(s)", as, h.Now().Sub(s.since).Round(time.Second), s.attempts) + } + body := map[string]any{ + "provider": h.Resource, "provider-node": h.Node, "consumer": as, "node": s.node, + "since": s.since.UTC().Format(time.RFC3339), "attempts": s.attempts, + } + if why != "" { + body["why"] = why + } + h.announce(EventRecovered, body) +} + +func (h *Harness) announce(event string, body map[string]any) { + if h.Announce != nil { + h.Announce(event, body) + } +} + +// classOf is an error's class: the adapter's word when it has one, else read from the text. +func (h *Harness) classOf(err error, text string) string { + if c, ok := h.Adapter.(Classifier); ok { + if class := c.Class(err); class != "" { + return class + } + } + return ClassOf(text) +} + +// ClassOf reads an error's class from its text — the words the backends the mesh runs use. +func ClassOf(text string) string { + t := strings.ToLower(text) + for _, w := range []string{"invalid_grant", "invalid user credentials", "password authentication failed", + "authentication failed", "unauthorized", " 401"} { + if strings.Contains(t, w) { + return ClassCredentials + } + } + for _, w := range []string{"connection refused", "no such host", "i/o timeout", "deadline exceeded", + "connection reset", "network is unreachable", "no route to host", "eof"} { + if strings.Contains(t, w) { + return ClassUnreachable + } + } + return ClassRefused +} + +// clip keeps an announced error to what belongs in a status line. +func clip(text string) string { + const most = 300 + if len(text) <= most { + return text + } + return text[:most] + "…" +} + +// scrub is an error's text with the consumer's password removed, raw and URL-encoded. +func scrub(err error, password string) string { + text := err.Error() + if password == "" { + return text + } + for _, form := range []string{password, url.QueryEscape(password), url.PathEscape(password)} { + text = strings.ReplaceAll(text, form, "***") + } + return text +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/harness_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/harness_test.go new file mode 100644 index 0000000..d06976a --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/harness_test.go @@ -0,0 +1,154 @@ +package harness + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type world struct { + t *testing.T + dir string + receives string + now time.Time + h *Harness + a *recorder + said []string +} + +func newWorld(t *testing.T) *world { + w := &world{t: t, dir: t.TempDir(), now: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), a: &recorder{held: true}} + w.receives = filepath.Join(w.dir, "mesh.json") + w.h = &Harness{Resource: "postgres-database", Receives: w.receives, Adapter: w.a, + Now: func() time.Time { return w.now }, + Log: func(f string, a ...any) { w.said = append(w.said, f) }} + return w +} + +func (w *world) give(given ...map[string]any) { + for _, g := range given { + secret := filepath.Join(w.dir, g["as"].(string)+".secret") + if err := os.WriteFile(secret, []byte("pw-"+g["as"].(string)+"\n"), 0o600); err != nil { + w.t.Fatal(err) + } + g["secret"] = secret + } + if given == nil { + given = []map[string]any{} + } + raw, _ := json.Marshal(map[string]any{"requirement": "postgres-database", "given": given}) + if err := os.WriteFile(w.receives, raw, 0o600); err != nil { + w.t.Fatal(err) + } +} + +func testAConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword(t *testing.T) { + w := newWorld(t) + w.give(map[string]any{"as": "mesh_ace_letta", "node": "ace", "values": map[string]any{"name": "letta"}}) + w.h.Reconcile(ctx) + w.h.Reconcile(ctx) + if len(w.a.created) != 1 { + t.Fatalf("created %d times", len(w.a.created)) + } + p := w.a.created[0] + if p.As != "mesh_ace_letta" || p.Password != "pw-mesh_ace_letta" || p.Consumer != "ace" { + t.Fatalf("%+v", p) + } +} + +func testAConsumerNoLongerAskedForIsRetiredOnceStable(t *testing.T) { + w := newWorld(t) + w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}) + w.h.Reconcile(ctx) + w.give(map[string]any{"as": "a"}) + w.settle() // five passes and ten minutes + if strings.Join(w.a.removed, ",") != "b" { + t.Fatal(w.a.removed) + } + // Only a file that says nobody asks retires the last one. + w.give() + w.settle() + if strings.Join(w.a.removed, ",") != "b,a" { + t.Fatal(w.a.removed) + } +} + +func testNothingReadIsNotNobodyAsking(t *testing.T) { + for name, content := range map[string]string{ + "unreadable": "", + "not JSON": "{", + "no given": `{"requirement": "postgres-database"}`, + "another": `{"requirement": "mssql-database", "given": []}`, + } { + t.Run(name, func(t *testing.T) { + w := newWorld(t) + w.give(map[string]any{"as": "a"}) + w.h.Reconcile(ctx) + if content == "" { + os.Remove(w.receives) + } else { + os.WriteFile(w.receives, []byte(content), 0o600) + } + w.h.Reconcile(ctx) + if len(w.a.removed) != 0 { + t.Fatalf("retired %v on a file it could not use", w.a.removed) + } + }) + } +} + +func testALostConsumerIsAppliedAgainAndBraked(t *testing.T) { + w := newWorld(t) + w.give(map[string]any{"as": "a"}) + w.h.Reconcile(ctx) + w.a.held = false + w.now = w.now.Add(2 * time.Minute) + w.h.Reconcile(ctx) + if len(w.a.created) != 2 { + t.Fatalf("created %d times", len(w.a.created)) + } + // Still not held a minute later: applied again, then braked for two minutes. + w.now = w.now.Add(61 * time.Second) + w.h.Reconcile(ctx) + w.now = w.now.Add(61 * time.Second) + w.h.Reconcile(ctx) + if len(w.a.created) != 3 { + t.Fatalf("not braked: created %d times", len(w.a.created)) + } +} + +func testAFailingCreateIsRetriedAndSaidOnce(t *testing.T) { + w := newWorld(t) + w.a.failing = &pgErr{"extension \"nope\" refused, password pw-a"} + w.give(map[string]any{"as": "a"}) + for i := 0; i < 5; i++ { + w.h.Reconcile(ctx) + } + n := 0 + for _, s := range w.said { + if strings.Contains(s, "create failed") { + n++ + } + } + if n != 1 { + t.Fatalf("said %d times", n) + } + w.a.failing = nil + w.h.Reconcile(ctx) + if len(w.a.created) != 1 { + t.Fatal("not retried") + } +} + +type pgErr struct{ s string } + +func (e *pgErr) Error() string { return e.s } + +func testScrubRemovesThePassword(t *testing.T) { + if got := scrub(&pgErr{"bad pw a/b c and a%2Fb+c"}, "a/b c"); strings.Contains(got, "a/b c") || strings.Contains(got, "a%2Fb+c") { + t.Fatal(got) + } +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/moved.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/moved.go new file mode 100644 index 0000000..d0d6e48 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/moved.go @@ -0,0 +1,312 @@ +package harness + +// A provider handed to the provisioner (novox/hq ADR 0373, ADR 0349 clause 12, issues 780 and 805): where its +// consumers file is set, it runs on the provider helper (package provider) instead of the harness reading the +// controller's grant files. Moved is that path, shared by every provider that keeps a resource per consumer; a module +// says only what differs, in a MovedSpec. +// +// The helper learns who binds the provider from the plans, composes each consumer's entry and reads its credential +// through ConsumersFile, the file this machine's provisioner client fills. It leaves the rest to the two hooks +// here: Make brings the consumer's resource into being exactly as the harness's adapter does, and Gone leaves +// retirement to the harness's guards (retirement.go): the resource is disabled and marked to delete, never removed. +// +// **A consumer not moved is still the grant file's** (issue 937, ADR 0349 clause 17), for a provider whose spec says +// WithGrants: only a moved consumer's plan names its provider, so the controller still gives every other consumer +// through the grant file (MESH_RECEIVES) until R1 (issue 682). Each login it gives is asked for, and one no plan +// names is made and checked by the harness's grant pass exactly as on the grant path; a login a plan names is the +// helper's alone. A grant file that cannot be read is not nobody asking (issue 241): nothing is retired until it is +// read again. + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient/provider" +) + +// MovedSpec is what differs per provider on the moved path. +type MovedSpec struct { + // Module and Provision are the provider module's name and what it provides, as its manifest says. + Module, Provision string + // Offer is what it serves: its credential form and served values. + Offer provider.Offer + // Login is the identity a consumer is made under at the provider's end (postgres: the entry's User; keycloak: its + // As); an entry with none is not made. + Login func(e provider.Entry) string + // Values are the values the adapter is handed for a consumer; the entry's contribution when not set. + Values func(e provider.Entry) map[string]any + // Noun names what Login is in words ("login", "client id"), Made what is made ("login", "client"), and Backend + // the service in "no longer holds it" ("the backend", "Keycloak"). + Noun, Made, Backend string + // WithGrants makes the logins the grant file gives, that no plan names, asked for and made by the harness's grant + // pass (issue 937). + WithGrants bool +} + +// Moved is a provider on the provider helper, with the harness's own memory and retirement behind it: what is held, +// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's Make +// and Gone and the loop below drive the same state, so the guards are the harness's, not a copy. +type Moved struct { + h *Harness + spec MovedSpec + + // FirstRetry is the first wait before a watch asked too early is asked again (issue 943); a second when zero. + FirstRetry time.Duration + + mu sync.Mutex // guards plans and loaded + plans map[string][]byte + loaded bool + verified map[string]time.Time +} + +func newMoved(h *Harness, spec MovedSpec) *Moved { + h.mu.Lock() + h.init() + h.mu.Unlock() + return &Moved{h: h, spec: spec, plans: map[string][]byte{}, verified: map[string]time.Time{}} +} + +// Make brings one consumer's resource into being under its login and the vault's credential, as the harness does: +// at once when it is new or changed, and otherwise only when the backend no longer holds it, which is asked every +// VerifyEvery. +func (m *Moved) Make(ctx context.Context, c provider.Consumer) error { + h, e := m.h, c.Entry + as := m.spec.Login(e) + if as == "" { + return fmt.Errorf("%s has no %s to make", e.Key(), m.spec.Noun) + } + if len(c.Credential) == 0 { + return fmt.Errorf("%s has no credential to make its %s with", e.Key(), m.spec.Made) + } + h.mu.Lock() + defer h.mu.Unlock() + password := string(c.Credential) + values := orEmpty(e.Contributes) + if m.spec.Values != nil { + values = m.spec.Values(e) + } + p := Provision{As: as, Password: password, Values: values, Consumer: e.Consumer} + hash := hashOf(as, password, p.Values, nil) + if was, ok := h.applied[as]; ok && was.hash == hash { + if h.Now().Sub(m.verified[as]) < h.VerifyEvery { + return nil + } + m.verified[as] = h.Now() + hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) + held, err := h.Adapter.Holds(hctx, p) + cancel() + if err != nil { + text := scrub(err, password) + h.failed(as, e.Consumer, h.classOf(err, text), text) + return errors.New(text) + } + if held { + h.succeeded(as) + return nil + } + h.say("%s: %s no longer holds it; applying again", as, m.spec.Backend) + } + if err := h.Adapter.Create(ctx, p); err != nil { + text := scrub(err, password) + h.failed(as, e.Consumer, h.classOf(err, text), text) + return errors.New(text) + } + h.succeeded(as) + h.applied[as] = appliedEntry{hash: hash, node: e.Consumer} + m.verified[as] = h.Now() + h.reenabled(as, e.Consumer) + return nil +} + +// Gone is the helper letting a consumer go, which it does only on a retiring provision the operator said yes to (novox/hq +// ADR 0570 clause 5): its login is retired at once, through the adapter, its resource and data kept and marked to +// delete, deleted only by a person's `cleanup delete`. A consumer another plan still binds under the same login loses +// nothing. +func (m *Moved) Gone(ctx context.Context, e provider.Entry) error { + h := m.h + as := m.spec.Login(e) + if as == "" { + return nil // refused: nothing was made for it + } + want, _ := m.asked(h.Node) + h.mu.Lock() + defer h.mu.Unlock() + if want[as] { + return nil // another holder of the same login still binds: nothing to retire + } + if _, held := h.applied[as]; !held { + return nil + } + if done := h.retire(ctx, []string{as}, "the operator approved its retirement ("+e.Key()+")", nil); len(done) != 1 { + return fmt.Errorf("%s was not retired; it is tried again at the next pass", as) + } + return nil +} + +// asked is the logins the seat's plans bind the provider for (novox/hq ADR 0570 clause 2), and whether they could all be +// read: a plan that cannot be read, or no read yet, is not an answer (novox/hq issue 241). +func (m *Moved) asked(node string) (map[string]bool, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.loaded || len(m.plans) == 0 { + return nil, false + } + want := map[string]bool{} + for key, raw := range m.plans { + pl, err := provisions.DecodePlan(raw) + if err != nil || pl.Node+"."+pl.Module != key { + return nil, false + } + for _, b := range pl.Plan.Provisions { + if b.Provider == node && b.ProviderModule == m.spec.Module && b.Provision == m.spec.Provision { + as := b.As + if as == "" { + as = provider.ConsumerIdentity(pl.Node, pl.Module) // a plan that names none + } + want[as] = true + } + } + } + return want, true +} + +// granted is what the grant file gives that no plan names (issue 937), and every login it gives; and whether it +// could be read. The caller holds h.mu. With no grant file named there is nothing given, and that is a read. +func (m *Moved) granted(planned map[string]bool) ([]contribution, map[string]bool, bool) { + h := m.h + if h.Receives == "" { + return nil, map[string]bool{}, true + } + given := h.readContributions() + if given == nil { + return nil, nil, false + } + var own []contribution + logins := map[string]bool{} + for _, g := range given { + logins[g.As] = true + if !planned[g.As] { + own = append(own, g) + } + } + return own, logins, true +} + +// retire is one pass over what the plans bind and the grant file gives now: the logins only the grant file gives are +// made and checked, as the grant path does. It retires nothing (ADR 0570 clause 5). +func (m *Moved) retire(ctx context.Context) { + h := m.h + want, ok := m.asked(h.Node) + h.mu.Lock() + defer h.mu.Unlock() + var given []contribution + if ok && m.spec.WithGrants { + var granted map[string]bool + given, granted, ok = m.granted(want) + for as := range granted { + want[as] = true + } + } + if !ok { + // Not a result: the passes that must agree before anything is retired start again. + h.r.key, h.r.count = "", 0 + return + } + h.seed(ctx, want) + h.apply(ctx, given) + // No login is retired for going unasked (novox/hq ADR 0570 clause 5): only Gone, on the operator's yes. + h.r.lastWant = want + for as := range h.failing { + if !want[as] { + delete(h.failing, as) + } + } + for as := range h.trouble { + if _, held := h.applied[as]; !want[as] && !held { + h.recovered(as, "no longer asked for") + } + } +} + +// watch keeps the plans as the bus holds them, from the first read on. +func (m *Moved) watch(ctx context.Context, bus provider.Bus) error { + err := bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) { + m.mu.Lock() + defer m.mu.Unlock() + if value == nil { + delete(m.plans, key) + } else { + m.plans[key] = append([]byte(nil), value...) + } + }) + if err == nil { + m.mu.Lock() + m.loaded = true + m.mu.Unlock() + } + return err +} + +// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends. +// +// The bus is reached through the runtime that launched this bundle, which answers only once Serve runs, and Run starts +// beside Serve: a watch asked too early is asked again, after a wait that doubles from FirstRetry up to a minute, +// until it is answered or ctx ends, so a start that loses the race never leaves the consumers unanswered (issue 943). +func (m *Moved) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error { + wait := m.FirstRetry + if wait <= 0 { + wait = time.Second + } + for tries := 1; ; tries++ { + err := m.watch(ctx, bus) + if err == nil { + break + } + if tries == 4 || tries%20 == 0 { + m.h.say("the plans cannot be watched yet (%d tries), asking again: %v", tries, err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(wait): + } + wait = min(wait*2, time.Minute) + } + go func() { + for { + m.retire(ctx) + select { + case <-ctx.Done(): + return + case <-time.After(m.h.Every): + } + } + }() + return p.Run(ctx) +} + +// NewMoved is a provider handed to the provisioner: the helper answering its consumers, and the harness (never run) +// whose retirement and tools it shares. consumersFile is where its machine's client fills its consumers. Run the +// result with Moved.Run, passing the provider and the bus returned with it. +func NewMoved(consumersFile string, h *Harness, spec MovedSpec) (*Moved, *provider.Provider, provider.Bus, error) { + if consumersFile == "" { + return nil, nil, nil, errors.New("no consumers' file") + } + run := newMoved(h, spec) + bus := provider.StateBus(spec.Module) + return run, provider.New(provider.Options{ + Node: h.Node, + Module: spec.Module, + Offers: map[string]provider.Offer{spec.Provision: spec.Offer}, + Bus: bus, + Vault: provider.ConsumersFile{Paths: []string{consumersFile}}, + Make: run.Make, + Gone: run.Gone, + Every: h.Every, + Logf: func(f string, a ...any) { h.say(f, a...) }, + }), bus, nil +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/moved_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/moved_test.go new file mode 100644 index 0000000..ffa5a0c --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/moved_test.go @@ -0,0 +1,464 @@ +package harness + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sync" + "testing" + "time" + + provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient/provider" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +const ( + ProvisionName = "postgres-database" + servedPort = 5432 +) + +// postgresSpec is what a postgres-like provider says: a login per consumer, and the grant file's logins. +var postgresSpec = MovedSpec{Module: "postgres", Provision: ProvisionName, + Offer: provider.Offer{Credential: provider.PerConsumer, Serves: map[string]any{"port": servedPort}}, + Login: func(e provider.Entry) string { return e.User }, + Noun: "login", Made: "login", Backend: "the backend", WithGrants: true} + +// bucketBus is the helper's bus over in-memory buckets. +type bucketBus struct { + mu sync.Mutex + buckets map[string]map[string][]byte +} + +func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { + b.mu.Lock() + defer b.mu.Unlock() + for k, v := range b.buckets[bucket] { + handle(k, v) + } + return nil +} + +func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.buckets[bucket] == nil { + b.buckets[bucket] = map[string][]byte{} + } + b.buckets[bucket][key] = append([]byte(nil), value...) + return nil +} + +func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.buckets[bucket], key) + return nil +} + +// putPlan is a consumer's plan on the seat mesh-provisioner, of these plan entries, each provision's key provided (novox/hq +// ADR 0570 clauses 1, 2): what the helper binds by. +func (b *bucketBus) putPlan(node, module string, entries ...map[string]any) { + raw, _ := json.Marshal(entries) + var ps []catalogue.PlanProvision + if err := json.Unmarshal(raw, &ps); err != nil { + panic(err) + } + for _, pp := range ps { + pr := provisions.Provision{Provision: pp.Provision, Local: pp.Local, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: pp.ProviderModule, Node: pp.Provider}, State: provisions.Provided} + key, _ := pr.Key() + value, err := provisions.Encode(pr) + if err != nil { + panic(err) + } + _ = b.Put(context.Background(), provisions.ProvisionsBucket, key, value) + } + plan, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: catalogue.Plan{Provisions: ps}}) + if err != nil { + panic(err) + } + _ = b.Put(context.Background(), provisions.PlansBucket, node+"."+module, plan) +} + +// retireOn is a consumer's provision retiring with the operator's yes (ADR 0570 clause 5), its plan gone: the one way +// a provider lets it go. +func (b *bucketBus) retireOn(node, module, provides string) { + pr := provisions.Provision{Provision: ProvisionName, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: provides, Node: "novox"}, State: provisions.Retiring, Why: module + " left", + Warrant: &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"}} + key, _ := pr.Key() + value, _ := provisions.Encode(pr) + _ = b.Delete(context.Background(), provisions.PlansBucket, node+"."+module) + _ = b.Put(context.Background(), provisions.ProvisionsBucket, key, value) +} + +// bind is one consumer's assignment, whose plan names this machine's postgres as the provider and carries what the +// consumer contributes (issue 776). +func (b *bucketBus) bind(node, module string, contributes map[string]any) { + b.bindNamed("postgres", node, module, contributes, nil) +} + +// bindNamed is bind to a provider module, with the names the controller composed for the consumer (issue 792). +func (b *bucketBus) bindNamed(provides, node, module string, contributes map[string]any, names map[string]string) { + b.putPlan(node, module, map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": provides, + "contributes": contributes, "names": names}) +} + +// consumersFile writes the file the machine's client fills: one credential per consumer. +func consumersFile(t *testing.T, bound map[[2]string]string) string { + var list []provider.ConsumerCredential + for k, password := range bound { + list = append(list, provider.ConsumerCredential{Consumer: k[0], Module: k[1], + Credential: provider.PairName("novox", ProvisionName, k[0], k[1], ""), Value: password}) + } + raw, _ := json.Marshal(list) + at := filepath.Join(t.TempDir(), "consumers.json") + if err := os.WriteFile(at, raw, 0o600); err != nil { + t.Fatal(err) + } + return at +} + +// movedWorld is postgres on the helper over an in-memory bus, with a recorder for a server and a clock to move. +type movedWorld struct { + t *testing.T + a *recorder + bus *bucketBus + h *Harness + spec MovedSpec + run *Moved + p *provider.Provider + now time.Time + file string +} + +func newMovedWorld(t *testing.T, bound map[[2]string]string) *movedWorld { + return newMovedWorldOf(t, bound, postgresSpec) +} + +func newMovedWorldOf(t *testing.T, bound map[[2]string]string, spec MovedSpec) *movedWorld { + w := &movedWorld{t: t, spec: spec, a: &recorder{}, bus: &bucketBus{buckets: map[string]map[string][]byte{}}, + now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)} + w.file = consumersFile(t, bound) + w.h = &Harness{Resource: ProvisionName, Adapter: w.a, Node: "novox", Now: func() time.Time { return w.now }, + Log: func(string, ...any) {}} + var bus provider.Bus + w.run, w.p, bus, _ = NewMoved(w.file, w.h, spec) + _ = bus + if err := w.run.watch(ctx, w.bus); err != nil { + t.Fatal(err) + } + return w +} + +// pass is one helper pass and one retirement pass, five seconds after the last. +func (w *movedWorld) pass() []provider.Outcome { + w.now = w.now.Add(5 * time.Second) + // the plans as the bus holds them now, as the watch would hand them + w.run.plans = map[string][]byte{} + for k, v := range w.bus.buckets[provisions.PlansBucket] { + w.run.plans[k] = v + } + // the helper reads its buckets from the in-memory bus + w.p = provider.New(provider.Options{Node: "novox", Module: w.spec.Module, Bus: w.bus, + Vault: provider.ConsumersFile{Paths: []string{w.file}}, + Offers: map[string]provider.Offer{ProvisionName: w.spec.Offer}, + Make: w.run.Make, Gone: w.run.Gone, Logf: func(string, ...any) {}}) + out, err := w.p.Pass(ctx) + if err != nil { + w.t.Fatal(err) + } + w.run.retire(ctx) + return out +} + +func bindings(n int) (map[[2]string]string, []string) { + bound := map[[2]string]string{} + var logins []string + for i := 1; i <= n; i++ { + module := fmt.Sprintf("app%d", i) + bound[[2]string{"ace", module}] = "pw-" + module + logins = append(logins, provider.ConsumerIdentity("ace", module)) + } + return bound, logins +} + +// The consumers' file makes the same calls on the adapter as the equivalent grant files do (the harness). +func testTheConsumersFileMakesWhatTheGrantFilesMake(t *testing.T) { + letta := provider.ConsumerIdentity("ace", "letta") + plain := provider.ConsumerIdentity("ace", "plain") + + // Today: the grant files, through the harness. + g := newWorld(t) + g.give( + map[string]any{"as": letta, "node": "ace", "values": map[string]any{"name": "letta", "extensions": []any{"vector"}}}, + map[string]any{"as": plain, "node": "ace", "values": map[string]any{"name": "plain"}}, + ) + g.h.Reconcile(ctx) + + // The consumers' file, through the provider helper. + w := newMovedWorld(t, map[[2]string]string{{"ace", "letta"}: "pw-" + letta, {"ace", "plain"}: "pw-" + plain}) + w.bus.bind("ace", "letta", map[string]any{"name": "letta", "extensions": []any{"vector"}}) + w.bus.bind("ace", "plain", map[string]any{"name": "plain"}) + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + byAs := map[string]Provision{} + for _, c := range w.a.created { + byAs[c.As] = c + } + if len(w.a.created) != 2 { + t.Fatalf("created %d, want 2", len(w.a.created)) + } + for _, want := range g.a.created { + c := byAs[want.As] + if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer || + !reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) { + t.Fatalf("made %+v, the grant files make %+v", c, want) + } + } + // Unchanged, and held: nothing is made again, within a minute or after it. + w.a.held = true + w.pass() + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 2 { + t.Fatalf("an unchanged pass created again: %d", len(w.a.created)) + } + // No longer held: made again. + w.a.held = false + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 4 { + t.Fatalf("not made again: %d", len(w.a.created)) + } +} + +// A consumer whose plan goes retires nothing by itself, however long (novox/hq ADR 0570 clause 5): its login is retired, +// never dropped, and its entry removed only once its provision is retiring with the operator's yes. +func testAConsumerMissingFromOneReadRetiresNothing(t *testing.T) { + bound, logins := bindings(2) + w := newMovedWorld(t, bound) + w.bus.bind("ace", "app1", nil) + w.bus.bind("ace", "app2", nil) + w.pass() + _ = w.bus.Delete(ctx, provisions.PlansBucket, "ace.app2") + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(30 * time.Minute) + w.pass() + if len(w.a.removed) != 0 || w.h.r.waiting != nil { + t.Fatalf("retired %v, waiting %v, with no warrant", w.a.removed, w.h.r.waiting) + } + if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; !kept { + t.Fatal("the entry went with no warrant") + } + w.bus.retireOn("ace", "app2", "postgres") + w.pass() + if !reflect.DeepEqual(w.a.removed, []string{logins[1]}) || len(w.a.deleted) != 0 { + t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) + } + if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; kept { + t.Fatal("the entry stayed after the login was retired") + } +} + +// A plan list that was never read is not nobody asking. +func testNoPlansReadRetiresNothing(t *testing.T) { + bound, _ := bindings(1) + w := newMovedWorld(t, bound) + w.bus.bind("ace", "app1", nil) + w.pass() + w.bus.Delete(ctx, provisions.PlansBucket, "ace.app1") + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(20 * time.Minute) + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v with no plans read", w.a.removed) + } +} + +// A consumer whose credential is not in the file yet is waited for: nothing is made for it. +func testAConsumerWithoutACredentialIsWaitedFor(t *testing.T) { + w := newMovedWorld(t, map[[2]string]string{}) + w.bus.bind("ace", "letta", map[string]any{"name": "letta"}) + outcomes := w.pass() + if len(outcomes) != 1 || outcomes[0].Status != provider.Waiting || len(w.a.created) != 0 { + t.Fatalf("%+v, created %v", outcomes, w.a.created) + } +} + +// Where the file is not there, postgres takes the grant directory, as before; where it is, the helper. +// The grant path restarts itself when the consumers file is first filled. +// A consumer whose module has a slug keeps the login its plan names, the slug first (issue 788): the adapter makes it +// under that login, and the retirement counts it as asked for, so a pass retires nothing. +func testASluggedConsumerKeepsTheLoginItsPlanNames(t *testing.T) { + agent := provider.ConsumerIdentity("ace", "agent") + w := newMovedWorld(t, map[[2]string]string{{"ace", "claude-code"}: "pw-agent"}) + w.bus.putPlan("ace", "claude-code", map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": "postgres", "as": agent}) + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + if len(w.a.created) != 1 || w.a.created[0].As != agent { + t.Fatalf("made %+v, want one under %s", w.a.created, agent) + } + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(15 * time.Minute) + w.pass() + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v: the plan's login was counted unasked", w.a.removed) + } +} + +// A provider whose spec names the plan's `as` as the login and no grant file (the keycloak shape) makes the consumer +// under that name with the values its spec composes, and a grant file that cannot be read does not hold up its +// retirement pass. +func testASpecWithoutGrantsMakesUnderThePlansAsAndIgnoresTheGrantFile(t *testing.T) { + a := &recorder{} + bus := &bucketBus{buckets: map[string]map[string][]byte{}} + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + h := &Harness{Resource: ProvisionName, Adapter: a, Node: "novox", Now: func() time.Time { return now }, + Receives: filepath.Join(t.TempDir(), "unreadable-or-missing.json"), Log: func(string, ...any) {}} + spec := MovedSpec{Module: "keycloak", Provision: ProvisionName, + Offer: provider.Offer{Credential: provider.PerConsumer}, + Login: func(e provider.Entry) string { return e.As }, Noun: "client id", Made: "client", Backend: "Keycloak", + Values: func(e provider.Entry) map[string]any { return map[string]any{"label": e.Contributes["label"]} }} + file := consumersFile(t, map[[2]string]string{{"ace", "wiki"}: "pw-wiki"}) + run, _, _, err := NewMoved(file, h, spec) + if err != nil { + t.Fatal(err) + } + bus.putPlan("ace", "wiki", map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": "keycloak", "as": "wiki-client", "contributes": map[string]any{"label": "Wiki"}}) + if err := run.watch(ctx, bus); err != nil { + t.Fatal(err) + } + p := provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: bus, + Vault: provider.ConsumersFile{Paths: []string{file}}, + Offers: map[string]provider.Offer{ProvisionName: spec.Offer}, Make: run.Make, Gone: run.Gone, + Logf: func(string, ...any) {}}) + out, err := p.Pass(ctx) + if err != nil || len(out) != 1 || out[0].Status != provider.Answered { + t.Fatalf("%v %+v", err, out) + } + run.retire(ctx) + if len(a.created) != 1 || a.created[0].As != "wiki-client" || a.created[0].Password != "pw-wiki" || + a.created[0].Values["label"] != "Wiki" { + t.Fatalf("made %+v", a.created) + } +} + +const testIssuer = "https://id.example.org/realms/Novox" + +var ( + grafanaContributes = map[string]any{"label": "grafana", "endpoint": "web", "callback": "/login/generic_oauth"} + grafanaNames = map[string]string{"name": "grafana.example.org", "internal-name": "grafana.home.internal"} +) + +// keycloakSpec is what a keycloak-like provider says: the client id is the plan's `as`, the values are what the +// consumer contributes with the controller's names over them, and no grant file is consulted. +var keycloakSpec = MovedSpec{Module: "keycloak", Provision: ProvisionName, + Offer: provider.Offer{Credential: provider.PerConsumer, Serves: map[string]any{"issuer": testIssuer, + "token-path": "/protocol/openid-connect/token"}}, + Login: func(e provider.Entry) string { return e.As }, + Values: func(e provider.Entry) map[string]any { + values := map[string]any{} + for k, v := range e.Contributes { + values[k] = v + } + for k, name := range e.Names { + values[k] = name + } + return values + }, + Noun: "client id", Made: "client", Backend: "Keycloak"} + +// The consumers' file makes the same call on the adapter as the equivalent grant file does (the harness), for a +// provider of the keycloak shape: the same client id, secret and values; and the entry tells the consumer what the +// provider serves. +func testTheConsumersFileMakesWhatTheGrantFileMakesForTheKeycloakShape(t *testing.T) { + grafana := provider.ConsumerIdentity("home", "grafana") + values := map[string]any{"name": grafanaNames["name"], "internal-name": grafanaNames["internal-name"]} + for k, v := range grafanaContributes { + values[k] = v + } + g := newWorld(t) + g.give(map[string]any{"as": grafana, "node": "home", "values": values}) + g.h.Reconcile(ctx) + + w := newMovedWorldOf(t, map[[2]string]string{{"home", "grafana"}: "pw-" + grafana}, keycloakSpec) + w.bus.bindNamed("keycloak", "home", "grafana", grafanaContributes, grafanaNames) + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + if len(w.a.created) != 1 || len(g.a.created) != 1 { + t.Fatalf("created %d through the helper, %d through the harness, want 1 each", len(w.a.created), len(g.a.created)) + } + c, want := w.a.created[0], g.a.created[0] + if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer || + !reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) { + t.Fatalf("made %+v, the grant file makes %+v", c, want) + } + var entry provider.Entry + if err := json.Unmarshal(w.bus.buckets[provider.BucketOf("keycloak")]["home.grafana."+ProvisionName], &entry); err != nil { + t.Fatal(err) + } + if entry.Values["issuer"] != testIssuer || entry.Values["token-path"] != "/protocol/openid-connect/token" { + t.Fatalf("served %v", entry.Values) + } + w.a.held = true + w.pass() + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 1 { + t.Fatalf("an unchanged pass created again: %d", len(w.a.created)) + } + w.a.held = false + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 2 { + t.Fatalf("not made again: %d", len(w.a.created)) + } +} + +// A consumer whose plan names its client id (a slugged module) keeps it, and the retirement counts it as asked for. +func testAConsumerNamedByItsPlanKeepsItsClientId(t *testing.T) { + agent := provider.ConsumerIdentity("home", "dashboard") + w := newMovedWorldOf(t, map[[2]string]string{{"home", "grafana-2"}: "pw"}, keycloakSpec) + w.bus.putPlan("home", "grafana-2", map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak", + "as": agent, "contributes": grafanaContributes, "names": grafanaNames}) + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + if len(w.a.created) != 1 || w.a.created[0].As != agent { + t.Fatalf("made %+v, want one under %s", w.a.created, agent) + } + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(15 * time.Minute) + w.pass() + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v: the plan's client id was counted unasked", w.a.removed) + } +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/retirement.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/retirement.go new file mode 100644 index 0000000..180213a --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/retirement.go @@ -0,0 +1,626 @@ +package harness + +// What becomes of a consumer the mesh no longer asks for (novox/hq ADR 0230 — the operator's model, +// decided 2026-10-06; it replaces ADR 0229's withdrawal brake, which let one consumer go every hour). +// +// A consumer — a login, a client, a key, and the data behind it — is in one of three states: +// +// 1. ACTIVE. +// 2. RETIRED. The mesh stopped asking for it: its access is disabled, reversibly, and its login and +// data are marked "to delete" with when and why. Nothing is deleted. Asked for again, the ordinary +// create re-enables it at once, as it was. +// 3. DELETED, only through `cleanup delete`, a person's act, which this provider executes because it +// owns its backend. +// +// **Stable removals.** A consumer is retired only once the same set of consumers has gone unasked BOTH +// in StablePasses (5) consecutive passes that read the file AND for at least StableFor (10 minutes) +// since the first pass that saw it. Five passes alone are twenty-five seconds — shorter than a +// controller restart, a store reconnecting or a file half written. A pass that could not read the file +// is not a result and starts both again; so does a different set. Additions and changes are never +// delayed. +// +// **Too many is a person.** A stable set of more than RetireAtOnce (3), or — where more than one is +// held — of more than RetireFraction (half) of those held, retires nothing: it WAITS, said in the +// journal and announced `provisioner.retirement` `waiting` (again every SayAgainEvery), which the +// controller raises as an urgent condition, until `retire approve ` or `retire reject`. +// An unassignment the controller made itself is no exception: many changes at once means a person is +// at work, and a person confirms once. On 2026-10-04 one misread file withdrew seven consumers at once +// (issue 241); under this rule that is a question, not an act. +// +// **The backend remembers, not this process.** What is retired, since when and why is read from the +// backend's own mark (Adapter.Inventory), so a restart forgets nothing; a consumer the backend holds +// active that the mesh no longer asks for is retired by the same rules after a restart as before one. +// A consumer found disabled without the mark — withdrawn before this record — is ADOPTED as retired: +// marked, said, announced `adopted`, and its clock starts then. +// +// **A rejection lives as long as this process.** Rejected, the set is kept active and not asked about +// again while it stays the same set; a restart asks again — loudly, never silently. + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// EventRetirement is the one event a provider says about retirement, its `change` saying what +// happened. The controller derives the permission to emit it for every module that receives +// contributions, as it does the standing events (novox/hq ADR 0224, 0230). +const EventRetirement = "provisioner.retirement" + +// The changes EventRetirement carries. +const ( + ChangeWaiting = "waiting" // a stable set over the bound waits for a person + ChangeSettled = "settled" // a waiting or rejected set ended without being retired + ChangeApproved = "approved" // a person approved the waiting (or rejected) set + ChangeRejected = "rejected" // a person kept the waiting set active + ChangeRetired = "retired" // consumers disabled and marked to delete + ChangeReenabled = "reenabled" // a retired consumer asked for again, enabled as it was + ChangeDeleted = "deleted" // a retired consumer deleted, by a person + ChangeAdopted = "adopted" // found disabled without the mark, now retired on record +) + +// StableFor is the least time the same unasked set must hold before it is retired (novox/hq ADR 0230): +// longer than a controller restart, a store reconnecting or a file half written. +const StableFor = 10 * time.Minute + +// KindConsumer is a retired consumer. A backend may list other things set aside for deletion under a +// word of its own (postgres: a database renamed aside). +const KindConsumer = "consumer" + +// Retired is one thing a provider holds retired, as its backend's mark says. +type Retired struct { + Consumer string + // Node is the consumer's machine, where the mark records it. + Node string + // RetiredAt is when it was retired; zero for one found disabled without the mesh's mark. + RetiredAt time.Time + Why string + // SizeBytes is what deleting it would free; -1 when the backend cannot say. + SizeBytes int64 + Kind string +} + +// Inventory is what a backend holds that the mesh made. +type Inventory struct { + Active []string + Retired []Retired + // Sizes is what each active consumer keeps on the backend, in bytes, where the backend can say: what + // the controller tells an empty replacement of a consumer's data by (novox/hq ADR 0233). + Sizes map[string]int64 +} + +// retirement is the loop's memory of the sets it is counting, waiting on and was told to keep. +type retirement struct { + key string // the unasked set, joined + count int // consecutive passes that saw it + firstSeen time.Time + waiting *waitingSet + rejected *rejectedSet + retired map[string]retiredEntry + lastWant map[string]bool + seeded bool + seedSaid string + // noneSaid is whether this process has said that nothing waits: an earlier process may have left a set + // waiting that this one never knew (issue 937). + noneSaid bool +} + +type waitingSet struct { + set []string + since time.Time + saidAt time.Time + held int +} + +type rejectedSet struct { + set []string + by, why string + at time.Time +} + +type retiredEntry struct { + node string + at time.Time + why string +} + +// seed reads what the backend holds, once per process: an active consumer the mesh no longer asks +// for is held, so it is retired by the same rules as one this process made; one found disabled +// without the mark is adopted as retired. +func (h *Harness) seed(ctx context.Context, want map[string]bool) { + if h.r.seeded { + return + } + inv, err := h.Adapter.Inventory(ctx) + if err != nil { + if said := err.Error(); said != h.r.seedSaid { + h.say("cannot list what the backend holds (%v); a consumer the mesh stopped asking for while this "+ + "provider was down is not retired until it can", err) + h.r.seedSaid = said + } + return + } + h.r.seeded = true + for _, as := range inv.Active { + if _, held := h.applied[as]; !held && !want[as] { + // No hash: asked for again, it is applied again, which is harmless and marks it. + h.applied[as] = appliedEntry{} + h.say("%s: held by the backend and no longer asked for; retired once that holds for %d passes "+ + "and %s (novox/hq ADR 0230)", as, h.StablePasses, h.StableFor) + } + } + now := h.Now() + for _, r := range inv.Retired { + if r.Kind != "" && r.Kind != KindConsumer { + continue + } + if !r.RetiredAt.IsZero() { + h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: r.RetiredAt, why: r.Why} + continue + } + if want[r.Consumer] { + continue // asked for: this pass's create enables it + } + why := "found disabled without the mesh's mark — withdrawn before retirement was recorded " + + "(novox/hq ADR 0230); retired as found" + if err := h.Adapter.Retire(ctx, r.Consumer, nil, why, now); err != nil { + h.say("%s: found disabled and could not be marked retired, will try at the next start: %v", r.Consumer, err) + continue + } + h.r.retired[r.Consumer] = retiredEntry{node: r.Node, at: now, why: why} + h.say("%s: ADOPTED as retired — %s", r.Consumer, why) + h.announce(EventRetirement, h.retirementBody(ChangeAdopted, []map[string]any{ + {"consumer": r.Consumer, "node": r.Node, "retired_at": stamp(now), "why": why, "size_bytes": r.SizeBytes}, + }, map[string]any{"why": why})) + } +} + +// retireUnasked counts the passes that saw the same set no longer asked for and, once it is stable, +// retires it — or, past the bound, waits for a person. +func (h *Harness) retireUnasked(ctx context.Context, want map[string]bool) { + var unasked []string + for as := range h.applied { + if !want[as] { + unasked = append(unasked, as) + } + } + sort.Strings(unasked) + key := strings.Join(unasked, "\x00") + now := h.Now() + switch { + case len(unasked) == 0: + h.settle("every consumer held is asked for again") + h.r.key, h.r.count = "", 0 + return + case key != h.r.key: + if h.r.waiting != nil || h.r.rejected != nil { + h.settle("the set the mesh no longer asks for changed") + } + h.r.key, h.r.count, h.r.firstSeen = key, 1, now + h.say("no longer asked for: %s; retired once the same set holds for %d passes and %s", + strings.Join(unasked, ", "), h.StablePasses, h.StableFor) + default: + h.r.count++ + } + if h.r.rejected != nil || h.r.count < h.StablePasses || now.Sub(h.r.firstSeen) < h.StableFor { + return + } + if h.overTheBound(len(unasked), len(h.applied)) { + h.wait(unasked) + return + } + why := fmt.Sprintf("the mesh stopped asking for it: the same result in %d consecutive passes over %s, from %s", + h.r.count, now.Sub(h.r.firstSeen).Round(time.Second), stamp(h.r.firstSeen)) + h.retire(ctx, unasked, why, nil) +} + +// overTheBound says retiring n of the held consumers at once needs a person. +func (h *Harness) overTheBound(n, held int) bool { + if n == 0 { + return false + } + return n > h.RetireAtOnce || (held > 1 && float64(n) > h.RetireFraction*float64(held)) +} + +func (h *Harness) bound(held int) string { + return fmt.Sprintf("more than %d at once, or more than %.0f%% of the %d held", h.RetireAtOnce, + h.RetireFraction*100, held) +} + +// wait holds a stable set over the bound for a person, said once and announced again every +// SayAgainEvery so a controller that missed the first hears the next. +func (h *Harness) wait(set []string) { + now := h.Now() + w := h.r.waiting + if w == nil { + w = &waitingSet{set: set, since: now, held: len(h.applied)} + h.r.waiting = w + h.say("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d consumer(s) this provider "+ + "holds (%s), %s. Nothing is retired; each stays active until `retire approve %s ` or "+ + "`retire reject` (novox/hq ADR 0230)", len(set), w.held, strings.Join(set, ", "), h.bound(w.held), h.Node) + } else if now.Sub(w.saidAt) < h.SayAgainEvery { + return + } + w.saidAt = now + h.announce(EventRetirement, h.retirementBody(ChangeWaiting, h.named(set), map[string]any{ + "held": w.held, "bound": h.bound(w.held), "since": stamp(w.since), + })) +} + +// settle ends a waiting or rejected set that the mesh's own answer made moot. +func (h *Harness) settle(why string) { + var set []string + switch { + case h.r.waiting != nil: + set = h.r.waiting.set + case h.r.rejected != nil: + set = h.r.rejected.set + case h.r.noneSaid: + return + default: + // A restart forgets a set an earlier process waited on, and so would never say it ended: the + // controller's condition would ask a person to approve what nobody waits on any more (issue 937). + // The first time this process finds every consumer held asked for, it says so once, with no set. + h.r.noneSaid = true + h.say("nothing waits for a person: %s", why) + h.announce(EventRetirement, h.retirementBody(ChangeSettled, []map[string]any{}, map[string]any{"why": why})) + return + } + h.r.waiting, h.r.rejected = nil, nil + h.say("retirement of %s settled without retiring: %s", strings.Join(set, ", "), why) + h.announce(EventRetirement, h.retirementBody(ChangeSettled, h.named(set), map[string]any{"why": why})) +} + +// retire disables and marks each consumer of set; one that fails stays held and is tried again once +// its set is stable again. +func (h *Harness) retire(ctx context.Context, set []string, why string, extra map[string]any) []string { + now := h.Now() + held := len(h.applied) + var done []string + var said []map[string]any + for _, as := range set { + was, ok := h.applied[as] + if !ok { + continue + } + if err := h.Adapter.Retire(ctx, as, was.derived, why, now); err != nil { + h.say("%s: retiring failed, will try again: %v", as, err) + continue + } + delete(h.applied, as) + delete(h.lost, as) + delete(h.failing, as) + h.r.retired[as] = retiredEntry{node: was.node, at: now, why: why} + h.recovered(as, "retired") + h.say("%s: RETIRED — its access disabled and its data kept, marked to delete (%s). Asked for again "+ + "it is re-enabled as it was; it is deleted only by `cleanup delete` (novox/hq ADR 0230)", as, why) + done = append(done, as) + said = append(said, map[string]any{"consumer": as, "node": was.node, "retired_at": stamp(now), "why": why}) + } + h.r.key, h.r.count, h.r.waiting, h.r.rejected = "", 0, nil, nil + if len(done) > 0 { + body := map[string]any{"held": held, "why": why} + for k, v := range extra { + body[k] = v + } + h.announce(EventRetirement, h.retirementBody(ChangeRetired, said, body)) + } + return done +} + +// reenabled says a retired consumer was asked for again and its create enabled it. +func (h *Harness) reenabled(as, node string) { + e, was := h.r.retired[as] + if !was { + return + } + delete(h.r.retired, as) + h.say("%s: asked for again — re-enabled as it was, its mark to delete cleared (retired %s: %s)", as, + stamp(e.at), e.why) + h.announce(EventRetirement, h.retirementBody(ChangeReenabled, []map[string]any{ + {"consumer": as, "node": node, "retired_at": stamp(e.at), "why": e.why}, + }, nil)) +} + +// Approve retires exactly the set waiting (or the set rejected) — a person's confirmation. +func (h *Harness) Approve(ctx context.Context, consumers []string, why, by, via string) ([]string, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.init() + if strings.TrimSpace(why) == "" { + return nil, errors.New("approve: say why") + } + set := sorted(consumers) + var held []string + switch { + case h.r.waiting != nil && same(set, h.r.waiting.set): + held = h.r.waiting.set + case h.r.rejected != nil && same(set, h.r.rejected.set): + held = h.r.rejected.set + default: + return nil, fmt.Errorf("approve: %s is not the set waiting here — %s", orNone(set), h.waitingWords()) + } + h.say("retirement of %s APPROVED by %s: %s", strings.Join(held, ", "), orSomebody(by), why) + h.announce(EventRetirement, h.retirementBody(ChangeApproved, h.named(held), + map[string]any{"why": why, "by": by, "via": via})) + reason := fmt.Sprintf("the mesh stopped asking for it; approved by %s: %s", orSomebody(by), why) + return h.retire(ctx, held, reason, map[string]any{"by": by, "via": via}), nil +} + +// Reject keeps the waiting set active; it is not asked about again while it stays the same set. +func (h *Harness) Reject(consumers []string, why, by, via string) ([]string, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.init() + if strings.TrimSpace(why) == "" { + return nil, errors.New("reject: say why") + } + set := sorted(consumers) + if h.r.waiting == nil || !same(set, h.r.waiting.set) { + return nil, fmt.Errorf("reject: %s is not the set waiting here — %s", orNone(set), h.waitingWords()) + } + h.r.rejected = &rejectedSet{set: h.r.waiting.set, by: by, why: why, at: h.Now()} + h.r.waiting = nil + h.say("retirement of %s REJECTED by %s: %s. Kept active, and not asked about again while the mesh goes on "+ + "not asking for exactly these (until this provider restarts)", strings.Join(set, ", "), orSomebody(by), why) + h.announce(EventRetirement, h.retirementBody(ChangeRejected, h.named(set), + map[string]any{"why": why, "by": by, "via": via})) + return set, nil +} + +// DeleteRetired deletes one retired consumer, by a person's word: never an active one, never one the +// mesh asks for. +func (h *Harness) DeleteRetired(ctx context.Context, consumer, why, by, via string) (int64, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.init() + if strings.TrimSpace(why) == "" { + return 0, errors.New("delete: say why") + } + if h.r.lastWant[consumer] { + return 0, fmt.Errorf("delete: the mesh asks for %s — it is not retired", consumer) + } + if _, held := h.applied[consumer]; held { + return 0, fmt.Errorf("delete: %s is active here — only a retired consumer is deleted", consumer) + } + inv, err := h.Adapter.Inventory(ctx) + if err != nil { + return 0, fmt.Errorf("delete: what the backend holds cannot be read, so nothing is deleted: %w", err) + } + var found *Retired + for i := range inv.Retired { + if inv.Retired[i].Consumer == consumer { + found = &inv.Retired[i] + } + } + if found == nil { + return 0, fmt.Errorf("delete: %s is not retired here — only a retired consumer is deleted", consumer) + } + freed, err := h.Adapter.Delete(ctx, *found) + if err != nil { + return 0, err + } + delete(h.r.retired, consumer) + h.say("%s: DELETED by %s: %s (retired %s: %s; %d bytes freed)", consumer, orSomebody(by), why, + stampOr(found.RetiredAt), found.Why, freed) + h.announce(EventRetirement, h.retirementBody(ChangeDeleted, []map[string]any{{ + "consumer": consumer, "node": found.Node, "retired_at": stampOr(found.RetiredAt), "why": found.Why, + "size_bytes": found.SizeBytes, "kind": kindOf(*found), + }}, map[string]any{"why": why, "by": by, "via": via, "freed_bytes": freed})) + return freed, nil +} + +// Retirement is what a person (and the controller's `cleanup list` and its probe) asks: what is +// held, waiting, rejected and retired here. +func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.init() + inv, err := h.Adapter.Inventory(ctx) + if err != nil { + return nil, err + } + var held []string + for as := range h.applied { + held = append(held, as) + } + sort.Strings(held) + retired := []map[string]any{} + for _, r := range inv.Retired { + retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node, + "retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)}) + } + sizes := map[string]int64{} + for _, as := range held { + if size, ok := inv.Sizes[as]; ok && size >= 0 { + sizes[as] = size + } + } + out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), "held_sizes": sizes, + "stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil, + "retired": retired} + if w := h.r.waiting; w != nil { + out["waiting"] = map[string]any{"consumers": h.named(w.set), "since": stamp(w.since), "held": w.held} + } + if r := h.r.rejected; r != nil { + out["rejected"] = map[string]any{"consumers": h.named(r.set), "by": r.by, "why": r.why, "at": stamp(r.at)} + } + return out, nil +} + +func (h *Harness) waitingWords() string { + switch { + case h.r.waiting != nil: + return "waiting: " + strings.Join(h.r.waiting.set, ", ") + case h.r.rejected != nil: + return "nothing waits; rejected and kept: " + strings.Join(h.r.rejected.set, ", ") + } + return "nothing waits for a person here" +} + +// named is a set with each consumer's machine, as the events and the tools say it. +func (h *Harness) named(set []string) []map[string]any { + out := make([]map[string]any, 0, len(set)) + for _, as := range set { + out = append(out, map[string]any{"consumer": as, "node": h.applied[as].node}) + } + return out +} + +func (h *Harness) retirementBody(change string, consumers []map[string]any, extra map[string]any) map[string]any { + body := map[string]any{"provider": h.Resource, "provider-node": h.Node, "change": change, + "consumers": consumers, "at": stamp(h.Now())} + for k, v := range extra { + body[k] = v + } + return body +} + +// RetirementTools are the four tools every provider serves for retirement, the same names in every +// module: a caller asks them on the provider's machine, with the operator's answer named in why and by. +func RetirementTools(h *Harness) []stdio.Tool { + if h == nil { + return nil + } + ask := func() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), 2*time.Minute) + } + who := map[string]any{ + "why": map[string]any{"type": "string", "description": "why — required, kept in the hand-act log"}, + "by": map[string]any{"type": "string", "description": "who decided"}, + "via": map[string]any{"type": "string", "description": "the part or session that asks, when not the caller itself"}, + } + withSet := map[string]any{"consumers": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, + "description": "the set, exactly as provisioner_retirement names it"}} + for k, v := range who { + withSet[k] = v + } + withOne := map[string]any{ + "consumer": map[string]any{"type": "string", "description": "the retired consumer to delete"}, + "confirm": map[string]any{"type": "string", "description": "the consumer's name again, to say this is meant"}, + } + for k, v := range who { + withOne[k] = v + } + return []stdio.Tool{ + {Name: "provisioner_retirement", + Description: "What this provider holds, what waits for a person to approve its retirement, what was rejected, " + + "and every retired consumer with when, why and its size (novox/hq ADR 0230).", + Run: func(map[string]any) (any, error) { + ctx, cancel := ask() + defer cancel() + return h.Retirement(ctx) + }}, + {Name: "provisioner_retire_approve", + Description: "Retire exactly the set waiting for a person (or the set rejected earlier): access disabled, data " + + "kept and marked to delete. It asks nobody: the operator's answer on the operator channel to the provisioner's " + + "ask (novox/hq ADR 0570 §5) is named in why and by, which the hand-act log keeps.", + Input: withSet, + Run: func(args map[string]any) (any, error) { + ctx, cancel := ask() + defer cancel() + done, err := h.Approve(ctx, strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via")) + if err != nil { + return nil, err + } + return map[string]any{"retired": orEmptyList(done)}, nil + }}, + {Name: "provisioner_retire_reject", + Description: "Keep the set waiting for a person active. It asks nobody: the operator's answer on the operator channel " + + "(novox/hq ADR 0570 §5) is named in why and by, which the hand-act log keeps.", + Input: withSet, + Run: func(args map[string]any) (any, error) { + kept, err := h.Reject(strs(args["consumers"]), argString(args, "why"), argString(args, "by"), argString(args, "via")) + if err != nil { + return nil, err + } + return map[string]any{"kept": kept}, nil + }}, + {Name: "provisioner_delete", + Description: "Delete one RETIRED consumer — its login and its data — for good. Refused for anything active or " + + "asked for. It asks nobody: ask the operator on the operator channel first, and name that answer in why and " + + "by, which the hand-act log keeps (novox/hq ADR 0604 §8).", + Input: withOne, + Run: func(args map[string]any) (any, error) { + consumer := argString(args, "consumer") + if consumer == "" || argString(args, "confirm") != consumer { + return nil, errors.New("delete: name the consumer, and repeat it in confirm") + } + ctx, cancel := ask() + defer cancel() + freed, err := h.DeleteRetired(ctx, consumer, argString(args, "why"), argString(args, "by"), argString(args, "via")) + if err != nil { + return nil, err + } + return map[string]any{"deleted": consumer, "freed_bytes": freed}, nil + }}, + } +} + +func strs(v any) []string { + list, _ := v.([]any) + out := []string{} + for _, x := range list { + if s, ok := x.(string); ok && s != "" { + out = append(out, s) + } + } + return out +} + +func sorted(list []string) []string { + out := append([]string(nil), list...) + sort.Strings(out) + return out +} + +func same(a, b []string) bool { + return strings.Join(sorted(a), "\x00") == strings.Join(sorted(b), "\x00") +} + +func orNone(set []string) string { + if len(set) == 0 { + return "an empty set" + } + return strings.Join(set, ", ") +} + +func orSomebody(by string) string { + if by == "" { + return "a person" + } + return by +} + +func orEmptyList(list []string) []string { + if list == nil { + return []string{} + } + return list +} + +func kindOf(r Retired) string { + if r.Kind == "" { + return KindConsumer + } + return r.Kind +} + +func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339) } + +func stampOr(t time.Time) string { + if t.IsZero() { + return "" + } + return stamp(t) +} + +func argString(args map[string]any, key string) string { + s, _ := args[key].(string) + return s +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/retirement_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/retirement_test.go new file mode 100644 index 0000000..4408e1f --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/retirement_test.go @@ -0,0 +1,575 @@ +package harness + +// Retirement (novox/hq ADR 0230), as the shared loop does it: a consumer the mesh stops asking for is +// retired only after the same result in five consecutive passes, too many at once wait for a person, +// asked for again it is re-enabled, and only a person deletes. The same file in every Go provider. + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" +) + +// recorder is a backend that remembers what it holds, active and retired, as a real one's mark does. +type recorder struct { + created []Provision + removed []string // retired, in order + deleted []string + held bool + failing error + holdsErr error + retErr error + inv Inventory + invErr error +} + +func (r *recorder) Create(_ context.Context, p Provision) error { + if r.failing != nil { + return r.failing + } + r.created = append(r.created, p) + r.inv.Retired = withoutRetired(r.inv.Retired, p.As) + if !listHas(r.inv.Active, p.As) { + r.inv.Active = append(r.inv.Active, p.As) + } + return nil +} + +func (r *recorder) Retire(_ context.Context, as string, _ map[string]any, why string, at time.Time) error { + if r.retErr != nil { + return r.retErr + } + r.removed = append(r.removed, as) + r.inv.Active = without(r.inv.Active, as) + r.inv.Retired = append(withoutRetired(r.inv.Retired, as), + Retired{Consumer: as, RetiredAt: at, Why: why, SizeBytes: 42, Kind: KindConsumer}) + return nil +} + +func (r *recorder) Holds(context.Context, Provision) (bool, error) { + if r.holdsErr != nil { + return false, r.holdsErr + } + return r.held, nil +} + +func (r *recorder) Inventory(context.Context) (Inventory, error) { return r.inv, r.invErr } + +func (r *recorder) Delete(_ context.Context, x Retired) (int64, error) { + r.deleted = append(r.deleted, x.Consumer) + r.inv.Retired = withoutRetired(r.inv.Retired, x.Consumer) + return x.SizeBytes, nil +} + +func listHas(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func without(list []string, s string) []string { + var out []string + for _, x := range list { + if x != s { + out = append(out, x) + } + } + return out +} + +func withoutRetired(list []Retired, s string) []Retired { + var out []Retired + for _, x := range list { + if x.Consumer != s { + out = append(out, x) + } + } + return out +} + +// holding gives the provider n consumers c1…cn and applies them. +func holding(w *world, n int) []map[string]any { + var given []map[string]any + for i := 1; i <= n; i++ { + given = append(given, map[string]any{"as": fmt.Sprintf("c%d", i), "node": "anchor"}) + } + w.give(given...) + w.h.Reconcile(ctx) + return given +} + +// pass is one reconcile five seconds after the last. +func (w *world) pass() { + w.now = w.now.Add(5 * time.Second) + w.h.Reconcile(ctx) +} + +func retirements(said []announced) []string { + var out []string + for _, a := range said { + if a.event == EventRetirement { + out = append(out, a.body["change"].(string)) + } + } + return out +} + +func noneWaits(a announced) bool { + c, _ := a.body["consumers"].([]map[string]any) + return a.body["change"] == ChangeSettled && len(c) == 0 +} + +func lastRetirement(t *testing.T, said []announced) map[string]any { + t.Helper() + for i := len(said) - 1; i >= 0; i-- { + if said[i].event == EventRetirement { + return said[i].body + } + } + t.Fatal("nothing about retirement was announced") + return nil +} + +func namesOf(body map[string]any) string { + var out []string + for _, c := range body["consumers"].([]map[string]any) { + out = append(out, c["consumer"].(string)) + } + return strings.Join(out, ",") +} + +// settle passes every five seconds until the same answer has held for StableFor, and one pass more. +func (w *world) settle() { w.passes(StableFor + 5*time.Second) } + +func testATransientEmptyListForFourPassesRetiresNothing(t *testing.T) { + w, said := standingWorld(t) + given := holding(w, 1) + w.give() + for i := 0; i < 4; i++ { + w.pass() + } + w.give(given...) + w.pass() + if len(w.a.removed) != 0 || len(retirements(*said)) != 0 { + t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said)) + } +} + +// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold: +// five passes AND ten minutes of the same answer (novox/hq ADR 0230). +func testFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) { + w, said := standingWorld(t) + holding(w, 1) + w.give() + for i := 0; i < 5; i++ { + w.pass() + } + if len(w.a.removed) != 0 || len(retirements(*said)) != 0 { + t.Fatalf("five passes in 25 s retired %v", w.a.removed) + } + w.passes(StableFor - 30*time.Second) + if len(w.a.removed) != 0 { + t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed) + } + w.passes(time.Minute) + if strings.Join(w.a.removed, ",") != "c1" { + t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed) + } + // Ten minutes in one slow pass are not five passes. + w2 := newWorld(t) + holding(w2, 1) + w2.give() + w2.pass() + w2.now = w2.now.Add(StableFor) + w2.pass() + if len(w2.a.removed) != 0 { + t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed) + } + w2.passes(15 * time.Second) + if len(w2.a.removed) != 1 { + t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed) + } +} + +func testAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) { + w, said := standingWorld(t) + given := holding(w, 3) + w.give(given[:2]...) + w.settle() + if strings.Join(w.a.removed, ",") != "c3" { + t.Fatalf("retired %v", w.a.removed) + } + body := lastRetirement(t, *said) + if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" || + !strings.Contains(body["why"].(string), "consecutive passes over 10m") { + t.Fatalf("%v", body) + } + if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" { + t.Fatalf("the backend does not hold it retired: %+v", w.a.inv) + } + // Asked for again: the ordinary create, on the first pass, and said. + created := len(w.a.created) + w.give(given...) + w.pass() + if len(w.a.created) != created+1 || w.a.created[created].As != "c3" { + t.Fatalf("not created again: %v", w.a.created) + } + if body := lastRetirement(t, *said); body["change"] != ChangeReenabled || namesOf(body) != "c3" { + t.Fatalf("%v", body) + } + if len(w.a.inv.Retired) != 0 { + t.Fatalf("still marked retired: %+v", w.a.inv.Retired) + } +} + +func testAnUnreadablePassStartsTheCountAgain(t *testing.T) { + w := newWorld(t) + holding(w, 1) + w.give() + w.passes(StableFor - time.Minute) + os.WriteFile(w.receives, []byte("{"), 0o600) + w.pass() + w.give() + w.passes(StableFor - time.Minute) + if len(w.a.removed) != 0 { + t.Fatalf("an unreadable pass counted: %v", w.a.removed) + } + w.passes(2 * time.Minute) + if len(w.a.removed) != 1 { + t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed) + } +} + +func testADifferentSetStartsTheCountAgain(t *testing.T) { + w := newWorld(t) + given := holding(w, 5) + w.give(given[:4]...) + w.passes(StableFor - time.Minute) + w.give(given[:3]...) + w.passes(StableFor - time.Minute) + if len(w.a.removed) != 0 { + t.Fatalf("a changed set kept its count: %v", w.a.removed) + } + w.passes(2 * time.Minute) + if strings.Join(w.a.removed, ",") != "c4,c5" { + t.Fatalf("%v", w.a.removed) + } +} + +func testAdditionsAndChangesAreNeverDelayed(t *testing.T) { + w := newWorld(t) + holding(w, 1) + w.give(map[string]any{"as": "c1", "node": "anchor"}, map[string]any{"as": "c2"}) + w.pass() + if len(w.a.created) != 2 || w.a.created[1].As != "c2" { + t.Fatalf("an addition waited: %v", w.a.created) + } + w.give(map[string]any{"as": "c1", "node": "anchor", "values": map[string]any{"name": "rotated"}}, map[string]any{"as": "c2"}) + w.pass() + if len(w.a.created) != 3 || w.a.created[2].As != "c1" { + t.Fatalf("a change waited: %v", w.a.created) + } +} + +func testTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) { + w, said := standingWorld(t) + holding(w, 4) + w.give() + w.passes(15 * time.Minute) + if len(w.a.removed) != 0 { + t.Fatalf("four of four were retired without a person: %v", w.a.removed) + } + if got := strings.Join(retirements(*said), ","); got != ChangeWaiting { + t.Fatalf("said %q, want one waiting", got) + } + body := lastRetirement(t, *said) + if namesOf(body) != "c1,c2,c3,c4" || body["held"] != 4 || body["bound"] == "" { + t.Fatalf("%v", body) + } + if !strings.Contains(strings.Join(w.said, "\n"), "RETIREMENT WAITS FOR A PERSON") { + t.Fatal("the wait was not said") + } + // Said again every quarter of an hour while it waits. + w.passes(11 * time.Minute) + if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" { + t.Fatalf("%q", got) + } + + if _, err := w.h.Approve(ctx, []string{"c1", "c2"}, "tidy", "operator", "mesh-controller"); err == nil { + t.Fatal("approved a set that is not the one waiting") + } + if _, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "", "operator", ""); err == nil { + t.Fatal("approved without a why") + } + done, err := w.h.Approve(ctx, []string{"c4", "c3", "c2", "c1"}, "the old machine is gone", "operator", "mesh-controller") + if err != nil || strings.Join(done, ",") != "c1,c2,c3,c4" || strings.Join(w.a.removed, ",") != "c1,c2,c3,c4" { + t.Fatal(done, err, w.a.removed) + } + changes := retirements(*said) + if strings.Join(changes[len(changes)-2:], ",") != "approved,retired" { + t.Fatalf("%v", changes) + } + if b := lastRetirement(t, *said); b["by"] != "operator" || b["via"] != "mesh-controller" || + !strings.Contains(b["why"].(string), "the old machine is gone") { + t.Fatalf("%v", b) + } + // Nothing more waits. + w.passes(time.Minute) + if r, _ := w.h.Retirement(ctx); r["waiting"] != nil || len(r["retired"].([]map[string]any)) != 4 { + t.Fatalf("%v", r) + } +} + +func testRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) { + w, said := standingWorld(t) + given := holding(w, 4) + w.give() + w.settle() + if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil { + t.Fatal("rejected a set that is not the one waiting") + } + kept, err := w.h.Reject([]string{"c1", "c2", "c3", "c4"}, "a person is moving them", "operator", "mesh-controller") + if err != nil || len(kept) != 4 { + t.Fatal(kept, err) + } + w.passes(time.Hour) + if len(w.a.removed) != 0 { + t.Fatalf("a rejected set was retired: %v", w.a.removed) + } + if got := strings.Join(retirements(*said), ","); got != "waiting,rejected" { + t.Fatalf("asked again after a rejection: %q", got) + } + r, _ := w.h.Retirement(ctx) + if r["rejected"] == nil || r["waiting"] != nil { + t.Fatalf("%v", r) + } + // One of them asked for again: a different answer, so the rejection is settled and counting + // starts over — three of four is over the bound again, and waits again. + w.give(given[0]) + w.pass() + if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" { + t.Fatalf("%q", got) + } + w.settle() + if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" { + t.Fatalf("%q", got) + } + // A rejected set can still be approved later. + w2, _ := standingWorld(t) + holding(w2, 4) + w2.give() + w2.settle() + w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "") + if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 { + t.Fatal(done, err) + } +} + +func testAWaitingSetAskedForAgainSettles(t *testing.T) { + w, said := standingWorld(t) + given := holding(w, 4) + w.give() + w.settle() + w.give(given...) + w.pass() + if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 { + t.Fatalf("%q %v", got, w.a.removed) + } + if _, err := w.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "late", "operator", ""); err == nil { + t.Fatal("approved a set that no longer waits") + } +} + +func testDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) { + w, said := standingWorld(t) + given := holding(w, 5) + w.give(given[:3]...) + w.settle() + if strings.Join(w.a.removed, ",") != "c4,c5" { + t.Fatalf("%v", w.a.removed) + } + if _, err := w.h.DeleteRetired(ctx, "c1", "tidy", "operator", ""); err == nil { + t.Fatal("deleted an active consumer") + } + if _, err := w.h.DeleteRetired(ctx, "c5", "", "operator", ""); err == nil { + t.Fatal("deleted without a why") + } + if _, err := w.h.DeleteRetired(ctx, "nobody", "tidy", "operator", ""); err == nil { + t.Fatal("deleted something not retired") + } + freed, err := w.h.DeleteRetired(ctx, "c5", "the experiment is over", "operator", "mesh-controller") + if err != nil || freed != 42 || strings.Join(w.a.deleted, ",") != "c5" { + t.Fatal(freed, err, w.a.deleted) + } + if b := lastRetirement(t, *said); b["change"] != ChangeDeleted || namesOf(b) != "c5" || b["freed_bytes"] != int64(42) { + t.Fatalf("%v", b) + } + r, _ := w.h.Retirement(ctx) + if left := r["retired"].([]map[string]any); len(left) != 1 || left[0]["consumer"] != "c4" { + t.Fatalf("%v", r) + } + // Retired and asked for again before anybody deleted it: refused, it is the mesh's again. + w.give(given[:4]...) + w.pass() + if _, err := w.h.DeleteRetired(ctx, "c4", "tidy", "operator", ""); err == nil { + t.Fatal("deleted a consumer the mesh asks for") + } +} + +func testTheBound(t *testing.T) { + h := &Harness{} + h.init() + for _, c := range []struct{ n, held int }{{1, 1}, {1, 2}, {1, 3}, {3, 7}, {3, 6}, {2, 5}} { + if h.overTheBound(c.n, c.held) { + t.Errorf("%d of %d waits for a person", c.n, c.held) + } + } + for _, c := range []struct{ n, held int }{{2, 2}, {2, 3}, {4, 7}, {4, 10}, {7, 7}} { + if !h.overTheBound(c.n, c.held) { + t.Errorf("%d of %d is retired without a person", c.n, c.held) + } + } +} + +func testARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t *testing.T) { + w, said := standingWorld(t) + w.a.inv = Inventory{Active: []string{"kept", "orphan"}, Retired: []Retired{ + {Consumer: "locked-before", SizeBytes: 7, Kind: KindConsumer}, + {Consumer: "old_deleted_20261005", RetiredAt: time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC), Kind: "set-aside-database"}, + }} + w.give(map[string]any{"as": "kept"}) + w.h.Reconcile(ctx) + if b := lastRetirement(t, *said); b["change"] != ChangeAdopted || namesOf(b) != "locked-before" { + t.Fatalf("%v", b) + } + if strings.Join(w.a.removed, ",") != "locked-before" { + t.Fatalf("adopting did not mark it: %v", w.a.removed) + } + w.settle() + if strings.Join(w.a.removed, ",") != "locked-before,orphan" { + t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed) + } +} + +func testABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) { + w := newWorld(t) + w.a.invErr = errors.New("connection refused") + holding(w, 1) + if !strings.Contains(strings.Join(w.said, "\n"), "cannot list what the backend holds") { + t.Fatal(w.said) + } + w.a.invErr = nil + w.a.inv = Inventory{Active: []string{"c1", "orphan"}} + w.pass() + w.settle() + if strings.Join(w.a.removed, ",") != "orphan" { + t.Fatalf("%v", w.a.removed) + } +} + +func testTheToolsAnswer(t *testing.T) { + w, _ := standingWorld(t) + holding(w, 4) + w.give() + w.settle() + tools := map[string]func(map[string]any) (any, error){} + for _, tool := range RetirementTools(w.h) { + tools[tool.Name] = tool.Run + // L3: the delete, approve and reject name no removed controller verb, and say where the operator answers (ADR 0604 §8). + if tool.Name != "provisioner_retirement" && (strings.Contains(tool.Description, "controller") || !strings.Contains(tool.Description, "operator channel")) { + t.Errorf("%s says %q", tool.Name, tool.Description) + } + } + if len(tools) != 4 { + t.Fatalf("%d tools", len(tools)) + } + got, err := tools["provisioner_retirement"](nil) + if err != nil || got.(map[string]any)["waiting"] == nil { + t.Fatal(got, err) + } + set := []any{"c1", "c2", "c3", "c4"} + if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set}); err == nil { + t.Fatal("approved without a why") + } + if _, err := tools["provisioner_retire_approve"](map[string]any{"consumers": set, "why": "gone", "by": "operator"}); err != nil { + t.Fatal(err) + } + if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "why": "gone"}); err == nil { + t.Fatal("deleted without confirm") + } + if _, err := tools["provisioner_delete"](map[string]any{"consumer": "c1", "confirm": "c1", "why": "gone"}); err != nil { + t.Fatal(err) + } + if strings.Join(w.a.deleted, ",") != "c1" { + t.Fatal(w.a.deleted) + } +} + +func testAFailingConsumerRetiredIsSaidRecovered(t *testing.T) { + w, said := standingWorld(t) + given := holding(w, 2) + w.a.held = false + w.a.failing = errors.New("boom") + w.passes(7 * time.Minute) + w.give(given[0]) + w.settle() + recovered := false + for _, a := range *said { + if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" { + recovered = true + } + } + if !recovered { + t.Fatalf("%v", *said) + } +} + +// everyWord is a world that keeps every word the provider announces. +func everyWord(t *testing.T) (*world, *[]announced) { + w := newWorld(t) + var said []announced + w.h.Announce = func(e string, b map[string]any) { said = append(said, announced{e, b}) } + w.h.Node = "anchor" + return w, &said +} + +// A restarted provider that finds every consumer it holds asked for says once that nothing waits, so the controller +// clears a waiting condition an earlier process raised and this one never knew (issue 937); it says it no more. +func testARestartedProviderSaysOnceThatNothingWaits(t *testing.T) { + w, said := everyWord(t) + holding(w, 2) + w.pass() + w.pass() + var words []announced + for _, a := range *said { + if a.event == EventRetirement { + words = append(words, a) + } + } + if len(words) != 1 || !noneWaits(words[0]) || words[0].body["provider-node"] != w.h.Node { + t.Fatalf("said %+v, want one settled with no set", words) + } +} + +// A provider whose own set waits does not say that nothing waits. +func testAProviderThatWaitsDoesNotSayNothingWaits(t *testing.T) { + w, said := everyWord(t) + w.a.inv.Active = []string{"c1", "c2", "c3", "c4", "c5"} + w.give(map[string]any{"as": "c1", "node": "anchor"}) + w.settle() + for _, a := range *said { + if a.event == EventRetirement && noneWaits(a) { + t.Fatalf("said nothing waits while %v waited", w.h.r.waiting) + } + } + if w.h.r.waiting == nil { + t.Fatal("nothing waited: the test does not hold the case") + } +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/standing_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/standing_test.go new file mode 100644 index 0000000..52fd137 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/standing_test.go @@ -0,0 +1,193 @@ +package harness + +// A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224): not on the first +// failure, which may be a restart; after FailingAfter of failures with no success between; again +// every SayAgainEvery while it lasts; and recovered on the first success, or when the consumer goes. + +import ( + "errors" + "os" + "strings" + "testing" + "time" +) + +type announced struct { + event string + body map[string]any +} + +func standingWorld(t *testing.T) (*world, *[]announced) { + w := newWorld(t) + var said []announced + w.h.Announce = func(e string, b map[string]any) { + // The first success since start, and the first word that nothing waits (issue 937), are their own + // tests'; every other test reads past them. + if c, _ := b["consumers"].([]map[string]any); b["why"] != "first-success" && + !(e == EventRetirement && b["change"] == ChangeSettled && len(c) == 0) { + said = append(said, announced{e, b}) + } + } + w.h.Node = "anchor" + return w, &said +} + +// passes reconciles every five seconds for d, as Run would. +func (w *world) passes(d time.Duration) { + for end := w.now.Add(d); w.now.Before(end); w.now = w.now.Add(5 * time.Second) { + w.h.Reconcile(ctx) + } +} + +func events(said []announced) string { + var out []string + for _, a := range said { + out = append(out, a.event) + } + return strings.Join(out, ",") +} + +func testAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass(t *testing.T) { + w, said := standingWorld(t) + w.a.failing = errors.New(`token request failed: 401 {"error":"invalid_grant","error_description":"Invalid user credentials"}`) + w.give(map[string]any{"as": "mesh_home_grafana", "node": "home-server"}) + + w.passes(4 * time.Minute) + if len(*said) != 0 { + t.Fatalf("announced before FailingAfter: %v", events(*said)) + } + w.passes(2 * time.Minute) + if events(*said) != EventFailing { + t.Fatalf("want one %s, got %q", EventFailing, events(*said)) + } + b := (*said)[0].body + if b["consumer"] != "mesh_home_grafana" || b["node"] != "home-server" || b["class"] != ClassCredentials || + b["provider"] != "postgres-database" || b["provider-node"] != "anchor" || b["attempts"].(int) < 60 { + t.Fatalf("%v", b) + } + + // Said again while it lasts, not every pass. + w.passes(14 * time.Minute) + if events(*said) != EventFailing { + t.Fatalf("repeated too soon: %q", events(*said)) + } + w.passes(2 * time.Minute) + if events(*said) != EventFailing+","+EventFailing { + t.Fatalf("not repeated: %q", events(*said)) + } + + // The first success takes it back. + w.a.failing = nil + w.passes(5 * time.Second) + if events(*said) != EventFailing+","+EventFailing+","+EventRecovered { + t.Fatalf("no recovery: %q", events(*said)) + } + if (*said)[2].body["consumer"] != "mesh_home_grafana" { + t.Fatal((*said)[2].body) + } +} + +func testOneSuccessBetweenFailuresStartsTheRunAgain(t *testing.T) { + w, said := standingWorld(t) + w.a.failing = errors.New("connection refused") + w.give(map[string]any{"as": "a"}) + w.passes(4 * time.Minute) + w.a.failing = nil + w.passes(5 * time.Second) + w.give(map[string]any{"as": "a", "values": map[string]any{"name": "changed"}}) + w.a.failing = errors.New("connection refused") + w.passes(4 * time.Minute) + if len(*said) != 0 { + t.Fatalf("two runs of four minutes are not one of eight: %q", events(*said)) + } +} + +// The check that failed for a day on 2026-10-05: clients already made, every minute's check refused +// at the token. A check that cannot be asked is a failure too. +func testACheckThatKeepsFailingIsAFailureToo(t *testing.T) { + w, said := standingWorld(t) + w.give(map[string]any{"as": "a"}) + w.h.Reconcile(ctx) + w.a.holdsErr = errors.New("401 invalid_grant") + w.passes(7 * time.Minute) + if events(*said) != EventFailing || (*said)[0].body["class"] != ClassCredentials { + t.Fatalf("%q %v", events(*said), *said) + } + w.a.holdsErr = nil + w.passes(time.Minute + 5*time.Second) + if events(*said) != EventFailing+","+EventRecovered { + t.Fatalf("%q", events(*said)) + } +} + +func testAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) { + w, said := standingWorld(t) + w.give(map[string]any{"as": "a"}) + os.Remove(w.dir + "/a.secret") + w.passes(6 * time.Minute) + if events(*said) != EventFailing || (*said)[0].body["class"] != ClassSecret { + t.Fatalf("%q %v", events(*said), *said) + } +} + +func testAConsumerNoLongerAskedForIsNoLongerFailing(t *testing.T) { + w, said := standingWorld(t) + w.a.failing = errors.New("boom") + w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}) + w.passes(6 * time.Minute) + if events(*said) != EventFailing+","+EventFailing { + t.Fatalf("%q", events(*said)) + } + w.give(map[string]any{"as": "a"}) + w.passes(5 * time.Second) + last := (*said)[len(*said)-1] + if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "no longer asked for" { + t.Fatalf("%v", *said) + } +} + +func testAnErrorIsClassedByItsWords(t *testing.T) { + for text, want := range map[string]string{ + `Keycloak token request failed: 401 {"error":"invalid_grant"}`: ClassCredentials, + `FATAL: password authentication failed for user "postgres"`: ClassCredentials, + `dial tcp 127.0.0.1:5432: connect: connection refused`: ClassUnreachable, + `context deadline exceeded`: ClassUnreachable, + `extension "nope" is not available`: ClassRefused, + } { + if got := ClassOf(text); got != want { + t.Errorf("%s: %s, want %s", text, got, want) + } + } +} + +func testAnAdapterThatClassesItsOwnErrorsIsBelieved(t *testing.T) { + w, said := standingWorld(t) + w.h.Adapter = classing{w.a} + w.a.failing = errors.New("anything") + w.give(map[string]any{"as": "a"}) + w.passes(6 * time.Minute) + if (*said)[0].body["class"] != "its-own" { + t.Fatal((*said)[0].body) + } +} + +type classing struct{ *recorder } + +func (classing) Class(error) string { return "its-own" } + +// A provider restarted after announcing a failure has forgotten it; its first success for each +// consumer is announced, so the controller clears what it kept rather than naming it for ever. +func testTheFirstSuccessSinceStartIsAnnouncedOnce(t *testing.T) { + w := newWorld(t) + var said []announced + w.h.Announce = func(e string, b map[string]any) { + if e != EventRetirement { + said = append(said, announced{e, b}) + } + } + w.give(map[string]any{"as": "a"}) + w.passes(3 * time.Minute) + if events(said) != EventRecovered || said[0].body["why"] != "first-success" || said[0].body["consumer"] != "a" { + t.Fatalf("%v", said) + } +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/harness/tables_test.go b/provisioning/go/clients/meshprovisionerclient/provider/harness/tables_test.go new file mode 100644 index 0000000..988396f --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/harness/tables_test.go @@ -0,0 +1,114 @@ +package harness + +import ( + "context" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient/provider" +) + +// TestHarnessRuns groups the cases below, each one a subtest. +func TestHarnessRuns(t *testing.T) { + // TestHarness holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests). + t.Run("Harness", func(t *testing.T) { + t.Run("AConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword", testAConsumerIsCreatedOnceUnderTheMeshsLoginAndPassword) + t.Run("AConsumerNoLongerAskedForIsRetiredOnceStable", testAConsumerNoLongerAskedForIsRetiredOnceStable) + t.Run("NothingReadIsNotNobodyAsking", testNothingReadIsNotNobodyAsking) + t.Run("ALostConsumerIsAppliedAgainAndBraked", testALostConsumerIsAppliedAgainAndBraked) + t.Run("AFailingCreateIsRetriedAndSaidOnce", testAFailingCreateIsRetriedAndSaidOnce) + t.Run("ScrubRemovesThePassword", testScrubRemovesThePassword) + }) + + t.Run("AMovedPathStartedTooEarlyAsksAgainUntilTheRuntimeAnswers", func(t *testing.T) { + w := newMovedWorld(t, map[[2]string]string{}) + w.run.FirstRetry = time.Millisecond + w.run.loaded = false + bus := &earlyBus{bucketBus: w.bus, refuse: 3} + p := provider.New(provider.Options{Node: "novox", Module: "postgres", Bus: w.bus, + Vault: provider.ConsumersFile{Paths: []string{w.file}}, Make: w.run.Make, Gone: w.run.Gone, + Every: time.Hour, Logf: func(string, ...any) {}}) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- w.run.Run(ctx, p, bus) }() + deadline := time.After(time.Second) + for { + w.run.mu.Lock() + loaded := w.run.loaded + w.run.mu.Unlock() + if loaded { + break + } + select { + case err := <-done: + cancel() + t.Fatalf("Run stopped before the runtime answered: %v", err) + case <-deadline: + cancel() + t.Fatal("the plans were never watched") + case <-time.After(time.Millisecond): + } + } + cancel() + <-done + if bus.refuse != 0 { + t.Fatalf("%d refusals left", bus.refuse) + } + }) + + // TestStanding holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests). + t.Run("Standing", func(t *testing.T) { + t.Run("AConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass", testAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass) + t.Run("OneSuccessBetweenFailuresStartsTheRunAgain", testOneSuccessBetweenFailuresStartsTheRunAgain) + t.Run("ACheckThatKeepsFailingIsAFailureToo", testACheckThatKeepsFailingIsAFailureToo) + t.Run("AnUnreadableSecretIsAnnouncedAsSuch", testAnUnreadableSecretIsAnnouncedAsSuch) + t.Run("AConsumerNoLongerAskedForIsNoLongerFailing", testAConsumerNoLongerAskedForIsNoLongerFailing) + t.Run("AnErrorIsClassedByItsWords", testAnErrorIsClassedByItsWords) + t.Run("AnAdapterThatClassesItsOwnErrorsIsBelieved", testAnAdapterThatClassesItsOwnErrorsIsBelieved) + t.Run("TheFirstSuccessSinceStartIsAnnouncedOnce", testTheFirstSuccessSinceStartIsAnnouncedOnce) + }) +} + +// TestHarnessMoves groups the cases below, each one a subtest. +func TestHarnessMoves(t *testing.T) { + // TestMoved holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests). + t.Run("Moved", func(t *testing.T) { + t.Run("TheConsumersFileMakesWhatTheGrantFilesMake", testTheConsumersFileMakesWhatTheGrantFilesMake) + t.Run("AConsumerMissingFromOneReadRetiresNothing", testAConsumerMissingFromOneReadRetiresNothing) + t.Run("NoPlansReadRetiresNothing", testNoPlansReadRetiresNothing) + t.Run("AConsumerWithoutACredentialIsWaitedFor", testAConsumerWithoutACredentialIsWaitedFor) + t.Run("ASluggedConsumerKeepsTheLoginItsPlanNames", testASluggedConsumerKeepsTheLoginItsPlanNames) + t.Run("ASpecWithoutGrantsMakesUnderThePlansAsAndIgnoresTheGrantFile", testASpecWithoutGrantsMakesUnderThePlansAsAndIgnoresTheGrantFile) + t.Run("TheConsumersFileMakesWhatTheGrantFileMakesForTheKeycloakShape", testTheConsumersFileMakesWhatTheGrantFileMakesForTheKeycloakShape) + t.Run("AConsumerNamedByItsPlanKeepsItsClientId", testAConsumerNamedByItsPlanKeepsItsClientId) + }) + + // TestGrantsMoved holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests). + t.Run("GrantsMoved", func(t *testing.T) { + t.Run("AConsumerOnlyTheGrantFileGivesIsAskedForAndMade", testAConsumerOnlyTheGrantFileGivesIsAskedForAndMade) + t.Run("ALoginAPlanNamesIsNotMadeFromTheGrantFile", testALoginAPlanNamesIsNotMadeFromTheGrantFile) + t.Run("AnUnreadableGrantFileRetiresNothing", testAnUnreadableGrantFileRetiresNothing) + t.Run("ALoginTheGrantFileStopsGivingIsRetired", testALoginTheGrantFileStopsGivingIsRetired) + }) + + // TestRetirement holds this file's cases, each a subtest of its own name (mesh-sdk's test budget: at most 500 unit tests). + t.Run("Retirement", func(t *testing.T) { + t.Run("ATransientEmptyListForFourPassesRetiresNothing", testATransientEmptyListForFourPassesRetiresNothing) + t.Run("FivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo", testFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo) + t.Run("AStableSetIsRetiredAndAskedAgainReEnables", testAStableSetIsRetiredAndAskedAgainReEnables) + t.Run("AnUnreadablePassStartsTheCountAgain", testAnUnreadablePassStartsTheCountAgain) + t.Run("ADifferentSetStartsTheCountAgain", testADifferentSetStartsTheCountAgain) + t.Run("AdditionsAndChangesAreNeverDelayed", testAdditionsAndChangesAreNeverDelayed) + t.Run("TooManyWaitsForAPersonAndApproveRetiresExactlyThatSet", testTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet) + t.Run("RejectedIsKeptAndNotAskedAgainUntilTheSetChanges", testRejectedIsKeptAndNotAskedAgainUntilTheSetChanges) + t.Run("AWaitingSetAskedForAgainSettles", testAWaitingSetAskedForAgainSettles) + t.Run("DeleteRemovesOnlyThatRetiredConsumer", testDeleteRemovesOnlyThatRetiredConsumer) + t.Run("TheBound", testTheBound) + t.Run("ARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled", testARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled) + t.Run("ABackendThatCannotBeListedIsSaidAndAskedAgain", testABackendThatCannotBeListedIsSaidAndAskedAgain) + t.Run("TheToolsAnswer", testTheToolsAnswer) + t.Run("AFailingConsumerRetiredIsSaidRecovered", testAFailingConsumerRetiredIsSaidRecovered) + t.Run("ARestartedProviderSaysOnceThatNothingWaits", testARestartedProviderSaysOnceThatNothingWaits) + t.Run("AProviderThatWaitsDoesNotSayNothingWaits", testAProviderThatWaitsDoesNotSayNothingWaits) + }) +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/provider.go b/provisioning/go/clients/meshprovisionerclient/provider/provider.go new file mode 100644 index 0000000..d3c0e82 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/provider.go @@ -0,0 +1,858 @@ +// Package provider is a providing module's half of a binding: it answers the provider's consumers itself, over the +// bus (novox/hq ADR 0349 clause 12, issue 678): the provider module, not the controller, makes each bound +// consumer's entry. +// +// **Who binds the provider.** Each node's provisioner client records the needs of the modules handed to it from +// their plans, and the controller names in each plan, per required provision and local name, the provider that +// resolution chose: `provisioning.provisions` in `.` of the assignments mirror (issue 760). A recorded +// need names no provider (issue 677's note 7), so the plan's entry is where a bind is seen, as the vault sees it +// (mesh-vault pair.go). A consumer binds this provider when an entry names this machine and this module, and +// unbinds when the entry, or the assignment, goes. A consumer still filled the old way has no plan and is not seen +// here; the controller answers it as before (ADR 0349 clause 17). +// +// **One entry per consumer, in the provider's own bucket.** For each bind the helper composes the consumer's +// served values from the provider's offer (its config already composed into it, ADR 0202 §7) and the consumer's +// identity (`${consumer:as}`, `${consumer:as:dns}`, ADR 0202 §1), and writes one entry, `.. +// [.]` of `_bound`: the one statement both ends read. It never carries a +// secret. It names the credential the vault keeps for the pair, which the consumer's client asks the vault for. +// +// **The credential is the vault's** (ADR 0349 clause 8). The helper mints nothing and reads nothing of the +// provider's own secrets: it reads the value from the vault and hands it to the provider's adapter, which makes the +// login its software keeps. An offer whose software holds one credential (ADR 0158) is the vault's one value per +// provider and provision, `provision//`, with the provider's one user name; an offer with +// a login per consumer (ADR 0048, ADR 0158 §11, the default) is each consumer's own value, +// `pair////[/]`, with the consumer's identity as its login (issue +// 677's notes 12 and 13). A credential the vault does not hold yet is waited for: nothing is made and no entry is +// written, so a consumer is never told of a login its provider has not made. A provider in the tool runner never asks +// the vault itself: its Vault is ConsumersFile, the file its machine's provisioner client fills (ADR 0373). +// +// **A multi-holder consumer gets no derived value** (ADR 0202 §10 as ADR 0349 clause 12 changes it): a consumer +// bound under a local name keeps several holders of the provision, each with its own login, while what the +// provider derives names one; its entry refuses it, naming both ends, and its client reports it. +// +// The helper speaks to the bus and the vault through the two interfaces below, so a provider wires it to its +// runtime and a test to a stand-in. Standard library only. +package provider + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "sync" + "time" + + provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// BucketOf is a providing module's own bucket of entries, one per consumer: `_bound`, the module's state +// `bound`. +func BucketOf(module string) string { return module + "_bound" } + +// Bus is what the helper needs of the bus: a key-value watch and the provider's writes to its own bucket. +type Bus interface { + // Watch hands handle every current key and value of bucket, then each change as it comes, a removed key with a + // nil value; it returns once the current values are handed, and the watch lasts until ctx ends. + Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error + Put(ctx context.Context, bucket, key string, value []byte) error + Delete(ctx context.Context, bucket, key string) error +} + +// ConditionalBus is a Bus that can also delete a key only while it still holds what the caller read. The helper uses +// it when the Bus offers it, so a pass that lets a consumer go never removes an entry another instance of the +// provider wrote after the pass read it; a Bus without it deletes unconditionally. DeleteIf returns an error that +// is stdio.ErrChanged when the key holds something else now. +type ConditionalBus interface { + Bus + DeleteIf(ctx context.Context, bucket, key string, held []byte) error +} + +// Vault is how the provider reads a credential the vault keeps, by its name, opened. +type Vault interface { + Value(ctx context.Context, name string) ([]byte, error) +} + +// CredentialForm is which credential an offer's consumers present. +type CredentialForm int + +const ( + // PerConsumer is a login of each consumer's own (ADR 0048, ADR 0158 §11): the default. + PerConsumer CredentialForm = iota + // Shared is the one credential the provider's software holds, shared with every consumer (ADR 0158). + Shared + // NoCredential is an offer whose consumers present none. + NoCredential +) + +// Offer is what the provider serves one provision with. +type Offer struct { + // Serves are the values served to each consumer, the provider's config already composed into them; a string + // may name the consumer's identity, `${consumer:as}` or `${consumer:as:dns}`. + Serves map[string]any + // Credential is the form of the credential its consumers present. A consumer's plan entry that names a + // credential form of its own wins over it (the plan's `credential-form`). + Credential CredentialForm + // User is the provider's one user name, for a Shared credential. + User string +} + +// Entry is one consumer's entry in the provider's bucket: the one statement both ends read. Never a secret. +type Entry struct { + Provider string `json:"provider"` + ProviderModule string `json:"provider-module"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + // As is the consumer's identity, the mesh's (ADR 0202 §1). + As string `json:"as,omitempty"` + // User is the login the consumer presents: its identity, or the provider's one user name for a shared credential. + User string `json:"user,omitempty"` + // Values are the served values, composed for this consumer: what `${bound::}` reads. + Values map[string]any `json:"values,omitempty"` + // VaultName is the name under which the vault keeps this consumer's credential, which the consumer's client asks + // the vault for: a name, never the credential. Its key is not "credential" because the runtime's state guard + // refuses any key ending in that word, whatever the value (issue 945). + VaultName string `json:"vault-name,omitempty"` + // Refused, when set, is why the consumer is not served, naming both ends. + Refused string `json:"refused,omitempty"` + // State is the provider's outcome for the consumer, which the holder of mesh-provisioner reads to set the + // provision's key (novox/hq ADR 0570 clause 3): provided, failed with Why, or waits with what is awaited. Empty from + // a provider before it. + State string `json:"state,omitempty"` + Why string `json:"why,omitempty"` + // Contributes is what the consumer's module tells its provider about this provision (a database's name, its + // extensions), from its plan's entry (issue 776). Handed to Make and Gone; never written to the bucket. + Contributes map[string]any `json:"-"` + // At and From are where the consumer's plan says the provider is and which machine it is (issue 788): the values + // today's binding file gives that consumer. Written into Values as `at` and `from`; never alone to the bucket. + At, From string `json:"-"` + // Names are the names the controller composed for this consumer on this provision (issue 792): its own name, its + // internal name, and one of each per local name it contributes under, keyed as its binding file names them. + // Written into Values over anything the provider serves under the key; never alone to the bucket. + Names map[string]string `json:"-"` + // CredentialForm and Name are from the consumer's plan entry (issue 825): the form of the credential the + // controller chose for this bind (`shared`, `pair`, `issued` or `given`) and the holder's name. When set, they + // win over the offer's form. Never written to the bucket. + CredentialForm, Name string `json:"-"` +} + +// Key is the entry's key in the provider's bucket: `..`, and `.` for a bind under a +// local name. +func (e Entry) Key() string { + k := e.Consumer + "." + e.Module + "." + e.Provision + if e.Local != "" { + k += "." + e.Local + } + return k +} + +// Consumer is what the provider's adapter makes: the consumer's entry and the credential the vault keeps for it. +type Consumer struct { + Entry Entry + Credential []byte +} + +// Options are the provider's. +type Options struct { + // Node and Module are this provider: the machine it runs on and the module serving. + Node, Module string + // Offers are the provisions it serves, by name. + Offers map[string]Offer + Bus Bus + Vault Vault + // Make brings a consumer into being in the provider's software under the vault's credential. Asked on every + // pass for every consumer answered, so it must leave one that is already as asked alone; a rotated credential + // reaches the software this way. + Make func(ctx context.Context, c Consumer) error + // Gone, when set, hears that a consumer no longer binds the provider, before its entry is removed; an error + // keeps the entry, and it is heard again on the next pass. Retiring what was kept for it is the provider's + // (ADR 0230). + Gone func(ctx context.Context, e Entry) error + // Identity is the consumer's identity for an entry whose plan names none; ConsumerIdentity of the node and the + // module when not set. + Identity func(node, module string) string + // Every is how often a pass runs without a change watched; a minute when not set. + Every time.Duration + // Logf says what a pass did that differs from the last; the standard log when not set. + Logf func(format string, args ...any) +} + +// Provider answers its consumers. +type Provider struct { + opts Options + + mu sync.Mutex + entries map[string][]byte + // plans and provisions are the seat mesh-provisioner's states (ADR 0570 clauses 1, 2), by key. + plans map[string][]byte + provisions map[string][]byte + said map[string]string + poke chan struct{} +} + +// New is a provider answering its consumers as opts say. +func New(opts Options) *Provider { + if opts.Identity == nil { + opts.Identity = ConsumerIdentity + } + if opts.Every <= 0 { + opts.Every = time.Minute + } + if opts.Logf == nil { + opts.Logf = func(f string, a ...any) { fmt.Printf("[provider] "+f+"\n", a...) } + } + return &Provider{opts: opts, entries: map[string][]byte{}, + plans: map[string][]byte{}, provisions: map[string][]byte{}, said: map[string]string{}, poke: make(chan struct{}, 1)} +} + +// ErrNotReady is what a Make returns, wrapped or alone, when the provider cannot make a consumer's provision yet but +// will without anyone's hand (its keys not read since a restart): the entry waits and keeps serving what it served. +// Any other error is a failure, and its entry serves nothing. +var ErrNotReady = errors.New("not ready yet") + +// What a pass did for one bind. +const ( + Answered = "answered" + Refused = "refused" + Waiting = "waiting" + Failed = "failed" + Removed = "removed" + // Superseded is an entry no bind asks for any more that another instance of the provider wrote after this pass + // read it: it is left standing. + Superseded = "superseded" +) + +// Outcome is what a pass did for one consumer's entry. +type Outcome struct { + Key string `json:"key"` + Status string `json:"status"` + Why string `json:"why,omitempty"` +} + +// Run watches the binds and the provider's own entries, and passes once at start, on every change and every +// Every, until ctx ends. +func (p *Provider) Run(ctx context.Context) error { + if err := p.watch(ctx, BucketOf(p.opts.Module), p.entries); err != nil { + return fmt.Errorf("watching %s: %w", BucketOf(p.opts.Module), err) + } + for _, bucket := range []string{provisions.PlansBucket, provisions.ProvisionsBucket} { + into := p.plans + if bucket == provisions.ProvisionsBucket { + into = p.provisions + } + if err := p.watch(ctx, bucket, into); err != nil { + return fmt.Errorf("watching %s: %w", bucket, err) + } + } + tick := time.NewTicker(p.opts.Every) + defer tick.Stop() + for { + p.mu.Lock() + in := Inputs{Entries: clone(p.entries), Plans: clone(p.plans), Provisions: clone(p.provisions)} + p.mu.Unlock() + if _, err := p.passIn(ctx, in); err != nil { + p.say("pass", err.Error()) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-p.poke: + case <-tick.C: + } + } +} + +func (p *Provider) watch(ctx context.Context, bucket string, into map[string][]byte) error { + return p.opts.Bus.Watch(ctx, bucket, func(key string, value []byte) { + p.mu.Lock() + if value == nil { + delete(into, key) + } else { + into[key] = append([]byte(nil), value...) + } + p.mu.Unlock() + if bucket != BucketOf(p.opts.Module) { + select { + case p.poke <- struct{}{}: + default: + } + } + }) +} + +// Pass reads the binds and the provider's entries from the bus and answers once: each bind of this provider gets +// its entry, and each entry of this provider no bind asks for is removed. An assignment that cannot be read, or that +// is not the one its key names, is skipped and said (an outcome of status Failed under the assignment's key, and a +// line of the log), and the entries of its consumer are left as they are, since a bind missed would read as an +// unbind; the pass goes on for the rest. Run passes by itself; Pass is for a provider that passes when it chooses. +// +// Over StateBus each call to Pass leaves two watches open for the life of the process, silenced once the call has +// ended (see StateBus): a provider that passes often should use Run, which watches once. +func (p *Provider) Pass(ctx context.Context) ([]Outcome, error) { + watchCtx, cancel := context.WithCancel(ctx) + defer cancel() + var mu sync.Mutex + read := func(bucket string) (map[string][]byte, error) { + got := map[string][]byte{} + err := p.opts.Bus.Watch(watchCtx, bucket, func(key string, value []byte) { + mu.Lock() + defer mu.Unlock() + if value == nil { + delete(got, key) + } else { + got[key] = append([]byte(nil), value...) + } + }) + mu.Lock() + defer mu.Unlock() + return clone(got), err + } + entries, err := read(BucketOf(p.opts.Module)) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", BucketOf(p.opts.Module), err) + } + plans, err := read(provisions.PlansBucket) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", provisions.PlansBucket, err) + } + provs, err := read(provisions.ProvisionsBucket) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", provisions.ProvisionsBucket, err) + } + return p.passIn(ctx, Inputs{Entries: entries, Plans: plans, Provisions: provs}) +} + +// Inputs are what one pass reads: the provider's own entries, and the seat mesh-provisioner's plans and provisions +// (novox/hq ADR 0570 clauses 1, 2), each by key. The controller's assignments are read no more (clause 2). +type Inputs struct { + Entries, Plans, Provisions map[string][]byte +} + +func (p *Provider) passIn(ctx context.Context, in Inputs) ([]Outcome, error) { + entries := in.Entries + var binds []Entry + var out []Outcome + unknown := map[string]bool{} // consumers whose assignment was skipped: their entries are not judged + // Every consumer is bound by its plan and its provision keys (ADR 0570 clauses 2, 3), and an entry no plan names is + // never let go by the provider itself (clause 5): only a retiring key with the operator's yes retires it. + seat := map[string]bool{} + gated := map[string]provisions.Provision{} + for _, key := range sortedKeys(in.Plans) { + plan, err := provisions.DecodePlan(in.Plans[key]) + if err != nil || plan.Node+"."+plan.Module != key { + o := Outcome{Key: key, Status: Failed, Why: fmt.Sprintf("the plan %s is not readable, so who it binds is not "+ + "known; it is skipped, and its entries are left as they are", key)} + out = append(out, o) + p.say(key, o.Status+" "+o.Why) + unknown[key], seat[key] = true, true + continue + } + seat[key] = true + for _, b := range plan.Plan.Provisions { + if b.Provider != p.opts.Node || b.ProviderModule != p.opts.Module { + continue + } + e := Entry{Provider: p.opts.Node, ProviderModule: p.opts.Module, Provision: b.Provision, Consumer: plan.Node, + Module: plan.Module, Local: b.Local, Contributes: b.Contributes, At: b.At, From: b.From, As: b.As, + Names: b.Names, CredentialForm: b.CredentialForm, Name: b.Name} + binds = append(binds, e) + } + } + for _, key := range sortedKeys(in.Provisions) { + if pr, err := provisions.Decode(in.Provisions[key]); err == nil && pr.Provider.Node == p.opts.Node && + pr.Provider.Module == p.opts.Module { + gated[key] = pr + } + } + // **A key no plan names is bound from the entry it carries** (issue 1126): the seat keys a consumer's provisions + // from its wanted and its newest builds, and keeps the plan at its wanted build alone, so a requirement the newest + // build adds has a key before any plan names it, and the want that waits for it waits on this provider. The plan + // wins where both say it. A key with no entry is said, never left without a word. + planned := map[string]bool{} + for _, b := range binds { + planned[b.Key()] = true + } + for _, key := range sortedKeys(gated) { + pr := gated[key] + if planned[key] || pr.State == provisions.Retiring || unknown[consumerOf(key)] { + continue + } + if pr.Plan == nil { + o := Outcome{Key: key, Status: Waiting, Why: "asked, but no plan names it and its key carries no plan entry, " + + "so whom it binds is not known; mesh-provisioner composes the entry"} + out = append(out, o) + p.say(key, o.Status+" "+o.Why) + continue + } + b := *pr.Plan + binds = append(binds, Entry{Provider: p.opts.Node, ProviderModule: p.opts.Module, Provision: pr.Provision, + Consumer: pr.Consumer.Node, Module: pr.Consumer.Module, Local: pr.Local, Contributes: b.Contributes, At: b.At, + From: b.From, As: b.As, Names: b.Names, CredentialForm: b.CredentialForm, Name: b.Name}) + } + want := map[string]bool{} + for _, b := range binds { + key := b.Key() + want[key] = true + if why, ok := p.ready(key, gated, in.Provisions); !ok { + o := Outcome{Key: key, Status: Waiting, Why: why} + out = append(out, o) + p.say(key, o.Status+" "+o.Why) + continue + } + o := p.answer(ctx, b, entries[key]) + out = append(out, o) + p.say(key, o.Status+" "+o.Why) + } + // ADR 0570 clause 5: a retiring key the operator said yes to is let go; one asked and not answered, or declined, + // keeps its entry. + for _, key := range sortedKeys(gated) { + if pr := gated[key]; pr.State == provisions.Retiring && pr.Warrant != nil && pr.Warrant.Option == "yes" { + want[key] = false + if _, held := entries[key]; held { + out = append(out, p.letGo(ctx, key, entries[key])...) + } + } + } + return out, nil +} + +// consumerOf is the `.` an entry's key starts with: its first two parts. +func consumerOf(key string) string { + parts := strings.SplitN(key, ".", 3) + if len(parts) < 3 { + return key + } + return parts[0] + "." + parts[1] +} + +// remove deletes an entry this instance read as held, and only while it is still as read when the Bus can say so. +func (p *Provider) remove(ctx context.Context, key string, held []byte) error { + if c, ok := p.opts.Bus.(ConditionalBus); ok { + return c.DeleteIf(ctx, BucketOf(p.opts.Module), key, held) + } + return p.opts.Bus.Delete(ctx, BucketOf(p.opts.Module), key) +} + +// answer makes one bind's entry and writes it when it differs from what the bucket holds. +func (p *Provider) answer(ctx context.Context, e Entry, have []byte) Outcome { + key := e.Key() + offer, offered := p.opts.Offers[e.Provision] + switch { + case !offered: + e.Refused = fmt.Sprintf("%s on %s is bound to %s on %s for %s, which %s does not serve", e.Module, + e.Consumer, e.ProviderModule, e.Provider, e.Provision, e.ProviderModule) + case e.Local != "" && derives(offer.Serves): + e.Refused = fmt.Sprintf("%s on %s keeps several holders of %s (this one is %q), and %s on %s derives %s for "+ + "each consumer from its login: each holder has a login of its own and a consumer is told one value per "+ + "requirement, so the two ends would name different things (novox/hq ADR 0202 §10, ADR 0349 clause 12)", + e.Module, e.Consumer, e.Provision, e.Local, e.ProviderModule, e.Provider, + strings.Join(derivedKeys(offer.Serves), ", ")) + } + if e.Refused != "" { + e.State, e.Why = provisions.Failed, e.Refused + return p.write(ctx, e, have, Outcome{Key: key, Status: Refused, Why: e.Refused}) + } + + // The plan's identity is the controller's, slug first; Identity only answers for an entry without one. + if e.As == "" { + e.As = p.opts.Identity(e.Consumer, e.Module) + } + values, err := ServedTo(offer.Serves, e.As) + if err != nil { + e.Refused = fmt.Sprintf("%s on %s serving %s to %s on %s: %v", e.ProviderModule, e.Provider, e.Provision, + e.Module, e.Consumer, err) + e.As = "" + e.State, e.Why = provisions.Failed, e.Refused + return p.write(ctx, e, have, Outcome{Key: key, Status: Refused, Why: e.Refused}) + } + if e.At != "" || e.From != "" { + if values == nil { + values = map[string]any{} + } + if e.At != "" { + values["at"] = e.At + } + if e.From != "" { + values["from"] = e.From + } + } + if len(e.Names) > 0 { + if values == nil { + values = map[string]any{} + } + for key, name := range e.Names { + values[key] = name + } + } + e.Values = values + var credential []byte + // The plan's form wins; the offer's is the default. + // An offer that declares no credential has none whatever the plan says: the controller writes "pair" for it. + form := e.CredentialForm + if offer.Credential == NoCredential { + form = "" + } else if form == "" { + form = map[CredentialForm]string{Shared: "shared", PerConsumer: "pair"}[offer.Credential] + } + switch form { + case "shared": + e.User = offer.User + e.VaultName = SharedName(e.Provider, e.Provision) + case "pair": + e.User = e.As + e.VaultName = PairName(e.Provider, e.Provision, e.Consumer, e.Module, e.Local) + case "issued", "given": + // A per-holder value, as mesh-vault names it (perholder.go). + e.User = e.As + e.VaultName = HolderName(e.Provider, e.Provision, e.Consumer, e.Module) + } + if e.VaultName != "" { + credential, err = p.opts.Vault.Value(ctx, e.VaultName) + if err != nil { + return Outcome{Key: key, Status: Waiting, Why: fmt.Sprintf("the vault's %s: %v", e.VaultName, err)} + } + } + if p.opts.Make != nil { + if err := p.opts.Make(ctx, Consumer{Entry: e, Credential: credential}); errors.Is(err, ErrNotReady) { + // **Not ready is no failure** (the regression of 2026-10-30: gitea after a restart, its keys not read yet, + // stopped serving every consumer): the entry says waits with why, and keeps what the last entry served, so a + // consumer goes on with it until the provider can make it again. + why := "the provider is not ready to make it yet: " + redact(err.Error(), credential) + waiting := e + var prev Entry + if len(have) > 0 && json.Unmarshal(have, &prev) == nil && prev.State == provisions.Provided { + waiting.Values, waiting.VaultName, waiting.User = prev.Values, prev.VaultName, prev.User + } else { + waiting.Values, waiting.VaultName, waiting.User = nil, "", "" + } + waiting.State, waiting.Why = provisions.Waits, why + p.write(ctx, waiting, have, Outcome{Key: key, Status: Waiting, Why: why}) + return Outcome{Key: key, Status: Waiting, Why: why} + } else if err != nil { + why := "the provider could not make it: " + redact(err.Error(), credential) + // Said in its entry, which the holder of mesh-provisioner reads (ADR 0570 clause 3), with nothing served. + failed := e + failed.Values, failed.VaultName, failed.User, failed.State, failed.Why = nil, "", "", provisions.Failed, why + p.write(ctx, failed, have, Outcome{Key: key, Status: Failed, Why: why}) + return Outcome{Key: key, Status: Failed, Why: why} + } + } + e.State = provisions.Provided + return p.write(ctx, e, have, Outcome{Key: key, Status: Answered}) +} + +// redact takes a credential out of text in the forms it can come back in: as it is, in base64 (standard and URL, +// padded and not) and as a JSON string says it (escapes included). +func redact(text string, credential []byte) string { + if len(credential) == 0 { + return text + } + quoted, _ := json.Marshal(string(credential)) + forms := []string{string(credential), base64.StdEncoding.EncodeToString(credential), + base64.RawStdEncoding.EncodeToString(credential), base64.URLEncoding.EncodeToString(credential), + base64.RawURLEncoding.EncodeToString(credential), string(quoted[1 : len(quoted)-1])} + sort.SliceStable(forms, func(i, j int) bool { return len(forms[i]) > len(forms[j]) }) + for _, f := range forms { + text = strings.ReplaceAll(text, f, "") + } + return text +} + +func (p *Provider) write(ctx context.Context, e Entry, have []byte, o Outcome) Outcome { + body, err := json.Marshal(e) + if err != nil { + return Outcome{Key: o.Key, Status: Failed, Why: err.Error()} + } + if sameJSON(have, body) { + return o + } + if err := p.opts.Bus.Put(ctx, BucketOf(p.opts.Module), o.Key, body); err != nil { + return Outcome{Key: o.Key, Status: Failed, Why: "its entry could not be written: " + err.Error()} + } + return o +} + +// say logs what a key's pass said when it differs from the last. +func (p *Provider) say(key, what string) { + p.mu.Lock() + same := p.said[key] == what + p.said[key] = what + p.mu.Unlock() + if !same { + p.opts.Logf("%s: %s", key, strings.TrimSpace(what)) + } +} + +// SharedName is the vault's one value per provider and provision (ADR 0158, issue 677). +func SharedName(provider, provision string) string { + return provisions.VaultName(provisions.FormShared, provisions.Provision{Provision: provision, Provider: provisions.End{Node: provider}}) +} + +// HolderName is the vault's value of one holder's own credential, issued or given for it (mesh-vault perholder.go). +func HolderName(provider, provision, node, module string) string { + return provisions.VaultName(provisions.FormIssued, provisions.Provision{Provision: provision, + Provider: provisions.End{Node: provider}, Consumer: provisions.End{Node: node, Module: module}}) +} + +// PairName is the vault's value of one consumer's own login (ADR 0048, ADR 0158 §11, issue 677's note 12). +func PairName(provider, provision, node, module, local string) string { + return provisions.VaultName(provisions.FormPair, provisions.Provision{Provision: provision, Local: local, + Provider: provisions.End{Node: provider}, Consumer: provisions.End{Node: node, Module: module}}) +} + +// identityUnusable is every character a consumer's identity does not hold (mesh-controller internal/catalogue +// identity.go, the one derivation). +var identityUnusable = regexp.MustCompile(`[^a-z0-9_]+`) + +// ConsumerIdentity is what one module on one machine is called wherever it authenticates: `mesh__`, +// lower case, anything but letters, digits and `_` written `_` (mesh-controller ConsumerIdentity). module is the +// identity source: the module's slug when it declares one, otherwise its name (ADR 0049). +func ConsumerIdentity(node, module string) string { + clean := func(s string) string { + return strings.Trim(identityUnusable.ReplaceAllString(strings.ToLower(s), "_"), "_") + } + return "mesh_" + clean(node) + "_" + clean(module) +} + +// consumerFact is `${consumer:}` or `${consumer::}`. +var consumerFact = regexp.MustCompile(`\$\{consumer:([a-z][a-z0-9-]*)(?::([a-z][a-z0-9-]*))?\}`) + +// ServedTo fills an offer's served values for one consumer whose identity is as (ADR 0202 §1): `${consumer:as}` as +// it is, `${consumer:as:dns}` with `-` for `_`. Only strings carry placeholders; any other fact or alphabet is +// refused, never left standing. +func ServedTo(serves map[string]any, as string) (map[string]any, error) { + if serves == nil { + return nil, nil + } + out := make(map[string]any, len(serves)) + for _, key := range sortedKeys(serves) { + text, ok := serves[key].(string) + if !ok || !strings.Contains(text, "${consumer:") { + out[key] = serves[key] + continue + } + var failed error + out[key] = consumerFact.ReplaceAllStringFunc(text, func(match string) string { + parts := consumerFact.FindStringSubmatch(match) + switch { + case parts[1] == "as" && parts[2] == "": + return as + case parts[1] == "as" && parts[2] == "dns": + return strings.ReplaceAll(as, "_", "-") + } + if failed == nil { + failed = fmt.Errorf("the value served as %q says %s, and the mesh states only ${consumer:as} and "+ + "${consumer:as:dns}", key, match) + } + return match + }) + if failed != nil { + return nil, failed + } + } + return out, nil +} + +func derives(serves map[string]any) bool { return len(derivedKeys(serves)) > 0 } + +func derivedKeys(serves map[string]any) []string { + var out []string + for _, k := range sortedKeys(serves) { + if text, ok := serves[k].(string); ok && strings.Contains(text, "${consumer:") { + out = append(out, k) + } + } + return out +} + +func clone(m map[string][]byte) map[string][]byte { + out := make(map[string][]byte, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +func sameJSON(a, b []byte) bool { + if a == nil || b == nil { + return false + } + var x, y any + if json.Unmarshal(a, &x) != nil || json.Unmarshal(b, &y) != nil { + return false + } + ja, _ := json.Marshal(x) + jb, _ := json.Marshal(y) + return string(ja) == string(jb) +} + +func sortedKeys[V any](m map[string]V) []string { + if len(m) == 0 { + return nil + } + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// StateBus is the Bus of a provider that runs as a module bundle over the runtime's stdio: the provider's own bucket, +// BucketOf(module), is the module's state `bound`, and provisions.PlansBucket and provisions.ProvisionsBucket are the +// states it reads, `mesh-provisioner.plans` and `mesh-provisioner.provisions` (each declared in the module's manifest). +// Any other bucket is refused. +// +// A stdio state watch lasts as long as the process and cannot be ended (stdio.KeptState.Watch), so Watch honours the +// Bus contract by silencing instead: once ctx has ended the handler is never called again, though the runtime keeps +// sending changes that are dropped. A Pass, which watches twice per call, is therefore correct, only the dropped +// changes of its two spent watches remain. +func StateBus(module string) Bus { return stateBus{module: module} } + +type stateBus struct{ module string } + +func (b stateBus) state(bucket string) (stdio.KeptState, error) { + switch bucket { + case BucketOf(b.module): + return stdio.State("bound"), nil + case provisions.PlansBucket: + return stdio.State(provisions.PlansRead), nil + case provisions.ProvisionsBucket: + return stdio.State(provisions.ProvisionsRead), nil + } + return stdio.KeptState{}, fmt.Errorf("%s is not a bucket this provider keeps or reads", bucket) +} + +func (b stateBus) Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error { + s, err := b.state(bucket) + if err != nil { + return err + } + return s.Watch("", func(ch stdio.StateChange) error { + if ctx.Err() != nil { + return nil + } + if ch.Op == "delete" { + handle(ch.Key, nil) + } else { + handle(ch.Key, ch.Value) + } + return nil + }) +} + +func (b stateBus) Put(_ context.Context, bucket, key string, value []byte) error { + s, err := b.state(bucket) + if err != nil { + return err + } + _, err = s.Put(key, json.RawMessage(value)) + return err +} + +// DeleteIf removes the key only while it still holds what the caller read; it reads the key first, so another +// writer's change in that short while can still be lost. +func (b stateBus) DeleteIf(_ context.Context, bucket, key string, held []byte) error { + s, err := b.state(bucket) + if err != nil { + return err + } + cur, err := s.Get(key) + if err != nil { + return err + } + if cur == nil { + return nil + } + if !sameJSON(cur.Value, held) { + return fmt.Errorf("%s is not as it was read: %w", key, stdio.ErrChanged) + } + return s.Delete(key) +} + +func (b stateBus) Delete(_ context.Context, bucket, key string) error { + s, err := b.state(bucket) + if err != nil { + return err + } + return s.Delete(key) +} + +// Standing is whether an assignment's value is a standing assignment: false when `ended` is set (the module was +// taken off its node, novox/hq ADR 0408 clause 4) or `state` is not empty (a pending one). Such a value is read +// exactly as a removed key. Every other field, known or not, changes nothing (issue 1098). +func Standing(value []byte) bool { + var v struct { + Ended json.RawMessage `json:"ended"` + State json.RawMessage `json:"state"` + } + if json.Unmarshal(value, &v) != nil { + return true // not decided here: the parse that follows refuses it + } + return !fieldSet(v.Ended) && !fieldSet(v.State) +} + +// fieldSet is whether a field carries something: not absent, null or empty. +func fieldSet(f json.RawMessage) bool { + s := strings.TrimSpace(string(f)) + return s != "" && s != "null" && s != `""` +} + +// letGo lets a consumer's entry go: the provider hears it gone (Gone, which retires what it kept for the consumer), and +// the entry is removed while it is still as read. +func (p *Provider) letGo(ctx context.Context, key string, held []byte) []Outcome { + var e Entry + if err := json.Unmarshal(held, &e); err != nil || e.Provider != p.opts.Node || e.ProviderModule != p.opts.Module { + return nil // another machine's instance of this provider wrote it, or nothing of this helper's + } + said := func(o Outcome) []Outcome { + p.say(key, o.Status+" "+o.Why) + return []Outcome{o} + } + if p.opts.Gone != nil { + if err := p.opts.Gone(ctx, e); err != nil { + return said(Outcome{Key: key, Status: Failed, Why: "no longer bound, and the provider could not let it go: " + err.Error()}) + } + } + if err := p.remove(ctx, key, held); err != nil { + if errors.Is(err, stdio.ErrChanged) { + return said(Outcome{Key: key, Status: Superseded, Why: "another instance of the provider wrote its entry " + + "since this pass read it, so it stands"}) + } + return said(Outcome{Key: key, Status: Failed, Why: "no longer bound, and its entry could not be removed: " + err.Error()}) + } + p.say(key, Removed) + p.mu.Lock() + delete(p.said, key) + p.mu.Unlock() + return []Outcome{{Key: key, Status: Removed}} +} + +// ready says whether the provision under key may be made now (novox/hq ADR 0570 clause 3): its key is on the seat, it is +// asked or provided, and every provision it depends on is provided. Otherwise why it waits, naming the key or the +// dependency. +func (p *Provider) ready(key string, mine map[string]provisions.Provision, all map[string][]byte) (string, bool) { + pr, ok := mine[key] + if !ok { + return "mesh-provisioner holds no key " + key + " naming this provider yet", false + } + if pr.State != provisions.Asked && pr.State != provisions.Provided { + return "its key is " + pr.State + ": " + pr.Why, false + } + for _, d := range pr.DependsOn { + dep, err := provisions.Decode(all[d]) + if err != nil { + return "it depends on " + d + ", which mesh-provisioner holds no readable key of", false + } + if dep.State != provisions.Provided { + return "it depends on " + d + ", which is " + dep.State, false + } + } + return "", true +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/provider_test.go b/provisioning/go/clients/meshprovisionerclient/provider/provider_test.go new file mode 100644 index 0000000..594052a --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/provider_test.go @@ -0,0 +1,294 @@ +package provider + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// standIn is a stand-in bus: buckets of keys and values, and watches that hand the current values first and then +// every change, as the bus's key-value watch does. +type standIn struct { + mu sync.Mutex + buckets map[string]map[string][]byte + watchers map[string][]func(string, []byte) +} + +func newStandIn() *standIn { + return &standIn{buckets: map[string]map[string][]byte{}, watchers: map[string][]func(string, []byte){}} +} + +func (s *standIn) Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error { + s.mu.Lock() + current := map[string][]byte{} + for k, v := range s.buckets[bucket] { + current[k] = v + } + s.watchers[bucket] = append(s.watchers[bucket], handle) + s.mu.Unlock() + for _, k := range sortedKeys(current) { + handle(k, current[k]) + } + return nil +} + +func (s *standIn) Put(_ context.Context, bucket, key string, value []byte) error { + s.mu.Lock() + if s.buckets[bucket] == nil { + s.buckets[bucket] = map[string][]byte{} + } + s.buckets[bucket][key] = append([]byte(nil), value...) + ws := slices.Clone(s.watchers[bucket]) + s.mu.Unlock() + for _, w := range ws { + w(key, value) + } + return nil +} + +func (s *standIn) Delete(_ context.Context, bucket, key string) error { + s.mu.Lock() + delete(s.buckets[bucket], key) + ws := slices.Clone(s.watchers[bucket]) + s.mu.Unlock() + for _, w := range ws { + w(key, nil) + } + return nil +} + +// DeleteIf removes the key only while it still holds what the caller read. +func (s *standIn) DeleteIf(ctx context.Context, bucket, key string, held []byte) error { + s.mu.Lock() + cur := s.buckets[bucket][key] + s.mu.Unlock() + if !bytes.Equal(cur, held) { + return fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return s.Delete(ctx, bucket, key) +} + +func (s *standIn) get(bucket, key string) []byte { + s.mu.Lock() + defer s.mu.Unlock() + return s.buckets[bucket][key] +} + +func (s *standIn) keys(bucket string) []string { + s.mu.Lock() + defer s.mu.Unlock() + return sortedKeys(s.buckets[bucket]) +} + +// everything is every value the bus holds, for a test that no secret is in a bucket. +func (s *standIn) everything() []byte { + s.mu.Lock() + defer s.mu.Unlock() + var all []byte + for _, b := range s.buckets { + for k, v := range b { + all = append(all, k...) + all = append(all, v...) + } + } + return all +} + +// vault is a stand-in vault: a value per name, or nothing yet. +type vault struct { + mu sync.Mutex + values map[string]string + asked []string +} + +func (v *vault) Value(_ context.Context, name string) ([]byte, error) { + v.mu.Lock() + defer v.mu.Unlock() + v.asked = append(v.asked, name) + val, ok := v.values[name] + if !ok { + return nil, fmt.Errorf("the vault holds no %s yet", name) + } + return []byte(val), nil +} + +// maker records what the provider's adapter was asked to make, with the credential it was handed. +type maker struct { + mu sync.Mutex + made map[string]string + // contributed is what each consumer's entry carried of its plan's contribution. + contributed map[string]map[string]any + // as is the identity each consumer's entry was made under. + as map[string]string + gone []string + fail error + calls int +} + +func (m *maker) Make(_ context.Context, c Consumer) error { + m.mu.Lock() + defer m.mu.Unlock() + m.calls++ + if m.fail != nil { + return m.fail + } + if m.made == nil { + m.made = map[string]string{} + } + m.made[c.Entry.Key()] = string(c.Credential) + if m.contributed == nil { + m.contributed = map[string]map[string]any{} + } + m.contributed[c.Entry.Key()] = c.Entry.Contributes + if m.as == nil { + m.as = map[string]string{} + } + m.as[c.Entry.Key()] = c.Entry.As + return nil +} + +func (m *maker) Gone(_ context.Context, e Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + m.gone = append(m.gone, e.Key()) + return nil +} + +// assign puts a consumer's plan with its provisions on the seat mesh-provisioner, each provision's key provided, as its +// holder writes them (novox/hq ADR 0570 clauses 1, 2): what a provider binds by. +func assign(t *testing.T, bus *standIn, node, module string, binds ...map[string]string) { + t.Helper() + var ps []catalogue.PlanProvision + for _, b := range binds { + raw, _ := json.Marshal(b) + var pp catalogue.PlanProvision + if err := json.Unmarshal(raw, &pp); err != nil { + t.Fatal(err) + } + ps = append(ps, pp) + } + planned(t, bus, node, module, ps...) +} + +// planned puts a consumer's plan of these provisions on the seat, each provision's key provided. +func planned(t *testing.T, bus *standIn, node, module string, ps ...catalogue.PlanProvision) { + t.Helper() + for _, pp := range ps { + pr := provisions.Provision{Provision: pp.Provision, Local: pp.Local, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: pp.ProviderModule, Node: pp.Provider}, State: provisions.Provided} + key, err := pr.Key() + if err != nil { + t.Fatal(err) + } + value, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + _ = bus.Put(context.Background(), provisions.ProvisionsBucket, key, value) + } + raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: catalogue.Plan{Provisions: ps}}) + if err != nil { + t.Fatal(err) + } + if err := bus.Put(context.Background(), provisions.PlansBucket, node+"."+module, raw); err != nil { + t.Fatal(err) + } +} + +// retire marks a consumer's provision retiring, with the operator's yes (novox/hq ADR 0570 clause 5): the one way a +// provider lets a consumer go. +func retire(t *testing.T, bus *standIn, node, module, provision, provider, providerModule string) { + t.Helper() + pr := provisions.Provision{Provision: provision, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: providerModule, Node: provider}, State: provisions.Retiring, Why: module + " left " + node, + Warrant: &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"}} + key, _ := pr.Key() + raw, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + _ = bus.Delete(context.Background(), provisions.PlansBucket, node+"."+module) + _ = bus.Put(context.Background(), provisions.ProvisionsBucket, key, raw) +} + +func bind(provision, local, provider, providerModule string) map[string]string { + b := map[string]string{"provision": provision, "provider": provider, "provider-module": providerModule} + if local != "" { + b["local"] = local + } + return b +} + +func entryOf(t *testing.T, bus *standIn, key string) Entry { + t.Helper() + raw := bus.get(BucketOf("postgres"), key) + if raw == nil { + t.Fatalf("no entry %s in %s; it holds %v", key, BucketOf("postgres"), bus.keys(BucketOf("postgres"))) + } + var e Entry + if err := json.Unmarshal(raw, &e); err != nil { + t.Fatalf("entry %s: %v", key, err) + } + return e +} + +// postgres is a provider of postgres-database on novox with a login per consumer (ADR 0048, ADR 0158 §11), its +// served values derived from each consumer's identity (ADR 0202 §1). +func postgres(bus *standIn, v *vault, m *maker) *Provider { + return New(Options{ + Node: "novox", Module: "postgres", Bus: bus, Vault: v, Make: m.Make, Gone: m.Gone, + Offers: map[string]Offer{ + "postgres-database": {Credential: PerConsumer, Serves: map[string]any{ + "host": "postgres.novox.mesh", "port": 5432, "database": "${consumer:as}"}}, + }, + Logf: func(string, ...any) {}, + }) +} + +func waitFor(t *testing.T, ok func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatal("not within 3s") + } + time.Sleep(5 * time.Millisecond) + } +} + +// refusedByTheStateGuard mirrors the key rule of the runtime's state guard (mesh-tools node-tools internal/bus +// state.go): a JSON key whose lowercased name, with `-`, `_` and spaces taken out, ends in one of these is refused. +func refusedByTheStateGuard(key string) bool { + k := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(key)) + for _, end := range []string{"password", "passwd", "secret", "token", "credential", "credentials", + "authorization", "apikey", "privatekey", "accesskey", "cookie"} { + if strings.HasSuffix(k, end) { + return true + } + } + return false +} + +func keysOf(v any, into *[]string) { + switch x := v.(type) { + case map[string]any: + for k, c := range x { + *into = append(*into, k) + keysOf(c, into) + } + case []any: + for _, c := range x { + keysOf(c, into) + } + } +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/statebus_test.go b/provisioning/go/clients/meshprovisionerclient/provider/statebus_test.go new file mode 100644 index 0000000..9475411 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/statebus_test.go @@ -0,0 +1,54 @@ +package provider + +import ( + "bufio" + "encoding/json" + "io" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// runtime drives a bundle over pipes as the runtime does: it reads what the bundle says and sends what the runtime would. +type runtime struct { + t *testing.T + lines *bufio.Scanner + in io.Writer +} + +func (r *runtime) next() map[string]any { + done := make(chan map[string]any, 1) + go func() { + if r.lines.Scan() { + var m map[string]any + _ = json.Unmarshal(r.lines.Bytes(), &m) + done <- m + } + }() + select { + case m := <-done: + return m + case <-time.After(3 * time.Second): + r.t.Fatal("the bundle said nothing") + return nil + } +} + +func (r *runtime) send(m map[string]any) { + b, _ := json.Marshal(m) + _, _ = r.in.Write(append(b, '\n')) +} + +func (r *runtime) call(id int, tool string, args map[string]any) { + r.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": map[string]any{"name": tool, "arguments": args}}) +} + +func serve(t *testing.T, tools []stdio.Tool) *runtime { + t.Setenv(stdio.ServedAs, "keeper") + inR, inW := io.Pipe() + outR, outW := io.Pipe() + go func() { _ = stdio.ServeOn("", tools, inR, outW) }() + t.Cleanup(func() { _ = inW.Close() }) + return &runtime{t: t, lines: bufio.NewScanner(outR), in: inW} +} diff --git a/provisioning/go/clients/meshprovisionerclient/provider/tables_test.go b/provisioning/go/clients/meshprovisionerclient/provider/tables_test.go new file mode 100644 index 0000000..2dbf2d6 --- /dev/null +++ b/provisioning/go/clients/meshprovisionerclient/provider/tables_test.go @@ -0,0 +1,909 @@ +package provider + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + "time" + + provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// TestEntries groups the cases below, each one a subtest. +func TestEntries(t *testing.T) { + // A provider answers each consumer that binds it with an entry of its own in its own bucket: composed from its + // served values and the consumer's identity, naming the credential the vault keeps for that consumer, and never + // the credential itself; its adapter is handed each consumer's own value read from the vault. + t.Run("EachConsumerGetsAnEntryOfItsOwnAndTheVaultsValue", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{ + "pair/novox/postgres-database/ace/letta": "letta-secret-1", + "pair/novox/postgres-database/g14/umami": "umami-secret-2", + }} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) + p := postgres(bus, v, m) + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"ace.letta.postgres-database", + "g14.umami.postgres-database"}) { + t.Fatalf("entries %v", got) + } + e := entryOf(t, bus, "ace.letta.postgres-database") + if e.Provider != "novox" || e.ProviderModule != "postgres" || e.Consumer != "ace" || e.Module != "letta" || + e.As != "mesh_ace_letta" || e.User != "mesh_ace_letta" || + e.VaultName != "pair/novox/postgres-database/ace/letta" || e.Refused != "" { + t.Fatalf("entry %+v", e) + } + if e.Values["database"] != "mesh_ace_letta" || e.Values["host"] != "postgres.novox.mesh" || + e.Values["port"] != float64(5432) { + t.Fatalf("values %v", e.Values) + } + if m.made["ace.letta.postgres-database"] != "letta-secret-1" || + m.made["g14.umami.postgres-database"] != "umami-secret-2" { + t.Fatalf("made %v", m.made) + } + if bytes.Contains(bus.everything(), []byte("secret-")) { + t.Fatal("a credential is in a bucket") + } + for _, o := range out { + if o.Status != Answered { + t.Fatalf("outcome %+v", o) + } + } + }) + + // A provider whose offer shares one credential (ADR 0158) names the vault's one value for the provision, and its + // one user name, to every consumer. + t.Run("ASharedCredentialIsTheVaultsOneValueAndTheProvidersOneUser", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"provision/novox/sonarr-api": "one-value"}} + assign(t, bus, "ace", "overseerr", bind("sonarr-api", "", "novox", "sonarr")) + assign(t, bus, "g14", "homepage", bind("sonarr-api", "", "novox", "sonarr")) + p := New(Options{Node: "novox", Module: "sonarr", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {}, + Offers: map[string]Offer{"sonarr-api": {Credential: Shared, User: "sonarr", + Serves: map[string]any{"url": "http://sonarr.novox.mesh:8989"}}}}) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + for _, key := range []string{"ace.overseerr.sonarr-api", "g14.homepage.sonarr-api"} { + var e Entry + if err := json.Unmarshal(bus.get(BucketOf("sonarr"), key), &e); err != nil { + t.Fatalf("%s: %v", key, err) + } + if e.VaultName != "provision/novox/sonarr-api" || e.User != "sonarr" || e.Values["url"] == nil { + t.Fatalf("%s: %+v", key, e) + } + if m.made[key] != "one-value" { + t.Fatalf("%s made with %q", key, m.made[key]) + } + } + if bytes.Contains(bus.everything(), []byte("one-value")) { + t.Fatal("the shared credential is in a bucket") + } + }) + + // A consumer that keeps several holders of one provision is served no derived value: its entry refuses it, naming + // both ends, so its client can report it (ADR 0349 clause 12, ADR 0202 §10); nothing is made for it. + t.Run("AMultiHolderConsumerIsRefusedADerivedValueNamingBothEnds", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta/archive": "x-secret"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "archive", "novox", "postgres")) + out, err := postgres(bus, v, m).Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + e := entryOf(t, bus, "ace.letta.postgres-database.archive") + for _, says := range []string{"letta on ace", "postgres on novox", "postgres-database", "archive"} { + if !strings.Contains(e.Refused, says) { + t.Fatalf("the refusal does not name %q: %s", says, e.Refused) + } + } + if e.Values != nil || e.VaultName != "" { + t.Fatalf("a refused entry carries values: %+v", e) + } + if m.calls != 0 { + t.Fatal("the adapter made a refused consumer") + } + if len(out) != 1 || out[0].Status != Refused { + t.Fatalf("outcomes %+v", out) + } + }) + + // A consumer bound under a local name is served when the offer derives nothing from it: its credential is its own + // pair row under that local name. + t.Run("ALocalNameIsServedWhenNothingIsDerived", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/redis-cache/ace/letta/queue": "q-secret"}} + assign(t, bus, "ace", "letta", bind("redis-cache", "queue", "novox", "redis")) + p := New(Options{Node: "novox", Module: "redis", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {}, + Offers: map[string]Offer{"redis-cache": {Credential: PerConsumer, Serves: map[string]any{"port": 6379}}}}) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + var e Entry + if err := json.Unmarshal(bus.get(BucketOf("redis"), "ace.letta.redis-cache.queue"), &e); err != nil { + t.Fatal(err) + } + if e.Refused != "" || e.VaultName != "pair/novox/redis-cache/ace/letta/queue" || e.Local != "queue" { + t.Fatalf("entry %+v", e) + } + if m.made["ace.letta.redis-cache.queue"] != "q-secret" { + t.Fatalf("made %v", m.made) + } + }) + + // A consumer's plan entry may carry what its module contributes to the provider (issue 776): the helper hands it to + // the provider's adapter with the consumer, and does not write it to the bucket. + t.Run("AConsumersContributionIsHandedToTheProviderPerConsumer", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{ + "pair/novox/postgres-database/ace/nextcloud": "a", "pair/novox/postgres-database/g14/umami": "b"}} + var pp catalogue.PlanProvision + if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+ + `"contributes":{"name":"nextcloud","extensions":["pg_trgm"]}}`), &pp); err != nil { + t.Fatal(err) + } + planned(t, bus, "ace", "nextcloud", pp) + assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) + if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil { + t.Fatal(err) + } + want := map[string]any{"name": "nextcloud", "extensions": []any{"pg_trgm"}} + if got := m.contributed["ace.nextcloud.postgres-database"]; !reflect.DeepEqual(got, want) { + t.Fatalf("the adapter was handed the contribution %v, want %v", got, want) + } + if got := m.contributed["g14.umami.postgres-database"]; got != nil { + t.Fatalf("a consumer contributing nothing was handed %v", got) + } + if bytes.Contains(bus.get(BucketOf("postgres"), "ace.nextcloud.postgres-database"), []byte("pg_trgm")) { + t.Fatal("a contribution is in the provider's bucket") + } + }) + + // A consumer's plan entry carries where its provider is, which machine, and who the consumer is (issue 788): the + // entry's values carry `at` and `from` as the binding file gives them, and its `as` is the plan's, so a module with a + // slug keeps the login it has today. An entry of a plan without them is as it was. + t.Run("AConsumersEntryCarriesAtFromAndTheAsOfItsPlan", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{ + "pair/novox/postgres-database/ace/claude-code": "a", "pair/novox/postgres-database/g14/umami": "b"}} + var pp catalogue.PlanProvision + if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+ + `"at":"10.7.0.1","from":"novox","as":"mesh_ace_agent"}`), &pp); err != nil { + t.Fatal(err) + } + planned(t, bus, "ace", "claude-code", pp) + assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) + if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil { + t.Fatal(err) + } + var e Entry + if err := json.Unmarshal(bus.get(BucketOf("postgres"), "ace.claude-code.postgres-database"), &e); err != nil { + t.Fatal(err) + } + if e.Values["at"] != "10.7.0.1" || e.Values["from"] != "novox" || e.Values["port"] == nil { + t.Fatalf("the entry's values are %v", e.Values) + } + if e.As != "mesh_ace_agent" || e.User != "mesh_ace_agent" || m.as["ace.claude-code.postgres-database"] != "mesh_ace_agent" { + t.Fatalf("the entry is %+v, made as %q", e, m.as["ace.claude-code.postgres-database"]) + } + var plain Entry + if err := json.Unmarshal(bus.get(BucketOf("postgres"), "g14.umami.postgres-database"), &plain); err != nil { + t.Fatal(err) + } + if _, has := plain.Values["at"]; has || plain.As != "mesh_g14_umami" { + t.Fatalf("an entry of a plan without them changed: %+v", plain) + } + }) + + // A consumer's plan entry may carry the names the controller composed for it (issue 792): each is copied into the + // entry's values under its key, over anything the provider serves under it. An entry without them is as it was. + t.Run("AConsumersEntryCarriesTheNamesOfItsPlan", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{ + "pair/novox/postgres-database/ace/gitea": "a", "pair/novox/postgres-database/g14/umami": "b"}} + var pp catalogue.PlanProvision + if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+ + `"names":{"name":"git.example.org","internal-name":"git.mesh","name-web":"web.example.org"}}`), &pp); err != nil { + t.Fatal(err) + } + planned(t, bus, "ace", "gitea", pp) + assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) + if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil { + t.Fatal(err) + } + var e Entry + if err := json.Unmarshal(bus.get(BucketOf("postgres"), "ace.gitea.postgres-database"), &e); err != nil { + t.Fatal(err) + } + if e.Values["name"] != "git.example.org" || e.Values["internal-name"] != "git.mesh" || + e.Values["name-web"] != "web.example.org" || e.Values["port"] == nil { + t.Fatalf("the entry's values are %v", e.Values) + } + var plain Entry + if err := json.Unmarshal(bus.get(BucketOf("postgres"), "g14.umami.postgres-database"), &plain); err != nil { + t.Fatal(err) + } + if _, has := plain.Values["name"]; has { + t.Fatalf("an entry of a plan without names changed: %+v", plain) + } + }) + + // The plan's credential form and holder name win over the offer's; the offer's form is the default (issue 825, + // finding 4). + t.Run("ThePlansCredentialFormWinsOverTheOffers", func(t *testing.T) { + run := func(offer CredentialForm, plan map[string]string) Entry { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "p", + "provision/novox/postgres-database": "s", "provision/novox/postgres-database/ace/letta": "h"}} + b := bind("postgres-database", "", "novox", "postgres") + for k, val := range plan { + b[k] = val + } + assign(t, bus, "ace", "letta", b) + p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {}, + Offers: map[string]Offer{"postgres-database": {Credential: offer, User: "shared-user"}}}) + out, err := p.Pass(context.Background()) + if err != nil || len(out) != 1 || out[0].Status != Answered { + t.Fatalf("%v %+v", err, out) + } + return entryOf(t, bus, "ace.letta.postgres-database") + } + if e := run(PerConsumer, map[string]string{"credential-form": "shared"}); e.VaultName != + "provision/novox/postgres-database" || e.User != "shared-user" { + t.Fatalf("shared over pair: %+v", e) + } + if e := run(Shared, map[string]string{"credential-form": "pair"}); e.VaultName != + "pair/novox/postgres-database/ace/letta" || e.User != "mesh_ace_letta" { + t.Fatalf("pair over shared: %+v", e) + } + for _, form := range []string{"issued", "given"} { + if e := run(PerConsumer, map[string]string{"credential-form": form, "name": "main"}); e.VaultName != + "provision/novox/postgres-database/ace/letta" || e.User != "mesh_ace_letta" { + t.Fatalf("%s holder: %+v", form, e) + } + } + if e := run(Shared, nil); e.VaultName != "provision/novox/postgres-database" { + t.Fatalf("the offer's form is the default: %+v", e) + } + { + // An offer that declares no credential is answered with none whatever form the plan says (the controller writes + // "pair" for such an offer): no vault name, and the Vault is never asked, so a provider with no Vault works (issue 825, + // finding 4; issue 923). + bus := newStandIn() + b := bind("route", "", "novox", "route-proxy") + b["credential-form"] = "pair" + assign(t, bus, "ace", "letta", b) + p := New(Options{Node: "novox", Module: "route-proxy", Bus: bus, Logf: func(string, ...any) {}, + Offers: map[string]Offer{"route": {Credential: NoCredential}}}) + out, err := p.Pass(context.Background()) + if err != nil || len(out) != 1 || out[0].Status != Answered { + t.Fatalf("%v %+v", err, out) + } + var e Entry + if err := json.Unmarshal(bus.get(BucketOf("route-proxy"), "ace.letta.route"), &e); err != nil || e.VaultName != "" || + e.User != "" { + t.Fatalf("%v %+v", err, e) + } + } + }) + + // Issue 1126 review 1 (mcp-endpoint asked on ace and shanks, 2026-11-02 11:56): a consumer's key composed from its newest + // build carries the plan entry it was composed from, though its wanted build's plan names no such provision; the + // provider binds it from the key and provides it, so the want that waits for it moves. A key of this provider that + // carries no entry and no plan names is said, never bound silently or left without a word. + t.Run("AKeyNoPlanNamesIsBoundFromTheEntryItCarries", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "letta-secret-1"}} + // letta's wanted plan names nothing of postgres; its key, from its newest build, carries the entry. + planned(t, bus, "ace", "letta") + entry := catalogue.PlanProvision{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres", As: "mesh_ace_letta"} + pr := provisions.Provision{Provision: "postgres-database", Consumer: provisions.End{Module: "letta", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Asked, Plan: &entry} + raw, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + _ = bus.Put(context.Background(), provisions.ProvisionsBucket, "ace.letta.postgres-database", raw) + // And umami's key carries none: asked, but no plan names it. + bare := provisions.Provision{Provision: "postgres-database", Consumer: provisions.End{Module: "umami", Node: "g14"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Asked} + raw, _ = provisions.Encode(bare) + _ = bus.Put(context.Background(), provisions.ProvisionsBucket, "g14.umami.postgres-database", raw) + + out, err := postgres(bus, v, m).Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if e := entryOf(t, bus, "ace.letta.postgres-database"); e.Consumer != "ace" || e.Module != "letta" || e.Refused != "" { + t.Errorf("the entry %+v", e) + } + said := map[string]Outcome{} + for _, o := range out { + said[o.Key] = o + } + if said["ace.letta.postgres-database"].Status != Answered { + t.Errorf("letta: %+v", said["ace.letta.postgres-database"]) + } + if o := said["g14.umami.postgres-database"]; o.Status != Waiting || !strings.Contains(o.Why, "no plan names it") { + t.Errorf("umami: %+v", o) + } + if bus.get(BucketOf("postgres"), "g14.umami.postgres-database") != nil { + t.Error("a key with no entry and no plan was bound") + } + }) +} + +// TestBinds groups the cases below, each one a subtest. +func TestBinds(t *testing.T) { + // A consumer that unbinds loses its entry and the adapter hears it is gone; an entry another machine's instance of + // the same provider wrote is not this one's to remove. + t.Run("AnUnbindRemovesTheEntryAndLeavesAnotherProvidersAlone", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + other, _ := json.Marshal(Entry{Provider: "shanks", ProviderModule: "postgres", Provision: "postgres-database", + Consumer: "g14", Module: "gitea"}) + _ = bus.Put(context.Background(), BucketOf("postgres"), "g14.gitea.postgres-database", other) + p := postgres(bus, v, m) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres") + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"g14.gitea.postgres-database"}) { + t.Fatalf("entries left %v", got) + } + if !slices.Equal(m.gone, []string{"ace.letta.postgres-database"}) { + t.Fatalf("gone %v", m.gone) + } + if len(out) != 1 || out[0].Status != Removed { + t.Fatalf("outcomes %+v", out) + } + }) + + // Only the binds naming this provider, on this machine, are its own; a bind naming it for a provision it does not + // offer is refused, naming both ends. + t.Run("OnlyBindsNamingThisProviderAreAnswered", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "shanks", "postgres"), + bind("mongodb-database", "", "novox", "postgres")) + assign(t, bus, "ace", "other", bind("postgres-database", "", "novox", "mssql")) + out, err := postgres(bus, v, m).Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"ace.letta.mongodb-database"}) { + t.Fatalf("entries %v", got) + } + e := entryOf(t, bus, "ace.letta.mongodb-database") + if !strings.Contains(e.Refused, "postgres on novox") || !strings.Contains(e.Refused, "letta on ace") { + t.Fatalf("refusal %q", e.Refused) + } + if len(out) != 1 || m.calls != 0 { + t.Fatalf("outcomes %+v, %d made", out, m.calls) + } + }) + + // A plan that cannot be read is skipped and said, by its key and why; the rest of the pass goes on (issue 825, + // finding 3). Its consumer's own entry is left alone, since a bind missed would read as an unbind. + t.Run("AnUnreadableAssignmentIsSkippedAndSaidAndTheRestGoesOn", func(t *testing.T) { + for name, bad := range map[string]string{"not json": "{not json", + "another key": `{"v":1,"node":"g14","module":"umami","plan":{"templates":{"source":"","digest":""},"files":null}}`} { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s", + "pair/novox/postgres-database/g14/umami": "u"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + p := postgres(bus, v, m) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + _ = bus.Put(context.Background(), provisions.PlansBucket, "ace.letta", []byte(bad)) + assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) + var logged []string + p.opts.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if bus.get(BucketOf("postgres"), "g14.umami.postgres-database") == nil { + t.Fatalf("%s: the good consumer was not answered", name) + } + if bus.get(BucketOf("postgres"), "ace.letta.postgres-database") == nil || len(m.gone) != 0 { + t.Fatalf("%s: the entry of the unreadable assignment's consumer was removed", name) + } + said := false + for _, o := range out { + if o.Key == "ace.letta" && o.Status == Failed && o.Why != "" { + said = true + } + } + if !said || !strings.Contains(strings.Join(logged, "\n"), "ace.letta") { + t.Fatalf("%s: the skipped assignment is not said: %+v, %q", name, out, logged) + } + } + }) + + // Run watches the binds on the stand-in bus: a consumer that binds is answered, and one that unbinds is removed, + // without a pass asked by hand. + t.Run("RunAnswersBindsAndUnbindsAsTheyAreWatched", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s"}} + p := postgres(bus, v, m) + p.opts.Every = time.Hour + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- p.Run(ctx) }() + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + waitFor(t, func() bool { return bus.get(BucketOf("postgres"), "ace.letta.postgres-database") != nil }) + retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres") + waitFor(t, func() bool { return bus.get(BucketOf("postgres"), "ace.letta.postgres-database") == nil }) + cancel() + if err := <-done; err != nil && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + }) + + // An instance deletes only an entry it wrote, and only while the entry is as it read it: one that another instance of + // the provider wrote after the consumer moved stands (issue 825, finding 5). + t.Run("AnEntryAnotherInstanceWroteAfterTheMoveStands", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + a := postgres(bus, v, m) + if _, err := a.Pass(context.Background()); err != nil { + t.Fatal(err) + } + // The consumer moves to the instance on shanks: A's pass reads its own entry and the unbind, and while it lets + // the consumer go, B writes the consumer's entry. + retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres") + theirs, _ := json.Marshal(Entry{Provider: "shanks", ProviderModule: "postgres", Provision: "postgres-database", + Consumer: "ace", Module: "letta", VaultName: "pair/shanks/postgres-database/ace/letta"}) + a.opts.Gone = func(ctx context.Context, e Entry) error { + return bus.Put(ctx, BucketOf("postgres"), e.Key(), theirs) + } + out, err := a.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if got := bus.get(BucketOf("postgres"), "ace.letta.postgres-database"); !bytes.Equal(got, theirs) { + t.Fatalf("the other instance's entry was removed: %s", got) + } + for _, o := range out { + if o.Status == Removed { + t.Fatalf("outcomes %+v", out) + } + } + }) +} + +// TestCredentialsAndKeys groups the cases below, each one a subtest. +func TestCredentialsAndKeys(t *testing.T) { + // A bind whose credential the vault does not hold yet is waited for: nothing is made and no entry is written, so a + // consumer is never told of a login its provider has not made; once the vault holds it, it is answered. + t.Run("ACredentialTheVaultDoesNotHoldYetIsWaitedFor", func(t *testing.T) { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + p := postgres(bus, v, m) + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].Status != Waiting || !strings.Contains(out[0].Why, "pair/novox/postgres-database/ace/letta") { + t.Fatalf("outcomes %+v", out) + } + if bus.keys(BucketOf("postgres")) != nil || m.calls != 0 { + t.Fatal("a consumer was answered before its credential was there") + } + v.values["pair/novox/postgres-database/ace/letta"] = "late" + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + if entryOf(t, bus, "ace.letta.postgres-database").Refused != "" || m.made["ace.letta.postgres-database"] != "late" { + t.Fatal("not answered once the vault held the value") + } + }) + + // An adapter that fails is said with the credential taken out of its words, and its entry says it failed, serving + // nothing (novox/hq ADR 0570 clause 3: the provider's outcome is its entry, which the holder of mesh-provisioner reads). + t.Run("AFailedMakeIsSaidWithoutTheCredential", func(t *testing.T) { + bus, m := newStandIn(), &maker{fail: errors.New(`password "hunter2-secret" rejected`)} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "hunter2-secret"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + var logged []string + p := postgres(bus, v, m) + p.opts.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].Status != Failed || strings.Contains(out[0].Why, "hunter2") { + t.Fatalf("outcomes %+v", out) + } + if strings.Contains(strings.Join(logged, "\n"), "hunter2") || len(logged) == 0 { + t.Fatalf("logged %q", logged) + } + var e Entry + raw := bus.get(BucketOf("postgres"), "ace.letta.postgres-database") + if err := json.Unmarshal(raw, &e); err != nil || e.State != "failed" || e.Values != nil || e.VaultName != "" || + strings.Contains(string(raw), "hunter2") { + t.Fatalf("the failed entry is %s", raw) + } + + // A provider not ready yet (gitea after a restart, its keys not read; the regression of 2026-10-30) is no failure: + // a first entry waits serving nothing, and once made, a later not ready keeps serving what it served. + m.fail = fmt.Errorf("keys not read yet: %w", ErrNotReady) + _ = bus.Delete(context.Background(), BucketOf("postgres"), "ace.letta.postgres-database") + if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Waiting { + t.Fatalf("not ready at first: %+v", out) + } + raw = bus.get(BucketOf("postgres"), "ace.letta.postgres-database") + if err := json.Unmarshal(raw, &e); err != nil || e.State != "waits" || !strings.Contains(e.Why, "keys not read yet") || e.Values != nil { + t.Fatalf("the first entry not ready is %s", raw) + } + m.fail = nil + if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Answered { + t.Fatalf("made: %+v", out) + } + var made Entry + _ = json.Unmarshal(bus.get(BucketOf("postgres"), "ace.letta.postgres-database"), &made) + m.fail = ErrNotReady + if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Waiting { + t.Fatalf("not ready after a restart: %+v", out) + } + raw = bus.get(BucketOf("postgres"), "ace.letta.postgres-database") + if err := json.Unmarshal(raw, &e); err != nil || e.State != "waits" || e.VaultName != made.VaultName || e.User != made.User || + fmt.Sprint(e.Values) != fmt.Sprint(made.Values) || len(made.Values) == 0 { + t.Fatalf("not ready after it served, the entry is %s, it served %+v", raw, made) + } + }) + + // A failed Make is said with the credential taken out in the forms it can come back in: as it is, in base64 (standard + // and URL, with and without padding) and as a JSON string says it (issue 825, finding 6). + t.Run("AFailedMakeIsSaidWithoutTheCredentialInAnyOfItsForms", func(t *testing.T) { + cred := "pa/ss+w\"rd?>>x" + b := []byte(cred) + jsonQuoted, _ := json.Marshal(cred) + forms := map[string]string{ + "raw": cred, "std": base64.StdEncoding.EncodeToString(b), "std unpadded": base64.RawStdEncoding.EncodeToString(b), + "url": base64.URLEncoding.EncodeToString(b), "url unpadded": base64.RawURLEncoding.EncodeToString(b), + "json": string(jsonQuoted[1 : len(jsonQuoted)-1]), + } + if forms["std"] == forms["url"] || forms["json"] == forms["raw"] { + t.Fatal("the test credential does not tell the forms apart") + } + for name, form := range forms { + bus, m := newStandIn(), &maker{fail: errors.New("login failed for " + form + " at the server")} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": cred}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + out, err := postgres(bus, v, m).Pass(context.Background()) + if err != nil || len(out) != 1 { + t.Fatalf("%v %+v", err, out) + } + if strings.Contains(out[0].Why, form) || !strings.Contains(out[0].Why, "") { + t.Fatalf("%s form: %q", name, out[0].Why) + } + } + }) + + // The consumer's identity is the mesh's (ADR 0202 §1): mesh__, lower case, `_` for anything else, + // and its DNS form writes `-` for `_`. + t.Run("TheIdentityIsTheMeshsAndItsDNSForm", func(t *testing.T) { + if got := ConsumerIdentity("Home.Server", "my-app"); got != "mesh_home_server_my_app" { + t.Fatal(got) + } + got, err := ServedTo(map[string]any{"bucket": "b-${consumer:as:dns}", "n": 3}, "mesh_ace_letta") + if err != nil || got["bucket"] != "b-mesh-ace-letta" || got["n"] != 3 { + t.Fatalf("%v %v", got, err) + } + if _, err := ServedTo(map[string]any{"x": "${consumer:node}"}, "mesh_ace_letta"); err == nil { + t.Fatal("a fact the mesh does not state was filled") + } + }) + + // Every entry the helper writes passes the state guard's key rule (issue 945), and names where the vault keeps the + // credential under `vault-name`, for a login per consumer and for a shared one. + t.Run("AnEntryHasNoKeyTheStateGuardRefuses", func(t *testing.T) { + for name, form := range map[string]CredentialForm{"pair": PerConsumer, "shared": Shared} { + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1", + "provision/novox/postgres-database": "s2"}} + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v, Make: m.Make, Logf: func(string, ...any) {}, + Offers: map[string]Offer{"postgres-database": {Credential: form, User: "shared-user"}}}) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(bus.get(BucketOf("postgres"), "ace.letta.postgres-database"), &doc); err != nil { + t.Fatal(err) + } + var keys []string + keysOf(doc, &keys) + for _, k := range keys { + if refusedByTheStateGuard(k) { + t.Fatalf("%s: the entry carries the key %q, which the state guard refuses", name, k) + } + } + want := map[string]string{"pair": "pair/novox/postgres-database/ace/letta", + "shared": "provision/novox/postgres-database"}[name] + if doc["vault-name"] != want { + t.Fatalf("%s: vault-name is %v, want %s", name, doc["vault-name"], want) + } + } + }) + + // novox/hq ADR 0570 clauses 3 and 5 (issue 1954 L6-P3, P4): a consumer the seat mesh-provisioner plans is bound by its + // plan and its provision key, never by its assignment. Its entry is made only once its key is asked and every provision + // it depends on is provided, and says provided; a plan gone retires nothing by itself; a retiring key is let go only on + // the operator's yes. + t.Run("TheSeatsKeysSayWhenAProviderActsAndWhenItLetsGo", func(t *testing.T) { + ctx := context.Background() + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1"}} + // The same consumer's assignment names another provider: the seat's plan wins. + assign(t, bus, "ace", "letta", bind("postgres-database", "", "shanks", "postgres")) + plan, _ := provisions.EncodePlan(provisions.Plan{Node: "ace", Module: "letta", Plan: catalogue.Plan{ + Provisions: []catalogue.PlanProvision{{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres"}}}}) + _ = bus.Put(ctx, provisions.PlansBucket, "ace.letta", plan) + key := "ace.letta.postgres-database" + put := func(k string, state string, depends []string, warrant *provisions.Warrant) { + pr := provisions.Provision{Provision: "postgres-database", Consumer: provisions.End{Module: "letta", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, DependsOn: depends, State: state, Warrant: warrant} + if k != key { + _, _, pr.Provision, _, _ = provisions.ParseKey(k) + pr.Provider = provisions.End{Module: "mesh-vault", Node: "novox"} + } + if state != provisions.Asked && state != provisions.Provided { + pr.Why = "said" + } + raw, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + _ = bus.Put(ctx, provisions.ProvisionsBucket, k, raw) + } + p := postgres(bus, v, m) + steps := []struct { + name string + before func() + status string + says string + entry bool + }{ + {name: "no key yet", status: Waiting, says: "holds no key"}, + {name: "a dependency asked", before: func() { + put("ace.letta.secret", provisions.Asked, nil, nil) + put(key, provisions.Asked, []string{"ace.letta.secret"}, nil) + }, status: Waiting, says: "depends on ace.letta.secret, which is asked"}, + {name: "the dependency provided", before: func() { put("ace.letta.secret", provisions.Provided, nil, nil) }, + status: Answered, entry: true}, + {name: "its plan gone: nothing let go by itself", before: func() { _ = bus.Delete(ctx, provisions.PlansBucket, "ace.letta") }, + entry: true}, + {name: "retiring, not yet answered", before: func() { put(key, provisions.Retiring, nil, nil) }, entry: true}, + {name: "retiring, the operator's yes", before: func() { + put(key, provisions.Retiring, nil, &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"}) + }, status: Removed}, + } + for _, s := range steps { + if s.before != nil { + s.before() + } + out, err := p.Pass(ctx) + if err != nil { + t.Fatalf("%s: %v", s.name, err) + } + var mine []Outcome + for _, o := range out { + if o.Key == key { + mine = append(mine, o) + } + } + if s.status != "" && (len(mine) != 1 || mine[0].Status != s.status || !strings.Contains(mine[0].Why, s.says)) { + t.Errorf("%s: %+v", s.name, mine) + } + if held := bus.get(BucketOf("postgres"), key) != nil; held != s.entry { + t.Errorf("%s: an entry held %v", s.name, held) + } + if s.entry && entryOf(t, bus, key).State != provisions.Provided { + t.Errorf("%s: the entry does not say provided", s.name) + } + } + if !slices.Equal(m.gone, []string{key}) || m.made[key] != "s1" { + t.Errorf("gone %v, made %v: let go once, on the warrant, and made from the vault's value", m.gone, m.made) + } + }) +} + +// TestConsumersFileAndStateBus groups the cases below, each one a subtest. +func TestConsumersFileAndStateBus(t *testing.T) { + // The provider reads its consumers' credentials only from the file its machine's provisioner client fills (novox/hq + // ADR 0373 clause 3, issue 679): one entry per consumer, naming the consumer and the vault's name for its credential, + // and carrying the value. The helper is wired to it as its Vault and asks nothing of the vault itself. + t.Run("TheConsumersFileAnswersEachConsumersCredential", func(t *testing.T) { + dir := t.TempDir() + at := filepath.Join(dir, "consumers.json") + file := `[{"consumer":"ace","module":"letta","credential":"pair/novox/postgres-database/ace/letta","value":"s3cret"}, + {"consumer":"g14","module":"gitea","local":"cache","credential":"pair/novox/postgres-database/g14/gitea/cache","value":"other"}]` + if err := os.WriteFile(at, []byte(file), 0o600); err != nil { + t.Fatal(err) + } + v := ConsumersFile{Paths: []string{filepath.Join(dir, "absent.json"), at}} + got, err := v.Value(context.Background(), PairName("novox", "postgres-database", "ace", "letta", "")) + if err != nil || string(got) != "s3cret" { + t.Fatalf("ace's letta: %q, %v; want s3cret", got, err) + } + got, err = v.Value(context.Background(), PairName("novox", "postgres-database", "g14", "gitea", "cache")) + if err != nil || string(got) != "other" { + t.Fatalf("g14's gitea under cache: %q, %v; want other", got, err) + } + // A consumer the file does not carry is waited for, and never told of a login its provider has not made. + _, err = v.Value(context.Background(), PairName("novox", "postgres-database", "shanks", "photos", "")) + if err == nil || !strings.Contains(err.Error(), "pair/novox/postgres-database/shanks/photos") { + t.Fatalf("a consumer the file does not carry: %v", err) + } + if strings.Contains(err.Error(), "s3cret") || strings.Contains(err.Error(), "other") { + t.Fatalf("an error names a credential: %v", err) + } + + // Wired as the helper's Vault, the file is what makes each consumer's login. + bus := newStandIn() + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) + var made []string + p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v, + Offers: map[string]Offer{"postgres-database": {Serves: map[string]any{"port": 5432}}}, + Make: func(_ context.Context, c Consumer) error { + made = append(made, c.Entry.User+"="+string(c.Credential)) + return nil + }, Logf: func(string, ...any) {}}) + if _, err := p.Pass(context.Background()); err != nil { + t.Fatal(err) + } + if len(made) != 1 || made[0] != "mesh_ace_letta=s3cret" { + t.Fatalf("made %v, want ace's letta under the file's credential", made) + } + }) + + // A file that is not the client's form is said, never read as no consumers. + t.Run("AConsumersFileNotInItsFormIsRefused", func(t *testing.T) { + at := filepath.Join(t.TempDir(), "consumers.json") + if err := os.WriteFile(at, []byte(`{"not": "a list"}`), 0o600); err != nil { + t.Fatal(err) + } + _, err := ConsumersFile{Paths: []string{at}}.Value(context.Background(), "pair/novox/p/ace/m") + if err == nil || !strings.Contains(err.Error(), at) { + t.Fatalf("a file not in the client's form: %v", err) + } + }) + + t.Run("StandingIsFalseOnlyForAnEndedOrPendingValue", func(t *testing.T) { + for raw, want := range map[string]bool{ + `{"node":"a","module":"b"}`: true, + `{"node":"a","module":"b","ended":"","state":"","x":{"y":1}}`: true, + `{"node":"a","module":"b","ended":null}`: true, + `{"node":"a","module":"b","build":"b1","commit":"c","rollout":"r","since":"t","artifact":{"name":"n"}}`: true, + `{"node":"a","module":"b","ended":"2026-10-18T01:00:00Z"}`: false, + `{"node":"a","module":"b","state":"pending"}`: false, + } { + if got := Standing([]byte(raw)); got != want { + t.Errorf("%s: %v, want %v", raw, got, want) + } + } + }) + + // The bus of a moved provider maps its own bucket to the state `bound` and the seat's plans to the state + // `mesh-provisioner.plans` (novox/hq ADR 0570 clause 2), writes only the own bucket's state, and refuses any other + // bucket, the controller's assignments among them. + t.Run("StateBusMapsTheProvidersBucketsToTheRuntimesStates", func(t *testing.T) { + bus := StateBus("postgres") + rt := serve(t, []stdio.Tool{ + {Name: "watch", Run: func(a map[string]any) (any, error) { + return nil, bus.Watch(context.Background(), a["bucket"].(string), func(string, []byte) {}) + }}, + {Name: "put", Run: func(map[string]any) (any, error) { + return "put", bus.Put(context.Background(), BucketOf("postgres"), "k", []byte(`{"a":1}`)) + }}, + {Name: "delete", Run: func(map[string]any) (any, error) { + return "deleted", bus.Delete(context.Background(), BucketOf("postgres"), "k") + }}, + }) + for i, c := range []struct{ bucket, state string }{ + {BucketOf("postgres"), "bound"}, + {provisions.PlansBucket, "mesh-provisioner.plans"}, + } { + rt.call(i+1, "watch", map[string]any{"bucket": c.bucket}) + w := rt.next() + if w["method"] != "mesh/state.watch" || w["params"].(map[string]any)["state"] != c.state { + t.Fatalf("bucket %s asked %v, want the state %s", c.bucket, w, c.state) + } + rt.send(map[string]any{"jsonrpc": "2.0", "id": w["id"], "result": map[string]any{}}) + rt.next() + } + rt.call(3, "put", nil) + p := rt.next() + if p["method"] != "mesh/state.put" || p["params"].(map[string]any)["state"] != "bound" || p["params"].(map[string]any)["key"] != "k" { + t.Fatalf("put asked %v", p) + } + rt.send(map[string]any{"jsonrpc": "2.0", "id": p["id"], "result": map[string]any{"revision": 1}}) + rt.next() + rt.call(4, "delete", nil) + d := rt.next() + if d["method"] != "mesh/state.delete" || d["params"].(map[string]any)["state"] != "bound" { + t.Fatalf("delete asked %v", d) + } + rt.send(map[string]any{"jsonrpc": "2.0", "id": d["id"], "result": map[string]any{}}) + rt.next() + + for _, bucket := range []string{"other_bound", "elsewhere", "mesh-controller_assignments"} { + if err := bus.Watch(context.Background(), bucket, func(string, []byte) {}); err == nil || !strings.Contains(err.Error(), "not a bucket") { + t.Fatalf("a watch of %s gave %v", bucket, err) + } + if err := bus.Put(context.Background(), bucket, "k", nil); err == nil { + t.Fatalf("a put to %s was allowed", bucket) + } + if err := bus.Delete(context.Background(), bucket, "k"); err == nil { + t.Fatalf("a delete in %s was allowed", bucket) + } + } + }) + + // A watch whose ctx has ended calls its handler no more, though the runtime keeps sending changes: the Bus contract + // says a watch lasts until ctx ends, and a stdio state watch lasts as long as the process. + t.Run("StateBusWatchCallsItsHandlerNoMoreOnceItsContextEnded", func(t *testing.T) { + bus := StateBus("postgres") + ctx, cancel := context.WithCancel(context.Background()) + handled := make(chan string, 8) + rt := serve(t, []stdio.Tool{ + {Name: "start", Run: func(map[string]any) (any, error) { + return "started", bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) { + handled <- key + "=" + string(value) + }) + }}, + {Name: "stop", Run: func(map[string]any) (any, error) { cancel(); return "stopped", nil }}, + }) + change := func(id int, watch any, key, op string) { + rt.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, + "change": map[string]any{"state": "mesh-controller.assignments", "key": key, "op": op, "value": "v"}}}) + if r := rt.next(); r["id"] != float64(id) || r["error"] != nil { + t.Fatalf("a change was answered %v", r) + } + } + rt.call(1, "start", nil) + w := rt.next() + rt.send(map[string]any{"jsonrpc": "2.0", "id": w["id"], "result": map[string]any{}}) + rt.next() + + change(50, w["id"], "before", "put") + if got := <-handled; got != `before="v"` { + t.Fatalf("handled %q", got) + } + rt.call(2, "stop", nil) + rt.next() + change(51, w["id"], "after", "put") + change(52, w["id"], "after", "delete") + select { + case got := <-handled: + t.Fatalf("the handler was called after its context ended: %q", got) + case <-time.After(100 * time.Millisecond): + } + }) +}