From fc66555ecc80050d453f3bc850e7e2570141cbd1 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:25:12 +0100 Subject: [PATCH] Keep the client filling secrets and reading the controller's assignments until the lease write; plans for held modules; warrant says yes Review round 3 (issue 1954): the node-engine of today writes ${secret:} as given and the new one fills only provision keys, so the client fills secrets this release; plans exist only for modules with a want, so a module without a seat plan is read from the controller's assignment and never leaves for want of one, and the holder composes held modules at the build they run; the SDK's provider lets go only on "yes". --- modules/mesh-provisioner-client/README.md | 31 +- .../bound_gitea_test.go | 2 +- .../cmd/mesh-provisioner-client/client.go | 84 ++++-- .../mesh-provisioner-client/client_test.go | 40 ++- .../drift_integration_test.go | 30 +- .../moved_forge_test.go | 9 +- .../cmd/mesh-provisioner-client/plan.go | 4 +- .../mesh-provisioner-client/pulled_in_test.go | 8 +- .../cmd/mesh-provisioner-client/secrets.go | 23 ++ .../mesh-provisioner-client/secrets_test.go | 276 ++++++++++++++++-- .../testdata/accounts.conf | 2 +- .../cmd/mesh-provisioner/holder.go | 39 ++- .../cmd/mesh-provisioner/holder_test.go | 28 ++ 13 files changed, 465 insertions(+), 111 deletions(-) diff --git a/modules/mesh-provisioner-client/README.md b/modules/mesh-provisioner-client/README.md index b4188388..7c62a365 100644 --- a/modules/mesh-provisioner-client/README.md +++ b/modules/mesh-provisioner-client/README.md @@ -4,24 +4,35 @@ Holds the `node-provisioner` seat on every node: the provisioner's client (novox P5). Go, one binary (`cmd/mesh-provisioner-client`), run as a service of its own under the account `mesh-provisioner-client`, outside the tool runner, on a bus account of its own, `provisioner.`, delivered as its own secret `provisioner-broker` (mesh-controller `internal/broker/provisioner.go`). The bus grants that account -its own machine's keys and, to read alone, the seat mesh-provisioner's whole `plans`, where a provider's consumers on -any machine are bound to it (ADR 0373); nothing else. +its own machine's keys and, to read alone, the seat mesh-provisioner's whole `plans` and the controller's whole +assignments mirror, where a provider's consumers on any machine are bound to it (ADR 0373); nothing else. ## What it does For each module placed on its node — a plan in the seat mesh-provisioner's `plans` (`mesh-provisioner_plans`, -`.`, novox/hq ADR 0570 clause 2), composed there from bus state: the templates archive +`.`, novox/hq ADR 0570 clause 2), or, for a module the seat holds no plan of, until the controller hands +its lease away, an assignment in `mesh-controller_assignments` whose value carries a `provisioning` plan; a module +leaves only when neither places it: the templates archive by reference and digest, the files it fills (path, template, `owner-at-runtime`, how its program takes a change) and the placement values its templates name — the client: - fetches the templates archive and uses it only when it hashes to the plan's digest, keeping it by digest; - watches each owner's keys for its node: placement in the plan, config in `mesh-config_resolved` (`..`), both named by `mesh-provisioner_registry`; -- leaves every `${secret:}` its plan names (a module's own secret, `provisioning.secrets`, or the credential - of what it requires, a `provisioning.provisions` entry with a `name`) as the placeholder, for the node-engine to - fill when it writes the file (novox/hq ADR 0570 clause 6): the client never asks the vault for it and never holds the - value. A secret with a `path` is handed as that file alone, holding the placeholder. A secret the plan does not name - fails the module; +- asks the vault (`mesh-vault.get`, named by the registry) for each of the module's own secrets its plan names + (`provisioning.secrets`), in its own name on the subject naming its account, with the public half of a key it + made at start and keeps in memory alone; the vault answers the secret sealed to that key (ADR 0206's box), so only + the box crosses the bus. A secret is asked once per run, and again when the vault says `secret-changed` for it, + so a rotation reaches its file without a new declaration. A secret with a `path` is written to that file alone, + as an `owned` file. A secret the vault does not hold yet, or a vault that does not answer, is waited for; a + refusal fails the module with the vault's reason and is asked again after ten minutes, or at once when the + module's assignment changes or the node-engine touches its write socket, either of which may end it. A value is never written + to a bucket, a report or a log; +- asks the vault the same way for each credential of what the module requires that its plan names + (a `provisioning.provisions` entry with a `name`, issue 757; every entry names the provider serving that + provision, issue 760, and its offer's credential form, issue 768, which the vault acts on): the value a provider's software issued for this module on this machine, + `provision////` (the forge's read token for claude-code, issue 737), filled + wherever `${secret:}` names it and written to its `path` alone, and asked again on its `secret-changed`; - fills each item the module contributes to a collection (`provisioning.items`: the collection and its template) from the same owners as a file, and records it in `mesh-provisioner_collect`, `..` (a character a key's token cannot hold written `=`: `shell:zsh:normal` is `shell=3Azsh=3Anormal`). An item @@ -50,10 +61,10 @@ and the placement values its templates name — the client: - when a module is taken off the node, removes its items and its report (the node-engine removes the files its declaration no longer names); and when the bus comes back, or the client starts, fills from the current values; - fills a provider's `${consumers:}` (novox/hq ADR 0373, issue 679) with one entry per consumer bound to - it for that provision, in the order of the plans' keys and, within a plan, of its `provisions` entries: a + it for that provision, in the order of the assignments' keys and, within a plan, of its `provisions` entries: a JSON list, each entry `{consumer, module, local, credential, value}`, the credential its own, `pair////[/]`, asked of the vault in the client's own name as a secret - is. Who binds a provider is read from every machine's plans, so the client watches the whole of `plans` + is. Who binds a provider is read from every machine's plans, so the client watches the whole assignments mirror (it carries no secret's value) and reads only its own machine's plans as its own. A consumer whose credential the vault does not hold yet, or will not hand, is left out and said in the log, so the others are served; the provider's helper waits for it (mesh-sdk `go/provider` ConsumersFile). It is filled again when a consumer binds or diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go index dc564cfd..9794aa39 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go @@ -65,7 +65,7 @@ func TestNatsAHandedGiteasServerEnvIsFilledAsTheControllerComposedIt(t *testing. `"values":{"at":"10.7.0.1","from":"novox","port":5432},"vault-name":"pair/novox/postgres-database/g14/gitea"}`, "FROM=${bound:postgres-database:from}\n") eventually(t, "server.env is filled to the controller's text", func() bool { - return e.read(t, giteaEnvPath) == giteaBoundExpected("${secret:internal-token}", "${secret:postgres-database}")+"FROM=novox\n" + return e.read(t, giteaEnvPath) == giteaBoundExpected("tok-never-on-the-bus", "pw-never-on-the-bus")+"FROM=novox\n" }) } diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go index 3fb44ab7..e6815a3e 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go @@ -8,13 +8,14 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery" "os" "slices" "sort" "strings" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -43,10 +44,14 @@ import ( const ( // PlansBucket is where each module's provisioning plan on each node is read: the seat mesh-provisioner's // `plans` (novox/hq ADR 0570 clause 2), read as an assignment's value carried it (asAssignment). - PlansBucket = provisions.PlansBucket - ConfigBucket = "mesh-config_resolved" - RegistryBucket = "mesh-provisioner_registry" - AppliedBucket = "mesh-provisioner_applied" + PlansBucket = provisions.PlansBucket + // ControllerAssignmentsBucket is the controller's assignments mirror, read for a module the seat holds no plan of + // until the controller hands its lease away: the holder writes plans only for modules with a want, so a module + // that runs without one (nats on novox) would otherwise lose its files' fill (review of 2026-10-30). + ControllerAssignmentsBucket = "mesh-controller_assignments" + ConfigBucket = "mesh-config_resolved" + RegistryBucket = "mesh-provisioner_registry" + AppliedBucket = "mesh-provisioner_applied" ) // AppliedSubject is where the client says what it applied for one module (ADR 0349 clauses 4 and 15). @@ -127,11 +132,14 @@ type Client struct { type state struct { registry map[string]Answer plans map[string]Plan - refused map[string]string // a module whose plan cannot be acted on, and why - config map[string]map[string]json.RawMessage - applied map[string]bool // module → a report is kept - reported map[string][]byte // module → the last report said, undated - secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere + // seatPlans and controllerAssigned are each source's value by key, as an assignment's value reads: the seat's + // plan wins, and the controller's assignment stands for a module the seat has no plan of. + seatPlans, controllerAssigned map[string][]byte + refused map[string]string // a module whose plan cannot be acted on, and why + config map[string]map[string]json.RawMessage + applied map[string]bool // module → a report is kept + reported map[string][]byte // module → the last report said, undated + secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere // items is every item recorded for this node, by collection and contributor: what a collector is filled from. items map[string]map[string]itemValue // recorded is each item's key and its value as kept: what this client compares before it records again. @@ -173,7 +181,8 @@ func (c *Client) Run(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{PlansBucket, ConfigBucket, RegistryBucket, AppliedBucket, CollectBucket} { + for _, b := range []string{PlansBucket, ControllerAssignmentsBucket, ConfigBucket, RegistryBucket, AppliedBucket, + CollectBucket} { kv, err := c.JS.KeyValue(ctx, b) if err != nil { return fmt.Errorf("the bucket %s: %w", b, err) @@ -211,7 +220,7 @@ func (c *Client) Run(ctx context.Context) error { mine := c.Node + ".>" // The registry whole, and every machine's assignment: a provider here is bound by consumers on any machine, in // their plans (ADR 0373). Every other bucket, this machine's keys alone. - for _, b := range []string{RegistryBucket, PlansBucket} { + for _, b := range []string{RegistryBucket, PlansBucket, ControllerAssignmentsBucket} { if err := watch(b, ">"); err != nil { return err } @@ -306,7 +315,7 @@ func (c *Client) Run(ctx context.Context) error { // Every current value first: nothing is filled from half of them. current := map[string]bool{} var early []*nats.Msg - for len(current) < 5 { // the registry, the plans, config, applied, collect + for len(current) < 6 { // the registry, the plans, the controller's assignments, config, applied, collect select { case <-ctx.Done(): return ctx.Err() @@ -482,7 +491,7 @@ func (c *Client) told(u update) { if module, name, ok := strings.Cut(rest, "."); ok { c.tell(module, Changed{Kind: AskConfig, Key: name}) } - case PlansBucket: + case PlansBucket, ControllerAssignmentsBucket: c.tell(rest, Changed{Kind: AskConfig}) } } @@ -530,16 +539,31 @@ func (c *Client) take(s *state, u update) []string { } // A kind's answer moves what every module waits on. return sortedKeys(s.plans) - case PlansBucket: - value := e.Value() - if !gone { - if v, err := asAssignment(value); err != nil { - logf("[provisioner-client] the plan %s cannot be read, so its module leaves: %v", key, err) - gone = true - } else { - value = v - } + case PlansBucket, ControllerAssignmentsBucket: + if s.seatPlans == nil { + s.seatPlans, s.controllerAssigned = map[string][]byte{}, map[string][]byte{} } + source := s.seatPlans + if u.bucket == ControllerAssignmentsBucket { + source = s.controllerAssigned + } + if gone { + delete(source, key) + } else if u.bucket == ControllerAssignmentsBucket { + source[key] = e.Value() + } else if v, err := asAssignment(e.Value()); err != nil { + logf("[provisioner-client] the plan %s cannot be read, so the controller's assignment is read: %v", key, err) + delete(source, key) + } else { + source[key] = v + } + // The seat's plan where it holds one, the controller's assignment otherwise: a module leaves only when + // neither places it, never because the seat has no plan of it yet. + value, placed := s.seatPlans[key] + if !placed { + value, placed = s.controllerAssigned[key] + } + gone = !placed // A value that is not a standing assignment (ended, or in a state: ADR 0408 clause 4) is read as a removed // key, so the module leaves the machine as it does when the key goes (issue 1098). gone = gone || !provider.Standing(value) @@ -879,14 +903,14 @@ func (c *Client) valueOf(ctx context.Context, s *state, module string, p Plan, n } return collectionText(s.items[collection], col.Comment, where), "", "" case n.Kind == "secret": - // Left for the node-engine, which opens the vault's value and fills it when it writes the file on the - // client's word (ADR 0570 clause 6): the client never holds the value. - if !slices.ContainsFunc(p.Secrets, func(sec PlannedSecret) bool { return sec.Name == n.Name }) && - !slices.ContainsFunc(p.Provisions, func(pr PlannedProvision) bool { return pr.Name == n.Name }) { - return "", "", fmt.Sprintf("%s: the plan names no secret %s, and the node-engine fills only what this "+ - "machine's plan names", n, n.Name) + if !known { + return "", n.String() + " (no domain answers secret in the registry yet)", "" } - return "${secret:" + n.Name + "}", "", "" + if answer.Ask != VaultSeat+"."+GetVerb { + return "", "", fmt.Sprintf("%s: the registry answers secret from %q, which this client does not ask", n, + answer.Ask) + } + return c.secretOf(ctx, s, module, p, n) case n.Kind == ConsumersKind: // A provider's consumers' credentials, each the vault's (ADR 0373). if !known { diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go index 303083ec..fb0734ee 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go @@ -81,7 +81,7 @@ func startBus(t *testing.T) *aBus { t.Fatal(err) } b := &aBus{url: s.ClientURL(), domain: js, conn: nc, kv: map[string]jetstream.KeyValue{}} - for _, name := range []string{PlansBucket, ConfigBucket, RegistryBucket, AppliedBucket, + for _, name := range []string{PlansBucket, ControllerAssignmentsBucket, ConfigBucket, RegistryBucket, AppliedBucket, CollectBucket, delivery.HaveBucket} { kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: name}) if err != nil { @@ -627,6 +627,44 @@ func TestNatsAModuleTakenOffTheNodeLosesItsReport(t *testing.T) { }) } +// Review of 2026-10-30: until the controller hands its lease away, a module the seat holds no plan of is read from +// the controller's assignment (nats on novox runs without a want, so the holder writes it no plan), and a module never +// leaves because its plan is missing: only when neither source places it. +func TestNatsAModuleWithoutASeatPlanIsFilledFromTheControllersAssignmentAndLeavesOnlyWhenNeitherPlacesIt(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned"}) + b.config(t, "dunst", "font-size", "13") + p, err := json.Marshal(dunst(t)) + if err != nil { + t.Fatal(err) + } + assignment := fmt.Sprintf(`{"node":%q,"module":"dunst","assigned":"2026-10-14T20:00:00Z","provisioning":%s}`, node, p) + b.put(t, ControllerAssignmentsBucket, node+".dunst", assignment) + aClient(t, b, e, t.TempDir()) + eventually(t, "dunstrc is filled from the controller's assignment", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 13") + }) + + // The seat's plan comes, and goes again: the module stays, filled, its report kept. + b.assign(t, "dunst", dunst(t)) + b.del(t, PlansBucket, node+".dunst") + b.config(t, "dunst", "font-size", "15") + eventually(t, "dunstrc follows a change with only the controller's assignment", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 15") + }) + if r, _ := heard.last("dunst"); r.Status == StatusRemoved || len(b.keys(t, AppliedBucket, node+".dunst")) != 1 { + t.Fatalf("the module left when its plan went, though the controller still places it: %+v", r) + } + + // Neither places it: it leaves. + b.del(t, ControllerAssignmentsBucket, node+".dunst") + eventually(t, "prov.applied says it is removed", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusRemoved + }) +} + func TestNatsANodeComingBackFillsFromTheCurrentValues(t *testing.T) { b := startBus(t) const zshrc = "/home/nox/.zshrc" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/drift_integration_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/drift_integration_test.go index 0b6c5c78..96f11991 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/drift_integration_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/drift_integration_test.go @@ -5,36 +5,22 @@ package main import ( "context" "encoding/json" - "errors" "os" "path/filepath" "strings" "testing" "time" - - "github.com/nats-io/nats.go/jetstream" ) // 1129/AC5 (integration): the client as main/serve would run it — reading its credential from disk, dialling a // real bus, and asking a real node-engine-shaped socket to write — fills a module once and then asks for no write -// for three seconds: there is no timer that asks again on its own (ADR 0349 clause 4). Its plans are read from the -// seat mesh-provisioner's `plans` under the account the grants part composes, a module's own secret is left as its -// `${secret:}` placeholder for the node-engine with the vault never asked, and no need is recorded (ADR 0570 -// clauses 1, 2, 6). Run in the pull request's one check. +// for three seconds: there is no timer that asks again on its own (ADR 0349 clause 4). Compiled (`go vet -tags +// integration ./...`), never run here: the lead runs it against a real bus through mesh-controller.try. func TestIntegrationServeFillsOnceAndThenAsksForNoWrite(t *testing.T) { b := startBus(t) - vault := startVault(t, b) - vault.keep("own/g14/app/app-token", "tok", false) - vault.keep("own/g14/app/db-password", "pw", false) - e := startEngine(t, map[string]string{dunstrc: "owned", appConf: "owned", appDB: "owned"}) + e := startEngine(t, map[string]string{dunstrc: "owned"}) b.config(t, "dunst", "font-size", "13") b.assign(t, "dunst", dunst(t)) - b.config(t, "app", "font-size", "13") - b.assign(t, "app", app(t)) - needs, err := b.domain.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: "mesh-provisioner_needs"}) - if err != nil { - t.Fatal(err) - } dir := t.TempDir() credential := filepath.Join(dir, "credential.json") @@ -55,10 +41,6 @@ func TestIntegrationServeFillsOnceAndThenAsksForNoWrite(t *testing.T) { }) eventually(t, "dunstrc is filled", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 13") }) - eventually(t, "app.conf is filled but for its secret", func() bool { - return e.read(t, appConf) == "token = ${secret:app-token}\nfont = 13\n" - }) - eventually(t, "the secret's own file holds its placeholder", func() bool { return e.read(t, appDB) == "${secret:db-password}" }) _, asked := e.counts(dunstrc) // Well over any former drift interval (a minute by default, before this issue removed the --drift flag). @@ -67,10 +49,4 @@ func TestIntegrationServeFillsOnceAndThenAsksForNoWrite(t *testing.T) { t.Errorf("the node-engine was asked to write %s again %d time(s) within three seconds, with nothing over "+ "the bus: there is no timer", dunstrc, after-asked) } - if n := vault.times("own/g14/app/app-token") + vault.times("own/g14/app/db-password"); n != 0 { - t.Errorf("the vault was asked %d times for a module's own secrets", n) - } - if keys, err := needs.Keys(context.Background()); !errors.Is(err, jetstream.ErrNoKeysFound) { - t.Errorf("needs recorded: %v (%v)", keys, err) - } } diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go index 278a2a10..9eb14cbb 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go @@ -95,16 +95,11 @@ func TestNatsAMovedGiteaAndClaudeCodeHaveTheirFilesFilledFromTheBus(t *testing.T aClient(t, b, e, t.TempDir()) for path, want := range map[string]string{ - serverEnv: giteaBoundExpected("${secret:internal-token}", "${secret:postgres-database}"), + serverEnv: giteaBoundExpected("tok-never-on-the-bus", "pw-never-on-the-bus"), origins: "https://git.example.org\n", facts: "{\n \"node\": \"g14\",\n \"console\": \"http://127.0.0.1:4270/mcp\"\n}\n", - readCred: "${secret:gitea-read}", + readCred: credential, } { eventually(t, path+" is filled to the controller's text", func() bool { return e.read(t, path) == want }) } - // The secrets are the node-engine's to fill at the write (ADR 0570 clause 6): the vault is not asked for them. - if n := vault.times("own/g14/gitea/internal-token") + vault.times("pair/novox/postgres-database/g14/gitea") + - vault.times(forgeHolder); n != 0 { - t.Errorf("the vault was asked %d times for the modules' own secrets", n) - } } diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go index 1bbbbba2..43cc4dfd 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go @@ -16,8 +16,8 @@ import ( // The provisioner's plan of a module on this node (novox/hq ADR 0349 clause 4, ADR 0570 clause 2): the seat // mesh-provisioner composes it from bus state and keeps it in `plans`, `.`, which this client reads for // its own machine (and every machine's, for a provider's consumers). asAssignment carries a plan into the Assignment -// it acts on. A module placed here with nothing to provision has an empty plan; a module with no plan key is not -// placed here, and its files are the node-engine's, from the declaration. A module only pulled in by another on this +// it acts on. A module the seat holds no plan of is read from the controller's assignment until the controller hands its +// lease away; a module neither places is not placed here, and its files are the node-engine's, from the declaration. A module only pulled in by another on this // node has a plan of its own there: read and filled the same way. // Assignment is one plan as the client acts on it. **Lenient at the top**; **strict in the plan**, which is the diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go index 826853bd..a8b32af5 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go @@ -40,7 +40,7 @@ func TestAPulledInModulesValueIsReadForItsPlan(t *testing.T) { } } -func TestNatsAPulledInModuleIsFilledAndItsItemIsRecorded(t *testing.T) { +func TestNatsAPulledInModuleIsFilledWithItsSecretAndItsItemIsRecorded(t *testing.T) { b := startBus(t) heard := b.listen(t) vault := startVault(t, b) @@ -54,10 +54,10 @@ func TestNatsAPulledInModuleIsFilledAndItsItemIsRecorded(t *testing.T) { b.pullIn(t, "mesh-cli", "desk", contributor(t, "shell:zsh:normal", "alias nox=mesh-cli\n")) aClient(t, b, e, t.TempDir()) - eventually(t, "the pulled-in app.conf is filled, its secret left for the node-engine", func() bool { - return e.read(t, appConf) == "token = ${secret:app-token}\nfont = 13\n" + eventually(t, "the pulled-in app.conf is filled with its secret", func() bool { + return e.read(t, appConf) == "token = "+token+"\nfont = 13\n" }) - eventually(t, "its secret's own file holds it alone", func() bool { return e.read(t, appDB) == "${secret:db-password}" }) + eventually(t, "its secret's own file holds it alone", func() bool { return e.read(t, appDB) == password }) eventually(t, "the pulled-in contributor's item is recorded", func() bool { got := b.keys(t, CollectBucket, node+".") return len(got) == 1 && got[0] == "g14.shell=3Azsh=3Anormal.mesh-cli" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go index 5be64d0f..c8c4149f 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go @@ -160,6 +160,29 @@ type changedSecret struct { Generation int `json:"generation"` } +// secretOf is a module's secret, from what the run holds or from the vault: its value, or what it waits for, or +// why it cannot be filled. The client fills it in this release: the node-engine of today writes a file's text as it +// is handed, and the new one fills only a provision key's reference, never a module's own secret (review of +// 2026-10-30); its fill is an addition, for provision keys, until it fills every secret. +func (c *Client) secretOf(ctx context.Context, s *state, module string, p Plan, n Need) (value, wait, fail string) { + name := "" + if slices.ContainsFunc(p.Secrets, func(sec PlannedSecret) bool { return sec.Name == n.Name }) { + name = VaultName(c.Node, module, n.Name) + } else if i := slices.IndexFunc(p.Provisions, func(pr PlannedProvision) bool { return pr.Name == n.Name }); i >= 0 { + // A credential of what the module requires, issued for this module on this machine (issue 757). + pr := p.Provisions[i] + name = HolderName(pr.Provider, pr.Provision, c.Node, module) + if pr.CredentialForm == "pair" { + // This consumer's own login from a provider that gives each consumer one (issue 779, ADR 0158 §11). + name = PairName(pr.Provider, pr.Provision, c.Node, module, pr.Local) + } + } else { + return "", "", fmt.Sprintf("%s: the plan names no secret %s, and the vault hands only what this machine's "+ + "declaration names", n, n.Name) + } + return c.vaultValue(ctx, s, n, name) +} + // vaultValue is the vault's secret name for the need n, from what the run holds or from the vault: its value, or what // it waits for, or why it cannot be filled. func (c *Client) vaultValue(ctx context.Context, s *state, n Need, name string) (value, wait, fail string) { diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go index 2e8d7805..5c48fb45 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "strings" "sync" @@ -148,41 +149,157 @@ func app(t *testing.T) Plan { Secrets: []PlannedSecret{{Name: "app-token"}, {Name: "db-password", Path: appDB}}} } -// ADR 0570 clause 6: a template's `${secret:}` placeholder, a secret's own file and a required credential's file are -// handed to the node-engine with the placeholder left in, for it to fill at the write; the client never asks the vault -// for them, so no value reaches it. The template's other values are filled. -func TestNatsAModulesSecretsAreLeftForTheNodeEngineAndTheVaultIsNotAsked(t *testing.T) { +func TestNatsAMovedModulesFileIsFilledWithItsSecretFromTheVault(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + wire := sniff(t, b) + vault := startVault(t, b) + const token, password = "tok-3f9a1c-never-on-the-bus", "pw-77b2e0-never-on-the-bus" + vault.keep("own/g14/app/app-token", token, false) + vault.keep("own/g14/app/db-password", password, false) + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir()) + + eventually(t, "app.conf is filled with its secret", func() bool { + return e.read(t, appConf) == "token = "+token+"\nfont = 13\n" + }) + eventually(t, "the secret's own file holds it alone", func() bool { return e.read(t, appDB) == password }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("app") + return r.Status == StatusApplied && len(r.Files) == 2 + }) + if wire.heard() == 0 { + t.Fatal("the sniffer heard nothing: it proves nothing") + } + for _, value := range []string{token, password} { + if said := wire.carrying(value); len(said) > 0 { + t.Errorf("the value crossed the bus in the clear: %q", said) + } + } + if vault.times("own/g14/app/app-token") != 1 || vault.times("own/g14/app/db-password") != 1 { + t.Errorf("the vault was asked %d and %d times, want once each", vault.times("own/g14/app/app-token"), + vault.times("own/g14/app/db-password")) + } +} + +func TestNatsASecretRotatedInTheVaultReachesItsFileWithoutANewDeclaration(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + vault.keep("own/g14/app/app-token", "first", false) + vault.keep("own/g14/app/db-password", "pw", false) + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir()) + eventually(t, "app.conf is filled", func() bool { return strings.Contains(e.read(t, appConf), "token = first\n") }) + assigned, err := b.kv[PlansBucket].Get(context.Background(), node+".app") + if err != nil { + t.Fatal(err) + } + + vault.keep("own/g14/app/app-token", "second", true) + eventually(t, "the rotated value reaches its file", func() bool { + return strings.Contains(e.read(t, appConf), "token = second\n") + }) + eventually(t, "prov.applied says it was updated", func() bool { + r, _ := heard.last("app") + return r.Status == StatusApplied && r.Files[0].Action == "updated" + }) + now, err := b.kv[PlansBucket].Get(context.Background(), node+".app") + if err != nil || now.Revision() != assigned.Revision() { + t.Errorf("the module's assignment was written again: revision %d, was %d", now.Revision(), assigned.Revision()) + } + if moved, _ := e.counts(appConf); moved != 2 { + t.Errorf("app.conf moved %d times, want 2: the program restarts once for the rotation", moved) + } + if moved, _ := e.counts(appDB); moved != 1 { + t.Errorf("the other secret's file moved %d times: only what changed is written", moved) + } + // What another machine's secret, or a secret no plan here names, changes is no reason to ask. + vault.keep("own/ace/app/app-token", "elsewhere", true) + vault.keep("own/g14/other/token", "unplanned", true) + time.Sleep(300 * time.Millisecond) + if n := vault.times("own/g14/app/app-token"); n != 2 { + t.Errorf("the vault was asked for app-token %d times, want 2", n) + } + if vault.times("own/ace/app/app-token")+vault.times("own/g14/other/token") != 0 { + t.Error("a change of a secret this machine's plans do not name made the client ask") + } +} + +func TestNatsASecretTheVaultDoesNotHoldYetIsWaitedFor(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + vault.keep("own/g14/app/db-password", "pw", false) + vault.allowed["own/g14/app/app-token"] = true // the plan names it; the vault does not hold it yet + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir()) + + eventually(t, "prov.applied says it waits for the secret", func() bool { + r, _ := heard.last("app") + return r.Status == StatusWaiting && len(r.WaitsFor) == 1 && + strings.HasPrefix(r.WaitsFor[0], "secret:app-token (the vault holds no") + }) + if _, asked := e.counts(appConf); asked != 0 { + t.Error("a file waiting for its secret was written") + } + // 1553: a file whose own values are all there is filled, though another file of the module waits. + eventually(t, "the secret's own file is filled while the template waits", func() bool { return e.read(t, appDB) == "pw" }) + vault.keep("own/g14/app/app-token", "arrived", true) + eventually(t, "it is filled once the vault holds it", func() bool { + return strings.Contains(e.read(t, appConf), "token = arrived\n") && e.read(t, appDB) == "pw" + }) +} + +// 1129/AC4: a secret the vault refuses is asked for once and not again while nothing about it changes, even +// though several unrelated changes over the bus each reconcile the module again (never a timer). +func TestNatsARefusalFailsTheModuleWithTheVaultsReasonAndIsNotAskedEveryPass(t *testing.T) { b := startBus(t) heard := b.listen(t) vault := startVault(t, b) vault.keep("own/g14/app/app-token", "tok", false) vault.keep("own/g14/app/db-password", "pw", false) - vault.keep(forgeHolder, "forge", false) - e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned", forgeFile: "owned", settingsFile: "owned"}) + vault.refuse = "provisioner.g14 may not ask for own/g14/app/app-token: app's plan on g14 names no secret app-token" + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) b.config(t, "app", "font-size", "13") b.assign(t, "app", app(t)) - b.assign(t, "claude-code", claudeCode(t)) aClient(t, b, e, t.TempDir()) - eventually(t, "app.conf is filled but for its secret", func() bool { - return e.read(t, appConf) == "token = ${secret:app-token}\nfont = 13\n" - }) - eventually(t, "the secret's own file holds its placeholder alone", func() bool { - return e.read(t, appDB) == "${secret:db-password}" - }) - eventually(t, "the credential's file holds its placeholder alone", func() bool { - return e.read(t, forgeFile) == "${secret:gitea-read}" - }) - eventually(t, "the template naming the credential keeps its placeholder", func() bool { - return e.read(t, settingsFile) == `{"forge":"${secret:gitea-read}"}` - }) - eventually(t, "prov.applied says applied", func() bool { + eventually(t, "prov.applied says it failed, with the vault's reason", func() bool { r, _ := heard.last("app") - return r.Status == StatusApplied && len(r.Files) == 2 + return r.Status == StatusFailed && strings.Contains(r.Reason, "names no secret app-token") }) - if n := vault.times("own/g14/app/app-token") + vault.times("own/g14/app/db-password") + vault.times(forgeHolder); n != 0 { - t.Errorf("the vault was asked %d times for a module's own secrets, want none", n) + // Several changes reach the module, each a reason to reconcile it again. + for _, size := range []string{"14", "15", "16"} { + b.config(t, "app", "font-size", size) + time.Sleep(100 * time.Millisecond) } + time.Sleep(200 * time.Millisecond) + if n := vault.times("own/g14/app/app-token"); n != 1 { + t.Errorf("a refused secret was asked %d times across several reconciles, want once", n) + } + if _, asked := e.counts(appConf); asked != 0 { + t.Error("a file whose secret was refused was written") + } +} + +func TestNatsAVaultThatDoesNotAnswerIsWaitedFor(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir()) + eventually(t, "prov.applied says it waits for the vault", func() bool { + r, _ := heard.last("app") + return r.Status == StatusWaiting && len(r.WaitsFor) == 2 && strings.Contains(r.WaitsFor[0], "nothing answers") + }) } // The plan's secrets are read strictly: a name that is not a secret's, one named twice, and a file path that is not @@ -230,6 +347,71 @@ func claudeCode(t *testing.T) Plan { Path: forgeFile}}} } +func TestNatsAModulesRequiredCredentialIssuedForItIsFilledFromTheVaultAndFollowsARotation(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + wire := sniff(t, b) + vault := startVault(t, b) + const first = "https://agent-g14:gta-first-never-on-the-bus@forge.example" + const second = "https://agent-g14:gta-second-never-on-the-bus@forge.example" + vault.keep(forgeHolder, first, false) + e := startEngine(t, map[string]string{forgeFile: "owned", settingsFile: "owned"}) + b.assign(t, "claude-code", claudeCode(t)) + aClient(t, b, e, t.TempDir()) + + eventually(t, "gitea-read's file holds the value issued for claude-code here, alone", func() bool { + return e.read(t, forgeFile) == first + }) + eventually(t, "the template naming it is filled with it", func() bool { + return e.read(t, settingsFile) == `{"forge":"`+first+`"}` + }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("claude-code") + return r.Status == StatusApplied && len(r.Files) == 2 + }) + vault.keep(forgeHolder, second, true) + eventually(t, "a value the forge issued anew reaches the file without a new declaration", func() bool { + return e.read(t, forgeFile) == second + }) + // Another machine's or another module's holder value changing is no reason to ask. + vault.keep("provision/novox/gitea-read/ace/claude-code", "elsewhere", true) + vault.keep("provision/novox/gitea-read/"+node+"/zsh", "unplanned", true) + time.Sleep(300 * time.Millisecond) + if n := vault.times(forgeHolder); n != 2 { + t.Errorf("the vault was asked for the holder value %d times, want 2", n) + } + if vault.times("provision/novox/gitea-read/ace/claude-code")+vault.times("provision/novox/gitea-read/"+node+"/zsh") != 0 { + t.Error("a change of a value no plan here names made the client ask") + } + if wire.heard() == 0 { + t.Fatal("the sniffer heard nothing: it proves nothing") + } + for _, value := range []string{first, second, "gta-first-never-on-the-bus", "gta-second-never-on-the-bus"} { + if said := wire.carrying(value); len(said) > 0 { + t.Errorf("the value crossed the bus in the clear: %q", said) + } + } +} + +// A required credential the vault refuses fails the module with the vault's reason, and nothing is written. +func TestNatsARequiredCredentialTheVaultRefusesFailsTheModule(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + vault.keep(forgeHolder, "https://agent-g14:tok@forge.example", false) + vault.refuse = "provisioner.g14 may not ask for " + forgeHolder + ": claude-code's plan on g14 names no credential of gitea-read from novox" + e := startEngine(t, map[string]string{forgeFile: "owned", settingsFile: "owned"}) + b.assign(t, "claude-code", claudeCode(t)) + aClient(t, b, e, t.TempDir()) + eventually(t, "prov.applied says it failed, with the vault's reason", func() bool { + r, _ := heard.last("claude-code") + return r.Status == StatusFailed && strings.Contains(r.Reason, "names no credential of gitea-read") + }) + if _, asked := e.counts(forgeFile); asked != 0 { + t.Error("a file whose credential was refused was written") + } +} + // The plan's provisions are read strictly. func TestAPlansProvisionsAreReadStrictly(t *testing.T) { base := func() Plan { @@ -280,6 +462,50 @@ func nextcloud(t *testing.T) Plan { ProviderModule: "postgres", CredentialForm: "pair", Path: dbFile}}} } +// A consumer's own login from a provider that gives each consumer one (issue 779) is the vault's pair row for this +// module on this machine: its secrets file holds it alone, and it is asked for again when the vault says it changed. +func TestNatsAModulesOwnLoginFromAPerLoginProviderIsFilledFromTheVaultAndFollowsARotation(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + wire := sniff(t, b) + vault := startVault(t, b) + const first, second = "login-first-never-on-the-bus", "login-second-never-on-the-bus" + vault.keep(dbHolder, first, false) + e := startEngine(t, map[string]string{dbFile: "owned", dbConf: "owned"}) + b.assign(t, "nextcloud", nextcloud(t)) + aClient(t, b, e, t.TempDir()) + + eventually(t, "the secrets file holds the module's own login, alone", func() bool { + return e.read(t, dbFile) == first + }) + eventually(t, "the template naming it is filled with it", func() bool { + return e.read(t, dbConf) == "dbpass="+first + }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("nextcloud") + return r.Status == StatusApplied && len(r.Files) == 2 + }) + vault.keep(dbHolder, second, true) + eventually(t, "a login rotated in the vault reaches the file without a new declaration", func() bool { + return e.read(t, dbFile) == second + }) + // Another consumer's login changing is no reason to ask. + other := "pair/novox/postgres-database/ace/nextcloud" + vault.keep(other, "elsewhere", true) + time.Sleep(300 * time.Millisecond) + if n := vault.times(dbHolder); n != 2 { + t.Errorf("the vault was asked for the login %d times, want 2", n) + } + if vault.times(other) != 0 { + t.Error("a change of another consumer's login made the client ask") + } + for _, value := range []string{first, second} { + if said := wire.carrying(value); len(said) > 0 { + t.Errorf("the login crossed the bus in the clear: %q", said) + } + } +} + func TestAConsumersLoginIsNamedWithItsLocalName(t *testing.T) { if got := PairName("novox", "postgres-database", "g14", "umami", "stats"); got != "pair/novox/postgres-database/g14/umami/stats" { t.Errorf("a login under a local name: %s", got) @@ -299,7 +525,7 @@ func TestNatsAKnownSecretsFileIsFilledWhileATemplateWaits1553(t *testing.T) { b.assign(t, "app", app(t)) aClient(t, b, e, t.TempDir()) - eventually(t, "the secret's file is filled", func() bool { return e.read(t, appDB) == "${secret:db-password}" }) + eventually(t, "the secret's file is filled", func() bool { return e.read(t, appDB) == "pw" }) eventually(t, "prov.applied says waiting for the value, with the file it filled", func() bool { r, _ := heard.last("app") return r.Status == StatusWaiting && len(r.WaitsFor) == 1 && strings.HasPrefix(r.WaitsFor[0], "config:font-size") && @@ -323,7 +549,7 @@ func TestNatsATemplateWithAWaitingValueKeepsItsFileAsItIs1553(t *testing.T) { aClient(t, b, e, t.TempDir()) eventually(t, "it waits", func() bool { r, _ := heard.last("app"); return r.Status == StatusWaiting }) - eventually(t, "the other file is filled, so the pass is over", func() bool { return e.read(t, appDB) == "${secret:db-password}" }) + eventually(t, "the other file is filled, so the pass is over", func() bool { return e.read(t, appDB) == "pw" }) time.Sleep(200 * time.Millisecond) if _, asked := e.counts(appConf); asked != 0 { t.Errorf("Fill was called %d times for a template with a waiting value", asked) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf index 37551ed3..783bf836 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf @@ -11,7 +11,7 @@ accounts { users = [ { user: "domain", password: "x" } { user: "provisioner.g14", password: "x", permissions: { - publish: { allow: ["$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_applied.*.$KV.mesh-provisioner_applied.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_collect.*.$KV.mesh-provisioner_collect.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.API.CONSUMER.CREATE.KV_postgres_bound.*.$KV.postgres_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-adapter_bound.*.$KV.route-adapter_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-proxy_bound.*.$KV.route-proxy_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_node-tools_bound.*.$KV.node-tools_bound.g14.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_applied.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_collect.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_postgres_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-adapter_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-proxy_bound.>", "$JS.API.CONSUMER.DELETE.KV_node-tools_bound.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_applied.$KV.mesh-provisioner_applied.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_collect.$KV.mesh-provisioner_collect.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>", "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-adapter_bound.$KV.route-adapter_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-proxy_bound.$KV.route-proxy_bound.g14.>", "$JS.API.DIRECT.GET.KV_node-tools_bound.$KV.node-tools_bound.g14.>", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", "$JS.API.STREAM.INFO.KV_mesh-provisioner_plans", "$JS.API.STREAM.INFO.KV_mesh-provisioner_applied", "$JS.API.STREAM.INFO.KV_mesh-provisioner_collect", "$JS.API.STREAM.INFO.KV_mesh-provisioner_registry", "$JS.API.STREAM.INFO.KV_postgres_bound", "$JS.API.STREAM.INFO.KV_route-adapter_bound", "$JS.API.STREAM.INFO.KV_route-proxy_bound", "$JS.API.STREAM.INFO.KV_node-tools_bound", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-provisioner_plans.>", "$JS.FC.KV_mesh-provisioner_applied.>", "$JS.FC.KV_mesh-provisioner_collect.>", "$JS.FC.KV_mesh-provisioner_registry.>", "$JS.FC.KV_postgres_bound.>", "$JS.FC.KV_route-adapter_bound.>", "$JS.FC.KV_route-proxy_bound.>", "$JS.FC.KV_node-tools_bound.>", "$KV.mesh-provisioner_applied.g14.>", "$KV.mesh-provisioner_collect.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.applied.g14.>", "prov.changed.g14.>"] } + publish: { allow: ["$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_applied.*.$KV.mesh-provisioner_applied.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_collect.*.$KV.mesh-provisioner_collect.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.API.CONSUMER.CREATE.KV_postgres_bound.*.$KV.postgres_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-adapter_bound.*.$KV.route-adapter_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-proxy_bound.*.$KV.route-proxy_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_node-tools_bound.*.$KV.node-tools_bound.g14.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.DELETE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_applied.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_collect.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_postgres_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-adapter_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-proxy_bound.>", "$JS.API.CONSUMER.DELETE.KV_node-tools_bound.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.>", "$JS.API.DIRECT.GET.KV_mesh-controller_assignments.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_applied.$KV.mesh-provisioner_applied.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_collect.$KV.mesh-provisioner_collect.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>", "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-adapter_bound.$KV.route-adapter_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-proxy_bound.$KV.route-proxy_bound.g14.>", "$JS.API.DIRECT.GET.KV_node-tools_bound.$KV.node-tools_bound.g14.>", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", "$JS.API.STREAM.INFO.KV_mesh-provisioner_plans", "$JS.API.STREAM.INFO.KV_mesh-controller_assignments", "$JS.API.STREAM.INFO.KV_mesh-provisioner_applied", "$JS.API.STREAM.INFO.KV_mesh-provisioner_collect", "$JS.API.STREAM.INFO.KV_mesh-provisioner_registry", "$JS.API.STREAM.INFO.KV_postgres_bound", "$JS.API.STREAM.INFO.KV_route-adapter_bound", "$JS.API.STREAM.INFO.KV_route-proxy_bound", "$JS.API.STREAM.INFO.KV_node-tools_bound", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-provisioner_plans.>", "$JS.FC.KV_mesh-controller_assignments.>", "$JS.FC.KV_mesh-provisioner_applied.>", "$JS.FC.KV_mesh-provisioner_collect.>", "$JS.FC.KV_mesh-provisioner_registry.>", "$JS.FC.KV_postgres_bound.>", "$JS.FC.KV_route-adapter_bound.>", "$JS.FC.KV_route-proxy_bound.>", "$JS.FC.KV_node-tools_bound.>", "$KV.mesh-provisioner_applied.g14.>", "$KV.mesh-provisioner_collect.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.applied.g14.>", "prov.changed.g14.>"] } subscribe: { allow: ["_INBOX.provisioner.g14.>", "mesh.mod.mesh-vault.event.secret-changed", "prov.ask.g14"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go index 961c60f8..c8a8b357 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -103,6 +103,13 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) if err != nil { return out, err } + // A module held on the node with no want (the bus on the control-node) is composed at the build its have says it + // runs, so it has a plan too and its client keeps filling it (review of 2026-10-30). + for module, fp := range h.runningOn(node) { + if _, ok := wanted[module]; !ok { + wanted[module] = fp + } + } atWants, err := h.Compose(ctx, node, wanted) if err != nil { return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err) @@ -266,6 +273,30 @@ func (h *Holder) wantedOn(node string) (map[string]string, error) { return out, nil } +// runningOn is the fingerprint each module's have on a node says it runs, by module: what a module held without a want +// is composed at. +func (h *Holder) runningOn(node string) map[string]string { + out := map[string]string{} + keys, err := h.Store.Keys(HaveRead) + if err != nil { + return out + } + for _, k := range keys { + n, module, isNode, err := delivery.ParseHaveKey(k) + if err != nil || isNode || n != node { + continue + } + e, err := h.Store.Get(HaveRead, k) + if err != nil || e == nil { + continue + } + if have, err := delivery.DecodeHave(e.Value); err == nil && have.FP != "" { + out[module] = have.FP + } + } + return out +} + // writePlans writes each plan that differs and removes the node's plans of modules no longer composed there. func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error { var problems []error @@ -589,10 +620,12 @@ func (h *Holder) retired(key string, p provisions.Provision) (bool, string) { // ---- the retiring ask ---------------------------------------------------------------------------------------------- -// The options a retiring ask offers. +// The options a retiring ask offers. The chosen one is the warrant's option in the key, and a provider lets its entry go +// only on "yes" (mesh-sdk provider.go, the pass's retiring keys), so the approving option is "yes" (review of +// 2026-10-30: the holder wrote "approve", which no provider acted on). const ( - OptionApprove = "approve" - OptionDecline = "decline" + OptionApprove = "yes" + OptionDecline = "no" ) // askLasts is how long a retiring ask stays open. diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go index 0f3c5609..238b589c 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -290,6 +290,10 @@ func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") { t.Fatalf("the warrant is not in the key: %+v", w) } + // The word the SDK's provider lets an entry go on (provider.go): any other, and no provider ever retires it. + if w := m.state(t, pg).Warrant; w.Option != "yes" { + t.Fatalf("the warrant says %q, and a provider lets go only on \"yes\"", w.Option) + } if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring { t.Errorf("retired while the provider's entry stands: %q", said) } @@ -310,6 +314,30 @@ func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { } } +// Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build +// its have says it runs, so the holder writes it a plan and its client keeps filling it. +func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) { + m := newMesh() + fp := "src2:" + strings.Repeat("dd", 32) + m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`) + m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`) + var keptAt map[string]string + m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, error) { + if at != nil { + keptAt = at + return map[string]catalogue.Plan{"nats": {}}, nil + } + return nil, nil + } + c := m.compose(t, "novox") + if keptAt["nats"] != fp || len(keptAt) != 1 { + t.Fatalf("composed at %v, want nats at the build it runs and nothing of another node", keptAt) + } + if !slices.Contains(c.Plans, "novox.nats") { + t.Fatalf("no plan written for a module held without a want: %+v", c) + } +} + // The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1). func TestTheHolderHoldsNoTimer(t *testing.T) { files, _ := filepath.Glob("*.go")