diff --git a/modules/mesh-provisioner-client/README.md b/modules/mesh-provisioner-client/README.md index 1e68dc03..e4382024 100644 --- a/modules/mesh-provisioner-client/README.md +++ b/modules/mesh-provisioner-client/README.md @@ -25,7 +25,8 @@ and the placement values its templates name — the client: the box crosses the bus. A secret is asked once per run, and again when the vault says `secret-changed` for it, so a rotation reaches its file without a new declaration. A secret with a `path` is written to that file alone, as an `owned` file. A secret the vault does not hold yet, or a vault that does not answer, is waited for; a - refusal fails the module with the vault's reason and is asked again after ten minutes. A value is never written + refusal fails the module with the vault's reason and is asked again after ten minutes, or at once when the + module's assignment changes or the node-engine touches its write socket, either of which may end it. A value is never written to a bucket, a report or a log; - asks the vault the same way for each credential of what the module requires that its plan names (a `provisioning.provisions` entry with a `name`, issue 757; every entry names the provider serving that @@ -49,6 +50,10 @@ and the placement values its templates name — the client: - has the node-engine write each changed file through its local write socket (`/run/nox-mesh-host/fill.sock`, mesh-host `internal/filling`), which writes it as the declaration says (an `owned` file whole, a `seed` only when absent, a `managed` file's own keys or block) and restarts or reloads its program as the declaration says; +- fills every module again when the node-engine makes its write socket (a new engine, issue 1165) or touches it, which + it does after an apply whose declaration may let through a write it refused or failed (a path now named as filled, + the provisioner's account now named, a write that failed; mesh-host `internal/filling` Knock): a file refused because + the declaration did not name it yet is written once the declaration does, and the journal says each such retry; - writes again on every change it watches and when its node comes back, never on a timer (novox/hq ADR 0349 clause 4): every file of the module goes through the node-engine again, so an `owned` file changed on disk is put back; - reports `prov.applied..` and keeps the same report in `mesh-provisioner_applied` (`.`): diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go index 844aa47e..7901e9f6 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go @@ -44,6 +44,48 @@ func TestARefusedSecretIsAskedAgainAfterRefusedForAndFilledWhenTheVaultHasIt(t * if fake.asked != 2 || v != "tok" || wait != "" || fail != "" { t.Fatalf("after RefusedFor: %d asks, value %q, waits %q, fails %q; want asked again and filled", fake.asked, v, wait, fail) } + + // What ends a refusal before RefusedFor asks again at once (ace, 2026-10-25 21:17: refused before the declaration + // named the module's files): the node-engine touching its write socket after an apply, which fills every module + // again, and the module's own assignment changing. Another module's assignment does not. + own := VaultName("g14", "claude-code", "token") + for _, row := range []struct { + name string + end func(c *Client, s *state) []string + asked bool + filled []string // the modules a wake fills again + }{ + {name: "the socket touched", asked: true, filled: []string{"claude-code", "gitea", "waited"}, + end: func(c *Client, s *state) []string { return c.woken(s) }}, + {name: "its assignment changed", asked: true, + end: func(c *Client, s *state) []string { c.forgetRefusals(s, "claude-code"); return nil }}, + {name: "another module's assignment changed", asked: false, + end: func(c *Client, s *state) []string { c.forgetRefusals(s, "gitea"); return nil }}, + } { + t.Run(row.name, func(t *testing.T) { + refused := errors.New("the vault refused it: g14's declaration names no such secret") + fake := &scriptedSecrets{errs: []error{refused, refused}} + c := &Client{Node: "g14", Secrets: fake, Now: func() time.Time { return now }} + s := &state{secrets: map[string]heldSecret{}, plans: map[string]Plan{"claude-code": {}, "gitea": {}}, + unplaced: map[string]bool{"waited": true}} + for _, name := range []string{name, own} { + if _, _, fail := c.vaultValue(context.Background(), s, n, name); fail == "" { + t.Fatal("a refusal did not fail the need") + } + } + fake.asked = 0 + fake.errs = nil + if got := row.end(c, s); row.filled != nil && fmt.Sprint(got) != fmt.Sprint(row.filled) { + t.Errorf("filled again %v, want %v", got, row.filled) + } + for _, name := range []string{name, own} { + v, _, _ := c.vaultValue(context.Background(), s, n, name) + if (v == "tok") != row.asked { + t.Errorf("%s: value %q after %d asks; asked again at once: want %v", name, v, fake.asked, row.asked) + } + } + }) + } } func TestANotYetSecretIsAskedAgainOnEveryPass(t *testing.T) { diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go index aecd1e95..e9745779 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go @@ -382,12 +382,7 @@ func (c *Client) Run(ctx context.Context) error { case u.asked != nil: c.respond(s, u.asked) case u.socketUp: - // Every module, not only those whose fill failed: the node-engine that made the socket may have - // applied a composition that left a filled file as the composition had it (issue 1945). - for m := range s.unplaced { - dirty[m] = true - } - for m := range s.plans { + for _, m := range c.woken(s) { dirty[m] = true } case u.bucket == delivery.HaveBucket: @@ -442,6 +437,35 @@ func (c *Client) Run(ctx context.Context) error { func hasPlan(s *state, m string) bool { _, ok := s.plans[m]; return ok } +// woken is what the node-engine's write socket made or touched again fills: every module, not only those whose fill +// was refused or failed. The node-engine makes it when it starts (issue 1165), and touches it after an apply whose +// declaration may let through a write it refused or failed (mesh-host internal/filling Knock; ace, 2026-10-25 21:17: +// mesh-build-go's files were refused 32 s before its want, and nothing filled them after the apply that named them). +// That apply may also have left a filled file as the composition had it (issue 1945). What the vault refused is asked +// again at once: it hands only what the declaration names, which may have changed too. +func (c *Client) woken(s *state) []string { + dropped := c.forgetRefusals(s, "") + out := map[string]bool{} + for m := range s.unplaced { + out[m] = true + } + for m := range s.plans { + out[m] = true + } + all := sortedKeys(out) + logf("[provisioner-client] the node-engine's write socket was made or touched: filling %d module(s) again "+ + "(refused or failed before: %s; the vault's refusals asked again: %d)", len(all), + orNone(sortedKeys(s.unplaced)), len(dropped)) + return all +} + +func orNone(xs []string) string { + if len(xs) == 0 { + return "none" + } + return strings.Join(xs, ", ") +} + // ownComposedOf is the digest of the node's own composition a have `.node` says the node-engine runs; empty for a // deleted key or a value it cannot read. func ownComposedOf(e jetstream.KeyValueEntry) string { @@ -535,6 +559,11 @@ func (c *Client) take(s *state, u update) []string { return append(providers, module) } s.assigned[module] = true + // A new assignment of the module may end what the vault refused of its secrets: asked again at the next fill. + if dropped := c.forgetRefusals(s, module); len(dropped) > 0 { + logf("[provisioner-client] %s: its assignment changed, so the vault is asked again for %s", module, + strings.Join(dropped, ", ")) + } m, p, err := planOf(c.Node, key, e.Value()) switch { case errors.Is(err, errNoPlan): @@ -769,6 +798,8 @@ func (c *Client) reconcile(ctx context.Context, kvs map[string]jetstream.KeyValu // runtime left at the path, cleared by the engine that followed). s.unplaced[module] = true failed = append(failed, fmt.Sprintf("%s: %v", f.Path, err)) + logf("[provisioner-client] %s: %s was not written (%v); it is filled again when the node-engine makes or "+ + "touches its write socket (after an apply that may let it through)", module, f.Path, err) continue } fa.Action, fa.Digest = a.Action, a.Digest diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go index b2bba03c..be7473c3 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go @@ -252,3 +252,39 @@ func (c *Client) secretChanged(s *state, body []byte) []string { delete(s.secrets, ch.Name) return []string{module} } + +// forgetRefusals drops what the vault refused of a module's secrets on this machine (every module's, for module ""), +// so the next fill asks again at once rather than after RefusedFor: the vault hands only what this machine's +// declaration names, so a refusal can end with the module's assignment or with a declaration the node-engine applied +// since (ace, 2026-10-25 21:17: the refusals came before the declaration named the module's files). It answers the +// secrets dropped. +func (c *Client) forgetRefusals(s *state, module string) []string { + var dropped []string + for _, name := range sortedKeys(s.secrets) { + if s.secrets[name].refused == "" { + continue + } + if module != "" && secretModule(c.Node, name) != module { + continue + } + delete(s.secrets, name) + dropped = append(dropped, name) + } + return dropped +} + +// secretModule is the module of this machine a vault name is issued for: own///, +// provision////, pair////[/]; "" for +// none. +func secretModule(node, name string) string { + parts := strings.Split(name, "/") + switch { + case len(parts) == 4 && parts[0] == "own" && parts[1] == node: + return parts[2] + case len(parts) == 5 && parts[0] == "provision" && parts[3] == node: + return parts[4] + case (len(parts) == 5 || len(parts) == 6) && parts[0] == "pair" && parts[3] == node: + return parts[4] + } + return "" +}