Merge pull request 'mesh-provisioner-client: retry a refused write on the node-engine's wake and ask the vault again at once (issue 924)' (#482) from fix/provisioner-retries-on-declaration into main

modules/mesh-provisioner-client/README.md

@@ -25,7 +25,8 @@ and the placement values its templates name — the client:
   the box crosses the bus. A secret is asked once per run, and again when the vault says `secret-changed` for it,
   so a rotation reaches its file without a new declaration. A secret with a `path` is written to that file alone,
   as an `owned` file. A secret the vault does not hold yet, or a vault that does not answer, is waited for; a
-  refusal fails the module with the vault's reason and is asked again after ten minutes. A value is never written
+  refusal fails the module with the vault's reason and is asked again after ten minutes, or at once when the
+  module's assignment changes or the node-engine touches its write socket, either of which may end it. A value is never written
   to a bucket, a report or a log;
 - asks the vault the same way for each credential of what the module requires that its plan names
   (a `provisioning.provisions` entry with a `name`, issue 757; every entry names the provider serving that
@@ -49,6 +50,10 @@ and the placement values its templates name — the client:
 - has the node-engine write each changed file through its local write socket (`/run/nox-mesh-host/fill.sock`,
   mesh-host `internal/filling`), which writes it as the declaration says (an `owned` file whole, a `seed` only when
   absent, a `managed` file's own keys or block) and restarts or reloads its program as the declaration says;
+- fills every module again when the node-engine makes its write socket (a new engine, issue 1165) or touches it, which
+  it does after an apply whose declaration may let through a write it refused or failed (a path now named as filled,
+  the provisioner's account now named, a write that failed; mesh-host `internal/filling` Knock): a file refused because
+  the declaration did not name it yet is written once the declaration does, and the journal says each such retry;
 - writes again on every change it watches and when its node comes back, never on a timer (novox/hq ADR 0349 clause
   4): every file of the module goes through the node-engine again, so an `owned` file changed on disk is put back;
 - reports `prov.applied.<node>.<module>` and keeps the same report in `mesh-provisioner_applied` (`<node>.<module>`):

modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go

@@ -44,6 +44,48 @@ func TestARefusedSecretIsAskedAgainAfterRefusedForAndFilledWhenTheVaultHasIt(t *
 	if fake.asked != 2 || v != "tok" || wait != "" || fail != "" {
 		t.Fatalf("after RefusedFor: %d asks, value %q, waits %q, fails %q; want asked again and filled", fake.asked, v, wait, fail)
 	}
+
+	// What ends a refusal before RefusedFor asks again at once (ace, 2026-10-25 21:17: refused before the declaration
+	// named the module's files): the node-engine touching its write socket after an apply, which fills every module
+	// again, and the module's own assignment changing. Another module's assignment does not.
+	own := VaultName("g14", "claude-code", "token")
+	for _, row := range []struct {
+		name   string
+		end    func(c *Client, s *state) []string
+		asked  bool
+		filled []string // the modules a wake fills again
+	}{
+		{name: "the socket touched", asked: true, filled: []string{"claude-code", "gitea", "waited"},
+			end: func(c *Client, s *state) []string { return c.woken(s) }},
+		{name: "its assignment changed", asked: true,
+			end: func(c *Client, s *state) []string { c.forgetRefusals(s, "claude-code"); return nil }},
+		{name: "another module's assignment changed", asked: false,
+			end: func(c *Client, s *state) []string { c.forgetRefusals(s, "gitea"); return nil }},
+	} {
+		t.Run(row.name, func(t *testing.T) {
+			refused := errors.New("the vault refused it: g14's declaration names no such secret")
+			fake := &scriptedSecrets{errs: []error{refused, refused}}
+			c := &Client{Node: "g14", Secrets: fake, Now: func() time.Time { return now }}
+			s := &state{secrets: map[string]heldSecret{}, plans: map[string]Plan{"claude-code": {}, "gitea": {}},
+				unplaced: map[string]bool{"waited": true}}
+			for _, name := range []string{name, own} {
+				if _, _, fail := c.vaultValue(context.Background(), s, n, name); fail == "" {
+					t.Fatal("a refusal did not fail the need")
+				}
+			}
+			fake.asked = 0
+			fake.errs = nil
+			if got := row.end(c, s); row.filled != nil && fmt.Sprint(got) != fmt.Sprint(row.filled) {
+				t.Errorf("filled again %v, want %v", got, row.filled)
+			}
+			for _, name := range []string{name, own} {
+				v, _, _ := c.vaultValue(context.Background(), s, n, name)
+				if (v == "tok") != row.asked {
+					t.Errorf("%s: value %q after %d asks; asked again at once: want %v", name, v, fake.asked, row.asked)
+				}
+			}
+		})
+	}
 }
 
 func TestANotYetSecretIsAskedAgainOnEveryPass(t *testing.T) {

modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go

@@ -382,12 +382,7 @@ func (c *Client) Run(ctx context.Context) error {
 				case u.asked != nil:
 					c.respond(s, u.asked)
 				case u.socketUp:
-					// Every module, not only those whose fill failed: the node-engine that made the socket may have
+					for _, m := range c.woken(s) {
-					// applied a composition that left a filled file as the composition had it (issue 1945).
-					for m := range s.unplaced {
-						dirty[m] = true
-					}
-					for m := range s.plans {
 						dirty[m] = true
 					}
 				case u.bucket == delivery.HaveBucket:
@@ -442,6 +437,35 @@ func (c *Client) Run(ctx context.Context) error {
 
 func hasPlan(s *state, m string) bool { _, ok := s.plans[m]; return ok }
 
+// woken is what the node-engine's write socket made or touched again fills: every module, not only those whose fill
+// was refused or failed. The node-engine makes it when it starts (issue 1165), and touches it after an apply whose
+// declaration may let through a write it refused or failed (mesh-host internal/filling Knock; ace, 2026-10-25 21:17:
+// mesh-build-go's files were refused 32 s before its want, and nothing filled them after the apply that named them).
+// That apply may also have left a filled file as the composition had it (issue 1945). What the vault refused is asked
+// again at once: it hands only what the declaration names, which may have changed too.
+func (c *Client) woken(s *state) []string {
+	dropped := c.forgetRefusals(s, "")
+	out := map[string]bool{}
+	for m := range s.unplaced {
+		out[m] = true
+	}
+	for m := range s.plans {
+		out[m] = true
+	}
+	all := sortedKeys(out)
+	logf("[provisioner-client] the node-engine's write socket was made or touched: filling %d module(s) again "+
+		"(refused or failed before: %s; the vault's refusals asked again: %d)", len(all),
+		orNone(sortedKeys(s.unplaced)), len(dropped))
+	return all
+}
+
+func orNone(xs []string) string {
+	if len(xs) == 0 {
+		return "none"
+	}
+	return strings.Join(xs, ", ")
+}
+
 // ownComposedOf is the digest of the node's own composition a have `<node>.node` says the node-engine runs; empty for a
 // deleted key or a value it cannot read.
 func ownComposedOf(e jetstream.KeyValueEntry) string {
@@ -535,6 +559,11 @@ func (c *Client) take(s *state, u update) []string {
 			return append(providers, module)
 		}
 		s.assigned[module] = true
+		// A new assignment of the module may end what the vault refused of its secrets: asked again at the next fill.
+		if dropped := c.forgetRefusals(s, module); len(dropped) > 0 {
+			logf("[provisioner-client] %s: its assignment changed, so the vault is asked again for %s", module,
+				strings.Join(dropped, ", "))
+		}
 		m, p, err := planOf(c.Node, key, e.Value())
 		switch {
 		case errors.Is(err, errNoPlan):
@@ -769,6 +798,8 @@ func (c *Client) reconcile(ctx context.Context, kvs map[string]jetstream.KeyValu
 			// runtime left at the path, cleared by the engine that followed).
 			s.unplaced[module] = true
 			failed = append(failed, fmt.Sprintf("%s: %v", f.Path, err))
+			logf("[provisioner-client] %s: %s was not written (%v); it is filled again when the node-engine makes or "+
+				"touches its write socket (after an apply that may let it through)", module, f.Path, err)
 			continue
 		}
 		fa.Action, fa.Digest = a.Action, a.Digest

modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go

@@ -252,3 +252,39 @@ func (c *Client) secretChanged(s *state, body []byte) []string {
 	delete(s.secrets, ch.Name)
 	return []string{module}
 }
+
+// forgetRefusals drops what the vault refused of a module's secrets on this machine (every module's, for module ""),
+// so the next fill asks again at once rather than after RefusedFor: the vault hands only what this machine's
+// declaration names, so a refusal can end with the module's assignment or with a declaration the node-engine applied
+// since (ace, 2026-10-25 21:17: the refusals came before the declaration named the module's files). It answers the
+// secrets dropped.
+func (c *Client) forgetRefusals(s *state, module string) []string {
+	var dropped []string
+	for _, name := range sortedKeys(s.secrets) {
+		if s.secrets[name].refused == "" {
+			continue
+		}
+		if module != "" && secretModule(c.Node, name) != module {
+			continue
+		}
+		delete(s.secrets, name)
+		dropped = append(dropped, name)
+	}
+	return dropped
+}
+
+// secretModule is the module of this machine a vault name is issued for: own/<node>/<module>/<name>,
+// provision/<provider>/<provision>/<node>/<module>, pair/<provider>/<provision>/<node>/<module>[/<local>]; "" for
+// none.
+func secretModule(node, name string) string {
+	parts := strings.Split(name, "/")
+	switch {
+	case len(parts) == 4 && parts[0] == "own" && parts[1] == node:
+		return parts[2]
+	case len(parts) == 5 && parts[0] == "provision" && parts[3] == node:
+		return parts[4]
+	case (len(parts) == 5 || len(parts) == 6) && parts[0] == "pair" && parts[3] == node:
+		return parts[4]
+	}
+	return ""
+}