Fill a provision's ${secret:} at the write from the vault's value sealed to this machine, so the provisioner's client never holds it (issue 1954 L6-E1, E2; ADR 0570 clause 6)

The provisioner's client fills a handed module's file but for each `${secret:<provision>}` (or a whole `secrets` file
that is the placeholder alone).
- The node-engine reads the provision's key `<node>.<consumer>.<provision>[.<local>]` in mesh-provisioner's
  `provisions`. The key must be `provided` and carry the vault's reference.
- It asks the vault's `own` for that pair credential sealed to this machine, at least at the reference's generation,
  and opens it with the node's sealing key.
- It writes the file as declared (its owner), 0600 when the declaration gives no mode.
- Anything short of that refuses the write, naming the provision and why.

A module the node composes itself already has its `secrets` files written so: its composition carries the vault's
value from `own`, written 0600 with the declared owner.

Pins mesh-sdk 4a96793 (go/provisions). Tests: rows in TestAWriteAndAnApplyOfTheSameFileSerialise.

cmd/mesh-host/filling.go

@@ -3,22 +3,25 @@ package main
 import (
 	"context"
 	"crypto/ed25519"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"path/filepath"
 	"sync/atomic"
 
 	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"git.novox.be/novox/mesh-sdk/go/provisions"
 	"github.com/novox/mesh-host/internal/apply"
 	"github.com/novox/mesh-host/internal/filling"
+	"github.com/novox/mesh-host/internal/link"
 	"github.com/novox/mesh-host/internal/store"
 	"github.com/novox/mesh-host/internal/system"
 )
 
 // serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
 // ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
-func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
-	e := fillingEngine(opts, signer, say)
+func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) {
+	e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say)
 	fillingNow.Store(e)
 	defer fillingNow.CompareAndSwap(e, nil)
 	if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
@@ -50,7 +53,10 @@ func knockFilling(after string, say func(string)) {
 // fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
 // the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
 // written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
-func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
+//
+// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a
+// file so filled is written 0600 when the declaration gives no mode.
+func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine {
 	return &filling.Engine{
 		Declared: func() (*declaration.Declaration, error) {
 			return declaredNow(opts.state, signer)
@@ -74,8 +80,13 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
 			if err != nil {
 				return "", "", err
 			}
-			report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
-				announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
+			content, secret, err := secrets.fill(ctx, filledModule(d, path), content)
+			if err != nil {
+				return "", "", fmt.Errorf("%s was not written: %w", path, err)
+			}
+			report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner,
+				announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)),
+				apply.Filled{Secret: secret})
 			// Saved whichever way it went, as an apply's: what was written is on the machine either way.
 			if err := store.Save(opts.state, updated); err != nil {
 				return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
@@ -94,3 +105,39 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
 		},
 	}
 }
+
+// filledModule is the module whose file the declaration declares at path, or "" when none or several do.
+func filledModule(d *declaration.Declaration, path string) string {
+	module := ""
+	for _, r := range d.Resources {
+		if f, ok := r.(*declaration.File); ok && f.Path == path {
+			if module != "" {
+				return ""
+			}
+			module, _ = d.ModuleOfID(f.ID)
+		}
+	}
+	return module
+}
+
+// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key.
+func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets {
+	p := provisionSecrets{node: node, open: sealOpener(statePath)}
+	if queue == nil {
+		return p
+	}
+	p.read = func(ctx context.Context, key string) ([]byte, bool, error) {
+		h, err := queue.KVHistory(ctx, provisions.ProvisionsBucket, key)
+		if err != nil {
+			return nil, false, err
+		}
+		if len(h) == 0 || h[len(h)-1].Deleted {
+			return nil, false, nil
+		}
+		return h[len(h)-1].Value, true, nil
+	}
+	p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
+		return queue.AskVault(ctx, "own", args)
+	}
+	return p
+}

cmd/mesh-host/filling_test.go

@@ -10,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/provisions"
+
 	"github.com/novox/mesh-host/internal/filling"
 	"github.com/novox/mesh-host/internal/store"
 )
@@ -45,7 +47,28 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
 		store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
 		t.Fatal(err)
 	}
-	e := fillingEngine(opts, public, func(string) {})
+	// The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked:
+	// the opener reads the box's "v").
+	provision := map[string]string{}
+	secrets := provisionSecrets{node: "n1",
+		read: func(_ context.Context, key string) ([]byte, bool, error) {
+			v, ok := provision[key]
+			return []byte(v), ok, nil
+		},
+		own: func(_ context.Context, args map[string]any) (json.RawMessage, error) {
+			raw, _ := json.Marshal(args)
+			var a struct{ Credentials []map[string]any }
+			_ = json.Unmarshal(raw, &a)
+			c := a.Credentials[0]
+			c["generation"], c["sealed"] = 2, map[string]string{"v": "s3cret"}
+			return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}})
+		},
+		open: func(sealed string) ([]byte, error) {
+			var b struct{ V string }
+			err := json.Unmarshal([]byte(sealed), &b)
+			return []byte(b.V), err
+		}}
+	e := fillingEngine(opts, public, secrets, func(string) {})
 	uid := uint32(os.Getuid())
 	if err := e.Accepts(uid); err != nil {
 		t.Fatal(err)
@@ -102,4 +125,52 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
 	if a := <-done; a.Action != "updated" {
 		t.Fatalf("the second write: %+v", a)
 	}
+
+	// ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write
+	// from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it.
+	keyed := func(state string, ref bool) string {
+		p := provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "m", Node: "n1"},
+			Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()}
+		if state != provisions.Provided {
+			p.Why = "the provider has not answered"
+		}
+		if ref {
+			p.Reference = &provisions.Reference{ID: "provision/postgres/db", Generation: 2}
+		}
+		raw, err := provisions.Encode(p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return string(raw)
+	}
+	for _, tc := range []struct {
+		name, state string
+		ref         bool
+		content     string
+		want        string // the file after; "" keeps what was there
+		mode        os.FileMode
+	}{
+		{name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"},
+		{name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"},
+		{name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true,
+			content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600},
+		{name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644},
+	} {
+		before, _ := os.ReadFile(path)
+		if tc.state != "" {
+			provision["n1.m.db"] = keyed(tc.state, tc.ref)
+		}
+		a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)})
+		got, _ := os.ReadFile(path)
+		if tc.want == "" {
+			if a.Failed == "" && a.Refused == "" || string(got) != string(before) {
+				t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got)
+			}
+			continue
+		}
+		info, _ := os.Stat(path)
+		if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode {
+			t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode())
+		}
+	}
 }

cmd/mesh-host/main.go

@@ -1200,7 +1200,7 @@ func runLink(ctx context.Context, opts options) error {
 	go serveMeshCLI(aside, queue, say)
 	// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
 	// 0349 §4).
-	go serveFilling(aside, opts, mine.Membership.Signer, say)
+	go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say)
 
 	held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
 	// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back

cmd/mesh-host/provision_secrets.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/provisions"
+)
+
+// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write
+// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:<provision>}`, and the node-engine
+// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the
+// placeholder alone.
+//
+// For each placeholder it reads the provision's key `<node>.<consumer>.<provision>` of the seat mesh-provisioner's state
+// `provisions` (clause 1), which must be `provided` and carry the vault's reference; asks the vault for that pair
+// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client,
+// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the
+// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot
+// connect.
+type provisionSecrets struct {
+	node string
+	// read is the provision's key from mesh-provisioner's `provisions`: its value and whether it is there.
+	read func(ctx context.Context, key string) ([]byte, bool, error)
+	// own asks the vault's verb `own` in this engine's name.
+	own func(ctx context.Context, args map[string]any) (json.RawMessage, error)
+	// open opens a value sealed to this machine.
+	open func(sealed string) ([]byte, error)
+}
+
+// secretPlaceholder is `${secret:<provision>}`, or `${secret:<provision>.<local>}` where the consumer keeps several values
+// of one provision (ADR 0094).
+var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)(?:\.([A-Za-z0-9_-]+))?\}`)
+
+// fill is content with each provision placeholder of module's filled, and whether any was.
+func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) {
+	found := secretPlaceholder.FindAllSubmatch(content, -1)
+	if len(found) == 0 {
+		return content, false, nil
+	}
+	if module == "" {
+		return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found))
+	}
+	out := string(content)
+	done := map[string]bool{}
+	for _, f := range found {
+		whole, provision, local := string(f[0]), string(f[1]), string(f[2])
+		if done[whole] {
+			continue
+		}
+		done[whole] = true
+		value, err := p.value(ctx, module, provision, local)
+		if err != nil {
+			return nil, false, err
+		}
+		out = strings.ReplaceAll(out, whole, string(value))
+	}
+	return []byte(out), true, nil
+}
+
+// value is one provision's value for module on this node, opened.
+func (p provisionSecrets) value(ctx context.Context, module, provision, local string) ([]byte, error) {
+	key, err := provisions.Key(p.node, module, provision, local)
+	if err != nil {
+		return nil, err
+	}
+	if p.read == nil || p.own == nil || p.open == nil {
+		return nil, fmt.Errorf("the provision %s cannot be read on this machine: the node-engine is not linked to the mesh", key)
+	}
+	raw, ok, err := p.read(ctx, key)
+	switch {
+	case err != nil:
+		return nil, fmt.Errorf("the provision %s cannot be read: %w", key, err)
+	case !ok:
+		return nil, fmt.Errorf("the provision %s is not in %s: nothing asks for it", key, provisions.ProvisionsRead)
+	}
+	pr, err := provisions.Decode(raw)
+	if err != nil {
+		return nil, fmt.Errorf("the provision %s: %w", key, err)
+	}
+	if pr.State != provisions.Provided || pr.Reference == nil {
+		why := pr.State
+		if pr.Why != "" {
+			why += ": " + pr.Why
+		}
+		if pr.State == provisions.Provided {
+			why = "provided with no reference to the vault's value"
+		}
+		return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why)
+	}
+	type asked struct {
+		Role           string `json:"role"`
+		Provision      string `json:"provision"`
+		Consumer       string `json:"consumer"`
+		ConsumerModule string `json:"consumer_module"`
+		Provider       string `json:"provider"`
+		Local          string `json:"local,omitempty"`
+	}
+	ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local}
+	answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}})
+	if err != nil {
+		return nil, fmt.Errorf("the vault's value of %s: %w", key, err)
+	}
+	var a struct {
+		Node        string `json:"node"`
+		Credentials []struct {
+			asked
+			Generation int             `json:"generation"`
+			Sealed     json.RawMessage `json:"sealed"`
+		} `json:"credentials"`
+	}
+	if err := json.Unmarshal(answer, &a); err != nil {
+		return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err)
+	}
+	if a.Node != p.node {
+		return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node)
+	}
+	for _, c := range a.Credentials {
+		if c.asked != ask {
+			continue
+		}
+		if c.Generation < pr.Reference.Generation {
+			return nil, fmt.Errorf("the vault holds %s at generation %d and the provision names %d: not written until it has it",
+				key, c.Generation, pr.Reference.Generation)
+		}
+		value, err := p.open(string(c.Sealed))
+		if err != nil {
+			return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err)
+		}
+		return value, nil
+	}
+	return nil, fmt.Errorf("the vault answered no value of %s for this machine", key)
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 	golang.org/x/crypto v0.57.0

go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 h1:LaBxsyU3HeTIPKYmvpJuw+Wm2plpf6O7qHO91O+mAi0=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

internal/apply/filled.go

@@ -45,6 +45,30 @@ func WriteFilled(
 	log func(string),
 	keep Keep,
 	windows *Windows,
+) (report Report, _ store.State, _ error) {
+	return WriteFilledWith(ctx, sys, d, known, path, content, run, log, keep, windows, Filled{})
+}
+
+// Filled is what the node-engine did to a filled file's content before its write.
+type Filled struct {
+	// Secret says the node-engine filled a value into the content, a provision's `${secret:}` (novox/hq ADR 0570
+	// clause 6): the file is then written 0600 when the declaration gives no mode, as a file carrying a secret is.
+	Secret bool
+}
+
+// WriteFilledWith is WriteFilled for content the node-engine filled before the write (Filled).
+func WriteFilledWith(
+	ctx context.Context,
+	sys system.System,
+	d *declaration.Declaration,
+	known store.State,
+	path string,
+	content []byte,
+	run Runner,
+	log func(string),
+	keep Keep,
+	windows *Windows,
+	filled Filled,
 ) (report Report, _ store.State, _ error) {
 	if log == nil {
 		log = func(string) {}
@@ -65,7 +89,7 @@ func WriteFilled(
 		return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
 			"is written for exactly one, so nothing was written", len(declared), path)}
 	}
-	file := filledAs(declared[0], content)
+	file := filledAs(declared[0], content, filled.Secret)
 	if held, isHeld := known.HeldAt(file.ID); isHeld {
 		return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
 			"yet taken (%s), so nothing was written", path, held.Why)}
@@ -163,10 +187,10 @@ func WriteFilled(
 }
 
 // filledAs is the declared file with the filled content in place of the declaration's.
-func filledAs(declared *declaration.File, content []byte) *declaration.File {
+func filledAs(declared *declaration.File, content []byte, secret bool) *declaration.File {
 	f := *declared
 	f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
-	if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) {
+	if f.Mode == "" && (secret || declared.Secret() || len(declared.Secrets) > 0) {
 		// Written as the declaration would have written it: a file carrying a credential is not world-readable.
 		f.Mode = "0600"
 	}