Write a filled file on the provisioner client's request, and nothing else

The provisioner's client (ADR 0349 §4) fills a module's templates and needs the node-engine to write them: it must not hold the right to write the machine's files itself. A second local socket accepts a write only from the account the declaration names as the provisioner client's (SO_PEERCRED, same namespaces), only for a clean path the declaration names as filled, under the apply's own locks, and writes it through the apply as the declared file would be, firing restart-on, reload-on and container reads. Issue 668.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/mesh-host/filling.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"crypto/ed25519"
+	"errors"
+	"path/filepath"
+
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/filling"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
+// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
+func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
+	if err := filling.Serve(ctx, filling.Socket, fillingEngine(opts, signer, say), say); err != nil {
+		say("filled files are not written on request on this machine: " + err.Error())
+	}
+}
+
+// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
+// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
+// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
+func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
+	return &filling.Engine{
+		Declared: func() (*declaration.Declaration, error) {
+			raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
+			if err != nil {
+				return nil, err
+			}
+			return declaration.Parse(raw)
+		},
+		UIDOf: filling.UIDOf,
+		Lock: func() (func(), error) {
+			applying.Lock()
+			unlock, err := store.Lock(opts.state, nil)
+			if err != nil {
+				applying.Unlock()
+				return nil, err
+			}
+			return func() { unlock(); applying.Unlock() }, nil
+		},
+		Place: func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (string, string, error) {
+			built, err := system.For(builtFor)
+			if err != nil {
+				return "", "", err
+			}
+			known, err := store.Load(opts.state)
+			if err != nil {
+				return "", "", err
+			}
+			report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
+				announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
+			// Saved whichever way it went, as an apply's: what was written is on the machine either way.
+			if err := store.Save(opts.state, updated); err != nil {
+				return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
+					err.Error()))
+			}
+			action, digest := "", ""
+			for _, o := range report.Outcomes {
+				if o.Target == path && o.Type == string(declaration.TypeFile) {
+					action = o.Action
+				}
+			}
+			if rec, ok := updated.At(string(declaration.TypeFile), path); ok {
+				digest = rec.Wrote
+			}
+			return action, digest, writeErr
+		},
+	}
+}

cmd/mesh-host/filling_test.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/json"
+	"os"
+	"os/user"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/filling"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Defends novox/hq ADR 0349 §4 (issue 668): a write of a filled file and an apply are one at a time — the write
+// takes the lock the apply holds, and writes only once the apply let it go — and the write is recorded in the
+// node's state as an apply's outcome is.
+func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
+	was := builtFor
+	builtFor = "arch"
+	t.Cleanup(func() { builtFor = was })
+	me, err := user.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if me.Uid == "0" {
+		t.Skip("run as root, whose account is never the provisioner's")
+	}
+	dir := t.TempDir()
+	opts := options{state: filepath.Join(dir, "state.json")}
+	public, private, err := ed25519.GenerateKey(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(dir, "m.conf")
+	raw, err := json.Marshal(map[string]any{"declaration": 1, "provisioner_account": me.Username,
+		"filled_files": []string{path}, "resources": []any{map[string]any{"id": "m.conf", "type": "file",
+			"path": path, "content": "from the declaration\n", "owner-at-runtime": "owned", "owner": me.Username}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := store.SaveDeclared(store.DeclaredPath(opts.state),
+		store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
+		t.Fatal(err)
+	}
+	e := fillingEngine(opts, public, func(string) {})
+	uid := uint32(os.Getuid())
+	if err := e.Accepts(uid); err != nil {
+		t.Fatal(err)
+	}
+
+	// An apply holds the lock.
+	applying.Lock()
+	done := make(chan filling.Answer)
+	go func() {
+		done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("filled\n")})
+	}()
+	select {
+	case a := <-done:
+		applying.Unlock()
+		t.Fatalf("written while an apply held the lock: %+v", a)
+	case <-time.After(200 * time.Millisecond):
+	}
+	if _, err := os.Stat(path); err == nil {
+		applying.Unlock()
+		t.Fatal("the file was written while an apply held the lock")
+	}
+	applying.Unlock()
+	a := <-done
+	if a.Refused != "" || a.Failed != "" || a.Action != "created" {
+		t.Fatalf("the write after the apply: %+v", a)
+	}
+	got, err := os.ReadFile(path)
+	if err != nil || string(got) != "filled\n" {
+		t.Fatalf("the file holds %q (%v)", got, err)
+	}
+	known, err := store.Load(opts.state)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rec, ok := known.Find("m.conf"); !ok || rec.Wrote != a.Digest || rec.Target != path {
+		t.Errorf("the write is not recorded in the node's state: %+v, answered %q", rec, a.Digest)
+	}
+
+	// And the cross-process lock beside the state too: a write waits for a `reconcile` run by hand.
+	unlock, err := store.Lock(opts.state, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	go func() {
+		done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("again\n")})
+	}()
+	select {
+	case a := <-done:
+		unlock()
+		t.Fatalf("written while another process held the apply's lock: %+v", a)
+	case <-time.After(200 * time.Millisecond):
+	}
+	unlock()
+	if a := <-done; a.Action != "updated" {
+		t.Fatalf("the second write: %+v", a)
+	}
+}

internal/declaration/provisioner_test.go

@@ -0,0 +1,37 @@
+package declaration
+
+import (
+	"strings"
+	"testing"
+)
+
+// Defends novox/hq ADR 0349 §4 (issue 668): the declaration names the account the provisioner's client runs as,
+// the only one whose write of a filled file the node-engine accepts. Read here before any controller sends it.
+
+func TestTheDeclarationNamesTheProvisionersAccount(t *testing.T) {
+	d, err := Parse([]byte(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/a"],
+	  "resources":[{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d.Provisioner != "mesh-node-provisioner" {
+		t.Errorf("the provisioner's account read as %q", d.Provisioner)
+	}
+	d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d.Provisioner != "" {
+		t.Errorf("a declaration naming no account names %q", d.Provisioner)
+	}
+}
+
+func TestAProvisionersAccountThatIsNoAccountIsRefused(t *testing.T) {
+	for name, account := range map[string]string{"root": "root", "not a name": "a b", "a path": "../x"} {
+		_, err := Parse([]byte(`{"declaration":1,"provisioner_account":"` + account + `",
+		  "resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`))
+		if err == nil || !strings.Contains(err.Error(), "provisioner_account") {
+			t.Errorf("%s: want a refusal naming provisioner_account, got %v", name, err)
+		}
+	}
+}

internal/meshcli/peer.go

@@ -0,0 +1,19 @@
+package meshcli
+
+import "net"
+
+// The kernel's word on who is at the other end of a socket, shared with the node-engine's other local socket, the
+// write of a filled file (internal/filling, novox/hq ADR 0349 §4): one reading of it, so the two never judge a peer
+// differently.
+
+// PeerCredentials is the user id and process id of the process at the other end, as the kernel recorded them when
+// it connected (SO_PEERCRED). Nothing the process says is read.
+func PeerCredentials(conn *net.UnixConn) (uint32, int32, error) { return peerCredentials(conn) }
+
+// SameNamespaces refuses a process whose mount or user namespace is not the engine's: a user id is this machine's
+// account only in this machine's namespaces.
+func SameNamespaces(pid int32) error { return namespacesOf(pid) }
+
+// NotOwnDirectory says what is wrong with a socket's directory, or nothing: it must be a directory, not a link,
+// owned by the account the engine runs as.
+func NotOwnDirectory(dir string) string { return notOwnDirectory(dir) }