Merge main (the container-groups change) into the bucket-per-node branch, keeping the mesh-sdk pin of #64

mesh-sdk is pinned to fix/1694-bucket-per-node 36d9eba51db9 until it merges.

README.md

@@ -102,6 +102,19 @@ what it did not understand would apply most of a declaration and report success.
 ]}
 ```
 
+**A directory or archive says how it treats files it did not write** (issue 1024,
+[`novox/hq` ADR 0422](https://git.novox.be/novox/hq)). `unknown` is `move-out`, `module` or `program`;
+absent, nothing about unknown paths is done. `knows` lists paths inside it, relative to its `path`, that
+are not declared but belong there; a trailing `/` names a folder and everything in it. `knows` without
+`unknown` is refused. For `move-out` the node-engine, after every apply and so on every reconcile, moves
+each path inside the directory that nothing declared, no archive unpacked and `knows` does not name,
+whole, into `<home>/.local/state/mesh-moved/<module>/<UTC time>/` beside a `<UTC time>.json` list. The
+list names each move before it is made. Nothing is deleted. A path younger than a minute waits for the
+next look; a socket, pipe or device is left and said. It is only for a directory under an account's
+home on one file system with the kept folder, and otherwise the resource fails and nothing moves.
+`module` and `program` are read by the module and the program that owns the path, and the node-engine
+does nothing for them.
+
 **It converges rather than executes.** Applying twice changes nothing the second time; applying
 to a drifted machine returns it. A mode is *maintained*, not merely set — a permission applied
 at creation is not a permission held.
@@ -244,7 +257,7 @@ without its neighbour checked out is a repository nobody can build.
 
 ```
 mesh-controller:  ./build/mesh-controller plan <node> --json > /tmp/d.json
-mesh-host:     MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v
+mesh-sdk:      MESH_EMITTED=/tmp/d.json go test ./declaration/ -v   (run in mesh-sdk's go/ directory)
 ```
 
 **What this node says when it joins, read by the mesh:**

RELEASE-57.md

@@ -0,0 +1,6 @@
+# Release 57: config and provisioning
+
+This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
+merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
+delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
+and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.

cmd/mesh-bootstrap/main.go

@@ -116,6 +116,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
 
   --tools-source     the repository the shared base is built from
   --tools-ref        what of it to build (default main)
+  --tools-go-source  the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)
+  --tools-go-ref     what of it to build (default main)
   --catalog-source   the catalogue REPOSITORY, for building its modules;
                      --catalog is the checkout that says what they are
   --catalog-ref      what of it to build (default main)
@@ -130,7 +132,7 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
 
   The foundation's ports are this machine's, each checked free before anything is
   raised and kept as the node's setting for the module that binds it:
-  --store-port 5432  --bus-port 5671  --amqp-port 5672  --management-port 15672
+  --store-port 5432  --bus-port 5671
   --registry-port 5000 (follows --registry, and must agree with it)
   --packages-port 3000  --hub-port 51820/udp
   --overlay-range    the private network's range (default 10.42.0.0/16); refused
@@ -308,6 +310,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
 		"the repository the shared base is built from")
 	set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
 		"what of it to build (default main)")
+	set.StringVar(&opts.ToolsGoSource.Repository, "tools-go-source", opts.ToolsGoSource.Repository,
+		"the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)")
+	set.StringVar(&opts.ToolsGoSource.Ref, "tools-go-ref", opts.ToolsGoSource.Ref,
+		"what of it to build (default main)")
 	set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
 		"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
 	set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
@@ -325,9 +331,7 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
 		into       *int
 	}{
 		{"store-port", "the store", &opts.Ports.Store},
-		{"bus-port", "the bus (amqps)", &opts.Ports.Bus},
-		{"amqp-port", "the broker's AMQP", &opts.Ports.AMQP},
-		{"management-port", "the broker's management, on loopback", &opts.Ports.Management},
+		{"bus-port", "the bus", &opts.Ports.Bus},
 		{"registry-port", "the registry", &opts.Ports.Registry},
 		{"packages-port", "the package registry", &opts.Ports.Packages},
 		{"hub-port", "the private network's hub (udp)", &opts.Ports.Hub},

cmd/mesh-host/busy_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// A container waiting because it said busy is news when it starts waiting and when it is replaced, never on
+// its output alone (novox/hq ADR 0335): the walk waiting on this machine starts its gate at the report that
+// says it replaced, and a reconcile is otherwise silent.
+func TestABusyWaitIsSaidWhenItStartsAndWhenItEnds(t *testing.T) {
+	quiet := link.Report{Outward: []string{"eth0"}}
+	waiting := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
+		Container: "mesh-build-agent", State: "waiting", Said: "building a"}}}
+	still := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
+		Container: "mesh-build-agent", State: "waiting", Said: "building b, a minute on"}}}
+	replaced := link.Report{Outward: []string{"eth0"}, Busy: []link.Busy{{ID: "build-agent.server",
+		Container: "mesh-build-agent", State: "replaced-when-idle"}}}
+	if adoptionFingerprint(waiting) == adoptionFingerprint(quiet) {
+		t.Error("a container starting to wait is not news")
+	}
+	if adoptionFingerprint(still) != adoptionFingerprint(waiting) {
+		t.Error("a wait whose output changed reads as news on every ask")
+	}
+	if adoptionFingerprint(replaced) == adoptionFingerprint(waiting) {
+		t.Error("a container replaced after it waited is not news")
+	}
+	if !worthSaying(link.Report{Busy: replaced.Busy}) {
+		t.Error("a replacement after a wait is not worth saying")
+	}
+}
+
+// While a container waits, a pass is asked for every so often, not at the reconcile's five minutes; with
+// nothing waiting, none (novox/hq ADR 0335 §2).
+func TestAWaitIsAskedAgainSoonerThanTheReconcile(t *testing.T) {
+	if BusyAskEvery >= ReconcileEvery || BusyAskEvery > time.Minute {
+		t.Fatalf("a waiting container is asked every %s", BusyAskEvery)
+	}
+	var reconciles atomic.Int32
+	queue := &link.Queue{Reconcile: func(context.Context) (link.Report, bool) {
+		reconciles.Add(1)
+		return link.Report{}, false
+	}}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go queue.Run(ctx)
+	busyWaiting.Store(false)
+	t.Cleanup(func() { busyWaiting.Store(false) })
+	go busyPasses(ctx, queue, 10*time.Millisecond)
+	time.Sleep(100 * time.Millisecond)
+	if n := reconciles.Load(); n != 0 {
+		t.Fatalf("%d passes were asked with nothing waiting", n)
+	}
+	busyWaiting.Store(true)
+	deadline := time.Now().Add(2 * time.Second)
+	for reconciles.Load() == 0 && time.Now().Before(deadline) {
+		time.Sleep(10 * time.Millisecond)
+	}
+	if reconciles.Load() == 0 {
+		t.Fatal("no pass was asked while a container waits")
+	}
+}

cmd/mesh-host/file_references_test.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// aFilesLink is a link that serves objects by name.
+type aFilesLink struct{ objects map[string][]byte }
+
+func (a aFilesLink) FileObject(_ context.Context, name string, _ int64) ([]byte, error) {
+	if b, ok := a.objects[name]; ok {
+		return b, nil
+	}
+	return nil, os.ErrNotExist
+}
+
+func referenceDeclaration(path string, body []byte) []byte {
+	return []byte(`{"declaration":1,"resources":[{"id":"m.big","type":"file","path":"` + path + `","digest":"` +
+		declaration.DigestOf(body) + `","size":` + strconv.Itoa(len(body)) + `}]}`)
+}
+
+// 1695/AC1 and AC2, through the whole apply: a reference is fetched, verified and written; a differing object
+// refuses the declaration whole and writes nothing.
+func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
+	unansweringPacman(t)
+	body := []byte(strings.Repeat("a line of a big file\n", 4000))
+	name, _ := declaration.ObjectName(declaration.DigestOf(body))
+	dir := t.TempDir()
+	target := filepath.Join(dir, "big")
+	opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
+	raw := referenceDeclaration(target, body)
+
+	bad := append([]byte(nil), body...)
+	bad[10] ^= 1
+	ctx := link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: bad}})
+	report := applyAndKeep(ctx, opts, raw, nil, nil, nil)
+	if report.Refused == "" || !strings.Contains(report.Refused, "m.big") {
+		t.Errorf("1695/AC2: a differing object did not refuse the declaration: %+v", report)
+	}
+	if _, err := os.Stat(target); err == nil {
+		t.Error("1695/AC2: a file was written from an object that differs")
+	}
+
+	ctx = link.WithFiles(context.Background(), aFilesLink{map[string][]byte{name: body}})
+	report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
+	if report.Refused != "" {
+		t.Fatalf("1695/AC1: refused: %s", report.Refused)
+	}
+	got, err := os.ReadFile(target)
+	if err != nil || string(got) != string(body) {
+		t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
+	}
+}
+
+// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
+// as it is, any other refuses and writes nothing — never an empty file.
+func TestTheHandApplyNeverWritesAReferenceEmpty(t *testing.T) {
+	unansweringPacman(t)
+	body := []byte(strings.Repeat("a line of a big file\n", 4000))
+	dir := t.TempDir()
+	target := filepath.Join(dir, "big")
+	opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
+	raw := referenceDeclaration(target, body)
+	d, err := declaration.ParseFileTrusted(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := runApply(context.Background(), opts, d, raw, fromFile); err == nil || !strings.Contains(err.Error(), "digest") {
+		t.Errorf("1695/AC8: a reference with no file and no bus was not refused: %v", err)
+	}
+	if _, err := os.Stat(target); err == nil {
+		t.Error("1695/AC8: a file was written for a reference that could not be had")
+	}
+	if err := os.WriteFile(target, body, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	d, _ = declaration.ParseFileTrusted(raw)
+	if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil && !strings.Contains(err.Error(), "applied only in part") {
+		t.Fatalf("1695/AC8: %v", err)
+	}
+	if got, _ := os.ReadFile(target); string(got) != string(body) {
+		t.Errorf("1695/AC8: the file holds %d bytes, want %d", len(got), len(body))
+	}
+}

cmd/mesh-host/filling.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"crypto/ed25519"
+	"errors"
+	"path/filepath"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/filling"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
+// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
+func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
+	if err := filling.Serve(ctx, filling.Socket, fillingEngine(opts, signer, say), say); err != nil {
+		say("filled files are not written on request on this machine: " + err.Error())
+	}
+}
+
+// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
+// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
+// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
+func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
+	return &filling.Engine{
+		Declared: func() (*declaration.Declaration, error) {
+			raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
+			if err != nil {
+				return nil, err
+			}
+			return declaration.Parse(raw)
+		},
+		UIDOf: filling.UIDOf,
+		Lock: func() (func(), error) {
+			applying.Lock()
+			unlock, err := store.Lock(opts.state, nil)
+			if err != nil {
+				applying.Unlock()
+				return nil, err
+			}
+			return func() { unlock(); applying.Unlock() }, nil
+		},
+		Place: func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (string, string, error) {
+			built, err := system.For(builtFor)
+			if err != nil {
+				return "", "", err
+			}
+			known, err := store.Load(opts.state)
+			if err != nil {
+				return "", "", err
+			}
+			report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
+				announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
+			// Saved whichever way it went, as an apply's: what was written is on the machine either way.
+			if err := store.Save(opts.state, updated); err != nil {
+				return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
+					err.Error()))
+			}
+			action, digest := "", ""
+			for _, o := range report.Outcomes {
+				if o.Target == path && o.Type == string(declaration.TypeFile) {
+					action = o.Action
+				}
+			}
+			if rec, ok := updated.At(string(declaration.TypeFile), path); ok {
+				digest = rec.Wrote
+			}
+			return action, digest, writeErr
+		},
+	}
+}

cmd/mesh-host/filling_test.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/json"
+	"os"
+	"os/user"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/filling"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Defends novox/hq ADR 0349 §4 (issue 668): a write of a filled file and an apply are one at a time — the write
+// takes the lock the apply holds, and writes only once the apply let it go — and the write is recorded in the
+// node's state as an apply's outcome is.
+func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
+	was := builtFor
+	builtFor = "arch"
+	t.Cleanup(func() { builtFor = was })
+	me, err := user.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if me.Uid == "0" {
+		t.Skip("run as root, whose account is never the provisioner's")
+	}
+	dir := t.TempDir()
+	opts := options{state: filepath.Join(dir, "state.json")}
+	public, private, err := ed25519.GenerateKey(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(dir, "m.conf")
+	raw, err := json.Marshal(map[string]any{"declaration": 1, "provisioner_account": me.Username,
+		"filled_files": []string{path}, "resources": []any{map[string]any{"id": "m.conf", "type": "file",
+			"path": path, "content": "from the declaration\n", "owner-at-runtime": "owned", "owner": me.Username}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := store.SaveDeclared(store.DeclaredPath(opts.state),
+		store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
+		t.Fatal(err)
+	}
+	e := fillingEngine(opts, public, func(string) {})
+	uid := uint32(os.Getuid())
+	if err := e.Accepts(uid); err != nil {
+		t.Fatal(err)
+	}
+
+	// An apply holds the lock.
+	applying.Lock()
+	done := make(chan filling.Answer)
+	go func() {
+		done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("filled\n")})
+	}()
+	select {
+	case a := <-done:
+		applying.Unlock()
+		t.Fatalf("written while an apply held the lock: %+v", a)
+	case <-time.After(200 * time.Millisecond):
+	}
+	if _, err := os.Stat(path); err == nil {
+		applying.Unlock()
+		t.Fatal("the file was written while an apply held the lock")
+	}
+	applying.Unlock()
+	a := <-done
+	if a.Refused != "" || a.Failed != "" || a.Action != "created" {
+		t.Fatalf("the write after the apply: %+v", a)
+	}
+	got, err := os.ReadFile(path)
+	if err != nil || string(got) != "filled\n" {
+		t.Fatalf("the file holds %q (%v)", got, err)
+	}
+	known, err := store.Load(opts.state)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rec, ok := known.Find("m.conf"); !ok || rec.Wrote != a.Digest || rec.Target != path {
+		t.Errorf("the write is not recorded in the node's state: %+v, answered %q", rec, a.Digest)
+	}
+
+	// And the cross-process lock beside the state too: a write waits for a `reconcile` run by hand.
+	unlock, err := store.Lock(opts.state, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	go func() {
+		done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("again\n")})
+	}()
+	select {
+	case a := <-done:
+		unlock()
+		t.Fatalf("written while another process held the apply's lock: %+v", a)
+	case <-time.After(200 * time.Millisecond):
+	}
+	unlock()
+	if a := <-done; a.Action != "updated" {
+		t.Fatalf("the second write: %+v", a)
+	}
+}

cmd/mesh-host/grants.go

@@ -1,181 +0,0 @@
-package main
-
-import (
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/novox/mesh-host/internal/apply"
-	"github.com/novox/mesh-host/internal/declaration"
-	"github.com/novox/mesh-host/internal/link"
-)
-
-// A read-write access whose group cannot write is never quiet (novox/hq ADR 0312 §2).
-//
-// The engine grants a read-write access's path group to the identities the module's code runs as, and changes
-// nothing on the path: whether its group may write is the operator's, as the data is. So an access whose path's
-// group has no write permission is stated, on every look, as its module's resource of kind `access`, unhealthy,
-// saying so and that the operator decides — and one owned by a group that is never granted (§4) is stated
-// unhealthy saying that. The controller raises it as that module's condition on two statements in a row and
-// clears it on the first healthy one, as it does every resource's health (ADR 0240). The path is the target,
-// never in the reason: the controller says a reason on the operator's channel, which withholds a path (ADR 0234
-// §6).
-
-// grantedAccess is one read-write access that grants, as the judge reads it.
-type grantedAccess struct {
-	module, id, path string
-}
-
-// accessJudge reads, on every look, whether each read-write access that grants may write its path through its
-// group. Safe for the apply and the looking loop at once.
-type accessJudge struct {
-	mu       sync.Mutex
-	accesses []grantedAccess
-	// was is each access's last verdict, with since when and how many looks in a row found it unhealthy.
-	was map[string]link.ResourceHealth
-
-	// reading guards inFlight: every path whose read has not returned yet.
-	reading  sync.Mutex
-	inFlight map[string]bool
-}
-
-// accesses is the one access judge on this machine.
-var accesses = newAccessJudge()
-
-func newAccessJudge() *accessJudge {
-	return &accessJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
-}
-
-// set is what the declaration just applied asks: every read-write access of a module that names a grant, not
-// held as found. An access no longer declared is forgotten.
-func (j *accessJudge) set(d *declaration.Declaration, held map[string]bool) {
-	var as []grantedAccess
-	if d != nil {
-		for _, r := range d.Resources {
-			a, ok := r.(*declaration.Access)
-			if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite || held[a.ID] {
-				continue
-			}
-			at := strings.LastIndex(a.ID, ".")
-			if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) {
-				continue
-			}
-			as = append(as, grantedAccess{module: a.ID[:at], id: a.ID, path: a.Path})
-		}
-	}
-	sort.Slice(as, func(a, b int) bool { return as[a].id < as[b].id })
-	j.mu.Lock()
-	defer j.mu.Unlock()
-	j.accesses = as
-	declared := map[string]bool{}
-	for _, a := range as {
-		declared[a.id] = true
-	}
-	for id := range j.was {
-		if !declared[id] {
-			delete(j.was, id)
-		}
-	}
-}
-
-// lookBound is how long a look waits for one accessed path to answer. A hard network mount whose server is down
-// blocks a stat for ever, and one access must never stop every health statement this machine makes.
-var lookBound = 5 * time.Second
-
-// pathGroupBounded reads a path's group within lookBound, outside any lock. A read that has not answered is said
-// so; while it still hangs, later looks do not start another on the same path, so a dead mount costs one blocked
-// read, not one per look.
-func (j *accessJudge) pathGroupBounded(path string) (apply.PathGroup, bool, error) {
-	j.reading.Lock()
-	if j.inFlight[path] {
-		j.reading.Unlock()
-		return apply.PathGroup{}, false, nil
-	}
-	j.inFlight[path] = true
-	j.reading.Unlock()
-	type read struct {
-		g   apply.PathGroup
-		err error
-	}
-	done := make(chan read, 1)
-	go func() {
-		g, err := apply.GroupOfPath(path)
-		j.reading.Lock()
-		delete(j.inFlight, path)
-		j.reading.Unlock()
-		done <- read{g, err}
-	}()
-	select {
-	case r := <-done:
-		return r.g, true, r.err
-	case <-time.After(lookBound):
-		return apply.PathGroup{}, false, nil
-	}
-}
-
-// look reads every access now: healthy when its group may write and may be granted, unhealthy saying why
-// otherwise. Whether any verdict changed since the last look, for the loop to say it. The paths are read with no
-// lock held, each within lookBound.
-func (j *accessJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
-	j.mu.Lock()
-	as := append([]grantedAccess(nil), j.accesses...)
-	j.mu.Unlock()
-
-	type verdict struct{ state, reason string }
-	read := make([]verdict, len(as))
-	for i, a := range as {
-		state, reason := link.StateHealthy, ""
-		g, answered, err := j.pathGroupBounded(a.path)
-		switch {
-		case !answered:
-			state, reason = link.StateUnhealthy, "its path does not answer: reading which group owns it did not "+
-				"return within "+lookBound.String()
-		case err != nil:
-			state, reason = link.StateUnhealthy, "its group cannot be read: "+strings.ReplaceAll(err.Error(), a.path, "the path")
-		case apply.GrantRefusal(g) != nil:
-			state, reason = link.StateUnhealthy, "its group is never granted: "+apply.GrantRefusal(g).Error()
-		case !g.Writable:
-			state, reason = link.StateUnhealthy, apply.CannotWriteWords(g)
-		}
-		read[i] = verdict{state, reason}
-	}
-
-	j.mu.Lock()
-	defer j.mu.Unlock()
-	changed := false
-	out := make([]link.ResourceHealth, 0, len(as))
-	for i, a := range as {
-		state, reason := read[i].state, read[i].reason
-		v := link.ResourceHealth{Module: a.module, Resource: a.id, Kind: link.KindAccess, Target: a.path,
-			State: state, Reason: reason, Since: now.UTC()}
-		if was, had := j.was[a.id]; had && was.State == state && was.Reason == reason {
-			v.Since = was.Since
-			v.Streak = was.Streak
-		} else {
-			changed = true
-		}
-		if state != link.StateHealthy {
-			v.Streak++
-		}
-		j.was[a.id] = v
-		out = append(out, v)
-	}
-	return out, changed
-}
-
-// withAccesses adds to a statement every read-write access that grants, as its module's resource of kind access.
-func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
-	h.Resources = append(h.Resources, vs...)
-	return h
-}
-
-// accessesHealthy says no access in vs is anything but healthy.
-func accessesHealthy(vs []link.ResourceHealth) bool {
-	for _, v := range vs {
-		if v.State != link.StateHealthy {
-			return false
-		}
-	}
-	return true
-}

cmd/mesh-host/grants_test.go

@@ -1,138 +0,0 @@
-package main
-
-import (
-	"strings"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/novox/mesh-host/internal/apply"
-	"github.com/novox/mesh-host/internal/declaration"
-	"github.com/novox/mesh-host/internal/link"
-)
-
-// A read-write access whose path's group cannot write is stated unhealthy on every look, saying so and that the
-// operator decides, and one whose group is never granted says that (novox/hq ADR 0312 §2, §4). The path is the
-// target and never in the reason (ADR 0234 §6); a read access, granting nothing, is not stated at all.
-func TestAnAccessWhoseGroupCannotWriteIsStatedUnhealthyWithTheOperatorsRemedy(t *testing.T) {
-	spool, library, shared, read := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir()
-	groups := map[string]apply.PathGroup{
-		spool:   {GID: 1002, Name: "downloads", Writable: false},
-		library: {GID: 1001, Name: "media", Writable: true},
-		shared:  {GID: 0, Name: "root", Writable: true},
-	}
-	was := apply.GroupOfPath
-	apply.GroupOfPath = func(path string) (apply.PathGroup, error) { return groups[path], nil }
-	t.Cleanup(func() { apply.GroupOfPath = was })
-	wasRoot := apply.RootGroupIDs
-	apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} }
-	t.Cleanup(func() { apply.RootGroupIDs = wasRoot })
-
-	d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
-	  {"id":"nzbget.downloads","type":"access","path":"` + spool + `","mode":"read-write","grant":{"account":"operator"}},
-	  {"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{"account":"operator"}},
-	  {"id":"importer.shared","type":"access","path":"` + shared + `","mode":"read-write","grant":{"account":"operator"}},
-	  {"id":"player.music","type":"access","path":"` + read + `","mode":"read"}]}`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	j := newAccessJudge()
-	j.set(d, nil)
-	at := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
-	vs, changed := j.look(at)
-	if !changed || len(vs) != 3 {
-		t.Fatalf("stated %d accesses (changed %v), want the three read-write ones: %+v", len(vs), changed, vs)
-	}
-	got := map[string]link.ResourceHealth{}
-	for _, v := range vs {
-		got[v.Resource] = v
-		if v.Kind != link.KindAccess || strings.Contains(v.Reason, v.Target) {
-			t.Errorf("%s is stated as %q with its path in the reason %q", v.Resource, v.Kind, v.Reason)
-		}
-	}
-	if v := got["nzbget.downloads"]; v.State != link.StateUnhealthy || v.Module != "nzbget" || v.Target != spool ||
-		!strings.Contains(v.Reason, "cannot write") || !strings.Contains(v.Reason, "operator's to decide") || v.Streak != 1 {
-		t.Errorf("an access whose group cannot write is stated %+v", v)
-	}
-	if v := got["importer.shared"]; v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "never granted") {
-		t.Errorf("an access of the root group is stated %+v", v)
-	}
-	if v := got["importer.music"]; v.State != link.StateHealthy {
-		t.Errorf("an access whose group may write is stated %+v", v)
-	}
-	if h := withAccesses(&link.Health{}, vs); accessesHealthy(vs) || len(h.Resources) != 3 {
-		t.Errorf("the statement does not carry the accesses as unhealthy: %+v", h.Resources)
-	}
-
-	// Looked at again: the same verdict, its since kept and its streak counted.
-	vs, changed = j.look(at.Add(time.Minute))
-	for _, v := range vs {
-		if v.Resource == "nzbget.downloads" && (changed || v.Streak != 2 || !v.Since.Equal(at)) {
-			t.Errorf("the second look is %+v (changed %v)", v, changed)
-		}
-	}
-	// The operator gives the group write: healthy at the next look, and nothing the engine did made it so.
-	groups[spool] = apply.PathGroup{GID: 1002, Name: "downloads", Writable: true}
-	vs, _ = j.look(at.Add(2 * time.Minute))
-	for _, v := range vs {
-		if v.Resource == "nzbget.downloads" && v.State != link.StateHealthy {
-			t.Errorf("an access whose group was given write is stated %+v", v)
-		}
-	}
-}
-
-// A path that does not answer — a hard network mount whose server is down — is stated unhealthy within the bound,
-// and stops neither this look nor the next: one blocked read per path, never one per look.
-func TestAnAccessWhosePathDoesNotAnswerIsStatedWithinTheBound(t *testing.T) {
-	hung, library := t.TempDir(), t.TempDir()
-	release, finished := make(chan struct{}), make(chan struct{})
-	var started atomic.Int32
-	was, wasBound := apply.GroupOfPath, lookBound
-	apply.GroupOfPath = func(path string) (apply.PathGroup, error) {
-		if path == hung {
-			started.Add(1)
-			<-release
-			defer close(finished)
-		}
-		return apply.PathGroup{GID: 1001, Name: "media", Writable: true}, nil
-	}
-	lookBound = 50 * time.Millisecond
-	wasRoot := apply.RootGroupIDs
-	apply.RootGroupIDs = func() map[int]string { return map[int]string{0: "root"} }
-	t.Cleanup(func() { apply.GroupOfPath, lookBound, apply.RootGroupIDs = was, wasBound, wasRoot })
-	// Run first: the read that hangs is let go and has returned before anything it read is put back.
-	t.Cleanup(func() { close(release); <-finished })
-
-	d, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
-	  {"id":"nzbget.downloads","type":"access","path":"` + hung + `","mode":"read-write","grant":{}},
-	  {"id":"importer.music","type":"access","path":"` + library + `","mode":"read-write","grant":{}}]}`))
-	if err != nil {
-		t.Fatal(err)
-	}
-	j := newAccessJudge()
-	j.set(d, nil)
-	for look := 0; look < 3; look++ {
-		done := make(chan []link.ResourceHealth, 1)
-		go func() { vs, _ := j.look(time.Now()); done <- vs }()
-		select {
-		case vs := <-done:
-			for _, v := range vs {
-				switch v.Resource {
-				case "nzbget.downloads":
-					if v.State != link.StateUnhealthy || !strings.Contains(v.Reason, "does not answer") {
-						t.Errorf("look %d: a path that does not answer is stated %+v", look, v)
-					}
-				case "importer.music":
-					if v.State != link.StateHealthy {
-						t.Errorf("look %d: the path that answers is stated %+v", look, v)
-					}
-				}
-			}
-		case <-time.After(5 * time.Second):
-			t.Fatalf("look %d did not return while one path hangs", look)
-		}
-	}
-	if n := started.Load(); n != 1 {
-		t.Errorf("%d reads were started on the path that hangs; want one", n)
-	}
-}

cmd/mesh-host/judge_directories.go

@@ -4,8 +4,8 @@ import (
 	"errors"
 	"fmt"
 
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/apply"
-	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/link"
 	"github.com/novox/mesh-host/internal/store"
 )

cmd/mesh-host/libraries.go

@@ -0,0 +1,289 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A library is judged on every look (novox/hq ADR 0328 §8, §9).
+//
+// Each library this machine is reached by is said as a resource `library:<name>` of kind library, with no module:
+// healthy while it is kept (or, off its home, while its mount is there), waiting with an access wait of reason keep
+// or disk, unknown while its filesystem could not be named, unhealthy otherwise. The root is the target, never in a
+// reason nor in a wait's words: the controller says them on the operator's channel, which withholds a path (ADR
+// 0234 §6). An access is judged by nothing, as before ADR 0325.
+
+// libraryJudge reads every library on every look. Safe for the apply and the looking loop at once.
+type libraryJudge struct {
+	mu        sync.Mutex
+	libraries []declaration.Library
+	declared  *declaration.Declaration
+	// was is each library's last verdict, with since when and how many looks in a row found it unhealthy.
+	was map[string]link.ResourceHealth
+
+	// reading guards inFlight: every library whose look has not returned yet.
+	reading  sync.Mutex
+	inFlight map[string]bool
+}
+
+// accesses is the one library judge on this machine (its name from when it judged accesses).
+var accesses = newLibraryJudge()
+
+func newLibraryJudge() *libraryJudge {
+	return &libraryJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
+}
+
+// set is what the declaration just applied declares: every library. One no longer declared is forgotten.
+func (j *libraryJudge) set(d *declaration.Declaration, _ map[string]bool) {
+	var ls []declaration.Library
+	if d != nil {
+		ls = append(ls, d.Libraries...)
+	}
+	sort.Slice(ls, func(a, b int) bool { return ls[a].Library < ls[b].Library })
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	j.libraries, j.declared = ls, d
+	declared := map[string]bool{}
+	for _, l := range ls {
+		declared[apply.LibraryAccess(l.Library)] = true
+	}
+	for id := range j.was {
+		if !declared[id] {
+			delete(j.was, id)
+		}
+	}
+}
+
+// declaration is the declaration last set.
+func (j *libraryJudge) declaration() *declaration.Declaration {
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	return j.declared
+}
+
+// lookBound is how long a look waits for one library to answer. A hard network mount whose server is down blocks
+// a stat for ever, and one library must never stop every health statement this machine makes.
+var lookBound = 5 * time.Second
+
+// verdict is one library's judgement in a look.
+type verdict struct {
+	state, reason string
+	waits         []link.AccessWait
+	act           *link.AccessActRecord
+}
+
+// judgeBounded judges one library within lookBound, outside any lock. A look that has not answered is said so;
+// while it still hangs, later looks start no other on the same library.
+func (j *libraryJudge) judgeBounded(l declaration.Library) (verdict, bool) {
+	j.reading.Lock()
+	if j.inFlight[l.Library] {
+		j.reading.Unlock()
+		return verdict{}, false
+	}
+	j.inFlight[l.Library] = true
+	j.reading.Unlock()
+	done := make(chan verdict, 1)
+	go func() {
+		v := judgeLibrary(l)
+		j.reading.Lock()
+		delete(j.inFlight, l.Library)
+		j.reading.Unlock()
+		done <- v
+	}()
+	select {
+	case v := <-done:
+		return v, true
+	case <-time.After(lookBound):
+		return verdict{}, false
+	}
+}
+
+// judgeLibrary is one library's judgement now (ADR 0328 §7, §8, §9).
+func judgeLibrary(l declaration.Library) verdict {
+	v := apply.JudgeLibrary(context.Background(), l)
+	access := apply.LibraryAccess(l.Library)
+	var out verdict
+	if a := v.Act; a != nil {
+		out.act = &link.AccessActRecord{Act: a.Act, Access: access, Warrant: a.Warrant, By: a.By, State: a.State,
+			Started: a.Started, Ended: a.Ended, Folders: a.Folders, Files: a.Files, Kept: a.Kept, Links: a.Links,
+			Other: a.Other, Mounts: a.Mounts, Shared: a.Shared, Linked: a.Linked, Stripped: a.Stripped, Why: a.Why}
+	}
+	switch v.Kind {
+	case apply.VerdictWaitKeep:
+		out.state, out.reason = link.StateWaiting, "its root holds data the mesh has not taken on the filesystem "+
+			"it is on now, and waits for the operator's word (novox/hq ADR 0328 §7, §8)"
+		out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
+			What: apply.KeepWords(l.Library), Placed: v.Placed, Reason: link.WaitKeep}}
+	case apply.VerdictWaitDisk:
+		out.state, out.reason = link.StateWaiting, v.Why+" (novox/hq ADR 0328 §9)"
+		out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
+			What: apply.DiskWords(l.Library), Reason: link.WaitDisk}}
+	case apply.VerdictUnknown:
+		// Not a fault and not the operator's: said as not known, and judged again on the next look.
+		out.state, out.reason = link.StateUnknown, v.Why+"; judged again on the next look"
+	case apply.VerdictBroken:
+		out.state, out.reason = link.StateUnhealthy, v.Why
+	default:
+		out.state = link.StateHealthy
+		switch {
+		case v.Act != nil && v.Act.State == link.ActRunning:
+			out.state, out.reason = link.StateStarting, "the take act is setting its group under its root"
+		case v.Act != nil && (v.Act.State == link.ActStopped || v.Act.State == link.ActFailed):
+			out.state, out.reason = link.StateStarting, "the take act goes on from where it stopped: "+v.Act.Why
+		}
+	}
+	return out
+}
+
+// look reads every library now. Whether any verdict changed since the last look, for the loop to say it.
+func (j *libraryJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
+	j.mu.Lock()
+	ls := append([]declaration.Library(nil), j.libraries...)
+	j.mu.Unlock()
+
+	read := make([]verdict, len(ls))
+	for i, l := range ls {
+		v, answered := j.judgeBounded(l)
+		if !answered {
+			v = verdict{state: link.StateUnknown, reason: "its root does not answer: judging it did not return " +
+				"within " + lookBound.String()}
+		}
+		read[i] = v
+	}
+
+	// And every tool runner's account the last apply could not put in a group, as a resource of its own (ADR 0334
+	// §2): unhealthy, never failing the apply.
+	warnings := apply.RunnerWarnings()
+
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	changed := false
+	out := make([]link.ResourceHealth, 0, len(ls)+len(warnings))
+	for _, w := range warnings {
+		v := link.ResourceHealth{Module: w.Module, Resource: w.Resource, Kind: link.KindRunner, State: link.StateUnhealthy,
+			Reason: "the tool runner's account joins no group: " + w.Why, Since: now.UTC(), Streak: 1}
+		if was, had := j.was[w.Resource]; had && was.Reason == v.Reason {
+			v.Since, v.Streak = was.Since, was.Streak+1
+		} else {
+			changed = true
+		}
+		j.was[w.Resource] = v
+		out = append(out, v)
+	}
+	for i, l := range ls {
+		r := read[i]
+		id := apply.LibraryAccess(l.Library)
+		v := link.ResourceHealth{Resource: id, Kind: link.KindLibrary, Target: l.Root, State: r.state,
+			Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act}
+		if was, had := j.was[id]; had && was.State == r.state && was.Reason == r.reason {
+			v.Since = was.Since
+			v.Streak = was.Streak
+		} else {
+			changed = true
+		}
+		if r.state == link.StateUnhealthy {
+			v.Streak++
+		}
+		j.was[id] = v
+		out = append(out, v)
+	}
+	return out, changed
+}
+
+// withAccesses adds to a statement every library, as a resource of kind library.
+func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
+	h.Resources = append(h.Resources, vs...)
+	return h
+}
+
+// accessesHealthy says no library in vs is anything but healthy.
+func accessesHealthy(vs []link.ResourceHealth) bool {
+	for _, v := range vs {
+		if v.State != link.StateHealthy {
+			return false
+		}
+	}
+	return true
+}
+
+// keeper goes on with take acts stopped at their bound, beside the applies, one at a time (ADR 0328 §7).
+type keeper struct {
+	running atomic.Bool
+	say     func(string)
+}
+
+// keeping is the one keeper on this machine.
+var keeping = &keeper{say: func(string) {}}
+
+// poke goes on with every stopped take act of the declaration last applied, unless that is running already.
+func (k *keeper) poke(ctx context.Context) {
+	d := accesses.declaration()
+	if d == nil || len(d.Libraries) == 0 || !k.running.CompareAndSwap(false, true) {
+		return
+	}
+	go func() {
+		defer k.running.Store(false)
+		for _, line := range apply.ReconcileLibraries(ctx, d) {
+			k.say(line)
+		}
+	}()
+}
+
+// startKeeping has the keeper say what it does.
+func startKeeping(ctx context.Context, say func(string)) {
+	keeping.say = say
+	keeping.poke(ctx)
+}
+
+// accessActFrom is the take act as the controller asks it over the link (ADR 0328 §7): judged against the
+// declaration this node keeps, and run beside the applies.
+func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc {
+	return func(act link.AccessAct) (string, error) {
+		kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
+		if errors.Is(err, store.ErrNothingDeclared) {
+			return "", errors.New("this node keeps no declaration, so it is declared no library")
+		}
+		if err != nil {
+			return "", err
+		}
+		d, err := declaration.Parse(kept.Declaration)
+		if err != nil {
+			return "", fmt.Errorf("the declaration this node keeps does not read: %w", err)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		defer cancel()
+		_, said, err := apply.TakeLibrary(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err)
+			return "", err
+		}
+		return said, nil
+	}
+}
+
+// withWaiting is held with every id of waiting beside it: what liveness does not judge.
+func withWaiting(held, waiting map[string]bool) map[string]bool {
+	if len(waiting) == 0 {
+		return held
+	}
+	out := map[string]bool{}
+	for id := range held {
+		out[id] = true
+	}
+	for id := range waiting {
+		out[id] = true
+	}
+	return out
+}

cmd/mesh-host/libraries_test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// A library whose root holds data never taken is said waiting with one access wait: the access `library:<name>`, the
+// part its name, words with no path, the filesystem's name, and reason keep (novox/hq ADR 0328 §8); one whose root
+// is absent waits with reason disk (§9). Neither carries a module.
+func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
+	was := apply.Libraries
+	apply.Libraries = apply.NewLibraryMemory()
+	defer func() { apply.Libraries = was }()
+	dir := t.TempDir()
+	root := filepath.Join(dir, "media")
+	if err := os.MkdirAll(filepath.Join(root, "a film"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	d, err := declaration.Parse([]byte(`{"declaration":1,"groups":[{"name":"mesh-media","gid":62000},` +
+		`{"name":"mesh-books","gid":62001}],"libraries":[` +
+		`{"library":"media","root":"` + root + `","group":"mesh-media","gid":62000,"home":true},` +
+		`{"library":"books","root":"` + filepath.Join(dir, "books") + `","group":"mesh-books","gid":62001,"home":true}],` +
+		`"owns_nothing":true,"resources":[]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	j := newLibraryJudge()
+	j.set(d, nil)
+	vs, _ := j.look(time.Now())
+	if len(vs) != 2 {
+		t.Fatalf("the libraries are not said: %+v", vs)
+	}
+	books, media := vs[0], vs[1]
+	if media.Kind != link.KindLibrary || media.Resource != "library:media" || media.Module != "" ||
+		media.State != link.StateWaiting || len(media.AccessWaits) != 1 {
+		t.Fatalf("a library holding data is not said waiting: %+v", media)
+	}
+	raw, _ := json.Marshal(media.AccessWaits[0])
+	want := `{"kind":"access","access":"library:media","part":"media","what":"let the mesh keep the library media",` +
+		`"placed":"` + media.AccessWaits[0].Placed + `","reason":"keep"}`
+	if string(raw) != want || !strings.HasPrefix(media.AccessWaits[0].Placed, "fs-") {
+		t.Fatalf("the wait reads %s", raw)
+	}
+	if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
+		t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
+	}
+}

cmd/mesh-host/local_network_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/outward"
+	"github.com/novox/mesh-host/internal/profile"
+)
+
+func reportOn(networks ...outward.Network) link.Report {
+	return link.Report{Outward: []string{"eth0"},
+		Profile: profileAsReported(profile.Profile{Machine: &profile.Machine{Networks: networks}})}
+}
+
+// **A change of the networks a machine is attached to is said unasked** (novox/hq ADR 0340 §7): a port
+// reached from the local network is admitted from them, so a laptop that joins another network is said at
+// the next reconcile, not at the next push — and the same networks again are not news.
+func TestAChangedNetworkIsSaidUnasked(t *testing.T) {
+	w := &adoptionWatch{}
+	home := reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})
+	w.said(home)
+	if w.differs(reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})) {
+		t.Fatal("the same networks read as a change, so the machine would speak on every reconcile")
+	}
+	if !w.differs(reportOn(outward.Network{Link: "eth0", Network: "10.20.0.0/16"})) {
+		t.Fatal("a machine that moved to another network does not say so until it is pushed to")
+	}
+}

cmd/mesh-host/machine_refresh_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// Defends novox/hq issue 160: a machine nobody pushed to still says what it is made of once a day, so the
+// newest of it the mesh holds is no older than a day. Before, the reconcile said it only with something
+// else (a hold, a firewall, an outward link changed), and a converged machine's picture was as old as the
+// last delivery to it.
+func TestAMachineNobodyPushedToSaysWhatItIsOnceADay(t *testing.T) {
+	at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
+	w := &adoptionWatch{now: func() time.Time { return at }}
+	quiet := link.Report{Outward: []string{"wlan0"},
+		Profile: map[string]any{"machine": map[string]any{"observed_at": at}}}
+
+	if !w.news(quiet) {
+		t.Fatal("the first reconcile after the engine started did not say what the machine is")
+	}
+	w.said(quiet)
+
+	for _, after := range []time.Duration{5 * time.Minute, 12 * time.Hour, 22*time.Hour + 55*time.Minute} {
+		at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(after)
+		if w.news(quiet) {
+			t.Errorf("%s after the mesh heard it, an unchanged reconcile was said again", after)
+		}
+	}
+	// The reconcile that falls past 23 hours says it, so it reaches the mesh inside the day.
+	at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(23*time.Hour + 5*time.Minute)
+	if !w.news(quiet) {
+		t.Fatal("a day on, the reconcile did not say what the machine is")
+	}
+	w.said(quiet)
+	at = at.Add(5 * time.Minute)
+	if w.news(quiet) {
+		t.Error("once said, it was said again at the next reconcile")
+	}
+}
+
+// A delivery's account carries the profile too: heard, it starts the day again.
+func TestADeliveryHeardStartsTheDayAgain(t *testing.T) {
+	at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
+	w := &adoptionWatch{now: func() time.Time { return at }}
+	report := link.Report{Outward: []string{"eth0"}, Profile: map[string]any{"capabilities": []any{}}}
+	w.said(report)
+	at = at.Add(20 * time.Hour)
+	w.said(report) // a delivery's apply
+	at = at.Add(20 * time.Hour)
+	if w.news(report) {
+		t.Error("said again unasked 20 hours after a delivery told the mesh")
+	}
+}
+
+// A refusal carries nothing about the machine, so it is never said for being a day old.
+func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) {
+	at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
+	w := &adoptionWatch{now: func() time.Time { return at }}
+	if w.machineDue(link.Report{Refused: "no"}) {
+		t.Error("a refusal was due for what the machine is made of")
+	}
+}
+
+// A laptop that slept still says what it is within a day of wall time (review of mesh-host#92). Go's monotonic
+// clock stops while a machine is suspended, and Sub between two times that both carry a monotonic reading uses
+// it, so a watch that kept one would count a day asleep as almost nothing. Simulated with a clock that first
+// reads now (a monotonic reading), and after the sleep reads the wall time a day on: the watch must keep no
+// monotonic reading, and judge the day as the wall clock says it.
+func TestALaptopThatSleptStillSaysWhatItIsWithinADay(t *testing.T) {
+	before := time.Now() // carries a monotonic reading
+	asleep := before.Add(3 * time.Second)
+	if asleep.Sub(before) != 3*time.Second {
+		t.Fatal("the monotonic reading is not what this test simulates with")
+	}
+	calls := 0
+	w := &adoptionWatch{now: func() time.Time {
+		calls++
+		if calls == 1 {
+			return before
+		}
+		// The wall clock a day on, as a machine's clock reads after waking.
+		return asleep.Round(0).Add(24 * time.Hour)
+	}}
+	report := link.Report{Outward: []string{"wlan0"}, Profile: map[string]any{"capabilities": []any{}}}
+	w.said(report)
+	if strings.Contains(w.machineSaid.String(), "m=") {
+		t.Fatalf("the watch kept a monotonic reading (%s): a sleep would not count toward the day", w.machineSaid)
+	}
+	if !w.news(report) {
+		t.Error("a day of wall time on, a laptop that slept did not say what it is")
+	}
+}
+
+// And the watch's own clock never carries a monotonic reading.
+func TestTheWatchClockIsWallTime(t *testing.T) {
+	w := &adoptionWatch{}
+	if s := w.clock().String(); strings.Contains(s, "m=") {
+		t.Errorf("the watch's clock carries a monotonic reading: %s", s)
+	}
+	w = &adoptionWatch{now: time.Now}
+	if s := w.clock().String(); strings.Contains(s, "m=") {
+		t.Errorf("a given clock's time kept its monotonic reading: %s", s)
+	}
+}

cmd/mesh-host/main.go

@@ -24,14 +24,15 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"text/tabwriter"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/accounts"
 	"github.com/novox/mesh-host/internal/apply"
 	"github.com/novox/mesh-host/internal/bundle"
-	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/firewall"
 	"github.com/novox/mesh-host/internal/identity"
 	"github.com/novox/mesh-host/internal/inventory"
@@ -241,6 +242,8 @@ func run(ctx context.Context, command string, opts options) error {
 	// found (novox/hq issue 339).
 	handedOverFrom(opts.state)
 	foundState.Store(opts.state)
+	// And what it knows of the libraries it keeps is beside its state (novox/hq ADR 0328 §6).
+	apply.Libraries = store.NewLibraryFile(opts.state)
 	// And the guard asks the running container runtimes where they keep their data (novox/hq issue 339).
 	apply.AskRuntimes = apply.ExecRunner
 	switch command {
@@ -605,6 +608,12 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
 		return errors.New(olderGenerationWords(*refused))
 	}
 
+	// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
+	// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
+	if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
+		return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
+	}
+
 	// Said before anything is done.
 	steps := apply.Plan(d, known, origin)
 	if opts.json && opts.dryRun {
@@ -1121,11 +1130,13 @@ func runLink(ctx context.Context, opts options) error {
 	// this process's, a `reconcile` run by hand, the installer's — reads it (novox/hq issue 224).
 	sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state)))
 	go sched.Run(ctx)
+	// The take act's walks that stopped at their bound go on beside the applies (novox/hq ADR 0328 §7).
+	startKeeping(ctx, func(line string) { say(line) })
 
 	// The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every
 	// report this host makes carries what it says.
 	if j, err := liveness.Open(filepath.Join(filepath.Dir(opts.state), liveness.FileName),
-		&liveness.Exec{Run: apply.ExecRunner}); j != nil {
+		&liveness.Exec{Run: apply.ExecRunner, System: builtFor}); j != nil {
 		if err != nil {
 			say(err.Error())
 		}
@@ -1232,9 +1243,21 @@ func runLink(ctx context.Context, opts options) error {
 		// A read-only judgement of the directories this node is declared, asked by the controller (novox/hq
 		// issue 358): which the walk that follows no link another account could make would refuse.
 		JudgeDirectories: judgeDirectoriesFrom(opts.state),
-		Say:              say,
-		Timeout:          opts.timeout,
+		// The take act on a library, on the operator's warrant (novox/hq ADR 0328 §7).
+		AccessAct: accessActFrom(opts.state, opts.timeout),
+		Say:       say,
+		Timeout:   opts.timeout,
 	}
+	// And the move of a value sealed to this machine into the vault, asked by the controller (design 57's V3,
+	// issue 676): the vault is asked over this queue's link, so the queue is named once it exists.
+	queue.MoveSecret = moveSecretFrom(opts.state, queue)
+	// And this machine's sealing key stated to the vault in its own name, at start and whenever it changes (ADR 0372,
+	// issue 786), whether or not any value was ever moved in.
+	go keepNodeKeyStated(ctx, func() (identity.SealingKey, error) {
+		return identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
+	}, func(verb string, args any) (json.RawMessage, error) {
+		return queue.AskVault(ctx, verb, args)
+	}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo)
 	worker := make(chan struct{})
 	go func() {
 		defer close(worker)
@@ -1253,6 +1276,7 @@ func runLink(ctx context.Context, opts options) error {
 		queue.ReconcileDue()
 	}
 	go holdTheMachine(ctx, queue)
+	go busyPasses(ctx, queue, BusyAskEvery)
 	// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
 	// ADR 0240) — between applies, which is when a container crash-loops.
 	if judging != nil {
@@ -1272,6 +1296,9 @@ func runLink(ctx context.Context, opts options) error {
 	// And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller
 	// (novox/hq ADR 0272).
 	go serveMeshCLI(aside, queue, say)
+	// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
+	// 0349 §4).
+	go serveFilling(aside, opts, mine.Membership.Signer, say)
 
 	held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
 	// The act in hand finishes and is kept before this process exits: an apply that stood aside with
@@ -1352,6 +1379,32 @@ type adoptionWatch struct {
 	last string
 	// faulty is whether the last report the mesh heard said an apply refused or failed (novox/hq issue 233).
 	faulty bool
+	// machineSaid is when the mesh last heard what this machine is made of (novox/hq issue 160), and now the
+	// clock it is judged by (time.Now when nil).
+	machineSaid time.Time
+	now         func() time.Time
+}
+
+// machineEvery is how old what a machine said of itself may grow before a reconcile says it again unasked
+// (novox/hq issue 160): a machine nobody pushed to still says it once a day, so no fact the mesh holds of it
+// is older than a day. Under a day by more than one reconcile interval, so the reconcile that says it falls
+// inside the day rather than just past it.
+const machineEvery = 23 * time.Hour
+
+// clock is the wall time, its monotonic reading stripped (Round(0)). **Wall, because a laptop sleeps**: Go's
+// monotonic clock does not advance while the machine is suspended, so a day judged by it on a laptop that slept
+// for half of it is a day and a half of wall time, and what the mesh holds of the machine is older than a day.
+func (w *adoptionWatch) clock() time.Time {
+	if w.now != nil {
+		return w.now().Round(0)
+	}
+	return time.Now().Round(0)
+}
+
+// machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago
+// or more. A refusal carries none, so it never is.
+func (w *adoptionWatch) machineDue(r link.Report) bool {
+	return r.Refused == "" && len(r.Profile) > 0 && w.clock().Sub(w.machineSaid) >= machineEvery
 }
 
 // fingerprint is what a report says about adoption: each hold and whether it changed, the
@@ -1365,6 +1418,18 @@ func adoptionFingerprint(r link.Report) string {
 	// it could only speak when a declaration arrived, and a declaration cannot be composed until it
 	// has spoken: a machine waiting for a push that is waiting for the machine.
 	parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
+	// **And the networks those links are on** (novox/hq ADR 0340 §7): a port reached from the local
+	// network is admitted from them, so a laptop that joins another network, or a lease that moves, is
+	// said at the next reconcile rather than at the next push or the next day.
+	if networks := attachedNetworksOf(r); networks != "" {
+		parts = append(parts, "networks="+networks)
+	}
+	// And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk
+	// waiting on this machine starts its gate at the report that says it replaced. Not its output, which
+	// may change on every ask.
+	for _, b := range r.Busy {
+		parts = append(parts, "busy "+b.ID+"="+b.State)
+	}
 	for _, h := range r.Held {
 		parts = append(parts, "held "+h.ID+"="+h.Changed)
 	}
@@ -1396,6 +1461,26 @@ func adoptionFingerprint(r link.Report) string {
 	return strings.Join(parts, "\n")
 }
 
+// attachedNetworksOf is the networks a report's profile says the machine is attached to, as one line: the
+// part of what the machine is made of that the filter is composed from (novox/hq ADR 0340).
+func attachedNetworksOf(r link.Report) string {
+	machine, ok := r.Profile["machine"].(map[string]any)
+	if !ok {
+		return ""
+	}
+	networks, ok := machine["networks"].([]any)
+	if !ok {
+		return ""
+	}
+	said := make([]string, 0, len(networks))
+	for _, n := range networks {
+		if entry, ok := n.(map[string]any); ok {
+			said = append(said, fmt.Sprintf("%v %v", entry["link"], entry["network"]))
+		}
+	}
+	return strings.Join(said, ",")
+}
+
 // applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not
 // apply and which keys its failure is said under.
 func applyState(r link.Report) string {
@@ -1416,7 +1501,8 @@ func applyState(r link.Report) string {
 // faulty is whether a report says an apply refused or failed.
 func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 }
 
-// news is whether a reconcile's report is worth saying unasked: it says something the last one the mesh heard
+// news is whether a reconcile's report is worth saying unasked: what the machine is made of last reached the
+// mesh a day ago or more (novox/hq issue 160); or it says something the last one the mesh heard
 // did not, and it is about the machine (worthSaying) — or about how the apply went, refused or failed now, or
 // clean after the last one the mesh heard was not (novox/hq issue 233). A refusal says nothing about the
 // machine, but that the machine refuses is what the mesh raises a condition from, and a clean apply after it is
@@ -1424,6 +1510,10 @@ func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 }
 func (w *adoptionWatch) news(r link.Report) bool {
 	w.mu.Lock()
 	defer w.mu.Unlock()
+	// What the machine is made of, once a day whatever else changed (novox/hq issue 160).
+	if w.machineDue(r) {
+		return true
+	}
 	if adoptionFingerprint(r) == w.last {
 		return false
 	}
@@ -1444,6 +1534,9 @@ func (w *adoptionWatch) said(r link.Report) {
 	defer w.mu.Unlock()
 	w.last = adoptionFingerprint(r)
 	w.faulty = faulty(r)
+	if len(r.Profile) > 0 {
+		w.machineSaid = w.clock()
+	}
 }
 
 // changed is differs and said together, for a report published as it is made.
@@ -1573,10 +1666,10 @@ func declaredResolvConf(d *declaration.Declaration) (string, string, bool) {
 	}
 	for _, r := range d.Resources {
 		f, ok := r.(*declaration.File)
-		if !ok || f.Path != network.ResolvConf || f.CreateOnce || f.Into != "" {
+		if !ok || f.Path != network.ResolvConf || f.Seeded() || f.Into != "" {
 			continue
 		}
-		module, ok := liveness.ModuleOf(f.ID)
+		module, ok := d.ModuleOfID(f.ID)
 		if !ok {
 			return "", "", false
 		}
@@ -1596,7 +1689,7 @@ const (
 // judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on
 // the bus when a state changed, again every minute while one is not healthy, and every five minutes
 // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
-// rule 6).
+// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
 func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) {
 	ticker := time.NewTicker(liveness.LookEvery)
 	defer ticker.Stop()
@@ -1635,8 +1728,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
 				}
 			}
 		}
-		// Every read-write access that grants, read on every look (novox/hq ADR 0312 §2).
+		// Every library, judged on every look (novox/hq ADR 0328 §8, §9); and a take act stopped at its bound
+		// goes on, beside.
 		accessSt, accessesChanged := accesses.look(time.Now())
+		keeping.poke(ctx)
 		owed = owed || accessesChanged
 		if accessesChanged {
 			for _, v := range accessSt {
@@ -1691,10 +1786,15 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
 
 // healthAsReported is a statement as the report and the event carry it.
 func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
-	// GrantContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
-	// user's declared `root` (novox/hq ADR 0266), and grants a read-write access's path group (novox/hq ADR 0312),
-	// which is what tells the controller it may be sent any of those fields.
-	h := &link.Health{Contract: link.GrantContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
+	// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
+	// user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries
+	// (novox/hq ADR 0328), a container's busy (ADR 0335) and a file's owner-at-runtime and the filled files (ADR
+	// 0349 §5) and the provisioner client's account (ADR 0349 §4), which is what tells the controller it may be sent
+	// any of those fields; and it writes a filled file only on the client's word (HandedContract, design 57 P7),
+	// which is what tells it it may hand a moved module's files to the provisioner here; and it opens the vault's box
+	// for a core credential (VaultBoxContract, ADR 0349 §16), which is what tells it it may send that box here; and it states its sealing key to the vault (NodeKeyContract, ADR
+	// 0372, issue 786).
+	h := &link.Health{Contract: link.NodeKeyContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
 	for _, r := range st.Resources {
 		h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
 			Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
@@ -1718,7 +1818,8 @@ func withUnits(h *link.Health, us *units.Statement) *link.Health {
 	if us == nil || us.State == "" {
 		return h
 	}
-	h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread}
+	h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread,
+		Away: us.Away}
 	for _, f := range us.Failed {
 		if f.Module == "" {
 			h.Units.Failed = append(h.Units.Failed, link.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load,
@@ -1826,6 +1927,32 @@ func holdTheMachine(ctx context.Context, queue *link.Queue) {
 	}
 }
 
+// busyWaiting is whether the last apply left a container running because it said busy (novox/hq ADR 0335).
+var busyWaiting atomic.Bool
+
+// BusyAskEvery is how often a container waiting because it said busy is asked again (novox/hq ADR 0335 §2):
+// each ask is a pass of its own, so a build that ends is followed by its replacement within this, not
+// within the reconcile's five minutes — and the bound of 15 minutes is met within this of its end.
+const BusyAskEvery = 20 * time.Second
+
+// busyPasses asks for a pass every so often while a container waits because it said busy. **It asks; it
+// does not apply**, as holdTheMachine: the queue's worker does, when its turn comes, so a pass never holds
+// the machine for the wait, and a delivery waiting is that pass's apply.
+func busyPasses(ctx context.Context, queue *link.Queue, every time.Duration) {
+	ticker := time.NewTicker(every)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if busyWaiting.Load() {
+				queue.ReconcileDue()
+			}
+		}
+	}
+}
+
 // reconcileKept is the reconcile's act, as the queue's worker runs it: hold the machine to what it
 // was last told, read once it is this act's turn, and say what did not go.
 //
@@ -1877,7 +2004,7 @@ func worthSaying(report link.Report) bool {
 	}
 	return len(report.Held) > 0 || report.Firewall != "" || len(report.Outward) > 0 ||
 		len(report.Filters) > 0 || report.FoundFirewall != nil || len(report.Windows) > 0 ||
-		len(report.Rollbacks) > 0
+		len(report.Rollbacks) > 0 || len(report.Busy) > 0
 }
 
 // announceOr is what the apply writes its detail with, given what the caller has to say things with.
@@ -1993,6 +2120,14 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 		known.Mode = apply.ModeOf(declared)
 	}
 
+	// **A file given by digest is fetched and verified before anything of the declaration is applied** (novox/hq ADR
+	// 0486 §1, issue 1695): one object missing or differing refuses the declaration whole. The signed bytes are what is
+	// kept; only what is applied has the content in place of the reference.
+	if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
+		return link.Report{Declared: digestOf(raw), Order: order,
+			Refused: "a file of this declaration could not be had by its digest: " + err.Error()}
+	}
+
 	built, err := system.For(builtFor)
 	if err != nil {
 		return link.Report{Refused: err.Error()}
@@ -2075,7 +2210,9 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 	// What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just
 	// applied, so every resource this apply started says `starting` and the gate waits for it.
 	if j := judging; j != nil {
-		j.Set(liveness.LongRunning(declared, held))
+		// A container that waits for a library whose disk or mount is not there was not created: the library says
+		// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
+		j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared))))
 		st, _ := j.Look(ctx)
 		netJudge.declare(declaredResolvConf(declared))
 		var netSt *network.Statement
@@ -2093,13 +2230,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 		// said with it, relogin needed included, in the report that says the apply.
 		var accountSt *accounts.Statement
 		if a := accountJudge; a != nil {
-			// A read-write access's account too (novox/hq ADR 0312 §6): relogin needed is said for a session of
-			// the account's own manager that began before the grant, as for a user resource's group.
-			a.Set(append(accounts.Of(declared, held), apply.GrantedAccounts(declared, held)...))
+			a.Set(accounts.Of(declared, held))
 			as, _ := a.Look(ctx)
 			accountSt = &as
 		}
 		accesses.set(declared, held)
+		// A take act stopped at its bound goes on, beside (ADR 0328 §7).
+		keeping.poke(ctx)
 		accessSt, _ := accesses.look(time.Now())
 		report.Health = withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
 			accountSt), foundDirectories(updated)), accessSt)
@@ -2124,6 +2261,20 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 	for _, w := range outcome.Windows {
 		report.Windows = append(report.Windows, link.Window{Step: w.Step, Holds: w.Holds, Since: w.Opened, Until: w.Until})
 	}
+	// And a container left running because it said busy, or replaced after it waited (novox/hq ADR 0335):
+	// the walk waiting on this machine reads it. While one waits, this machine asks it again sooner than its
+	// reconcile (busyPasses).
+	for _, b := range outcome.Busy {
+		report.Busy = append(report.Busy, link.Busy{ID: b.ID, Container: b.Container, State: b.State,
+			Since: b.Since, Until: b.Until, Said: b.Said})
+	}
+	for _, m := range outcome.MovedOut {
+		report.MovedOut = append(report.MovedOut, link.MovedOut{Resource: m.Resource, From: m.From, To: m.To})
+	}
+	for _, l := range outcome.Left {
+		report.Left = append(report.Left, link.Left{Resource: l.Resource, Path: l.Path})
+	}
+	busyWaiting.Store(outcome.Waiting())
 	// And what runs here that nobody asked for (novox/hq ADR 0163).
 	if strays, err := apply.Strays(ctx, apply.ExecRunner, updated); err != nil {
 		fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not list what else runs here: %v\n", err)
@@ -2177,7 +2328,17 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 	}
 	// Kept whichever way it went, so a node that is disconnected next minute still knows what it
 	// was told. Saved after applying rather than before: what is kept is what this node acted on.
+	var before []string
 	if signed != nil {
+		// The modules the declaration kept until now held, so a have key written for one the new declaration no longer
+		// holds is written removed even by a node-engine that started since (issue 1369).
+		if kept, keptErr := store.ReadDeclared(store.DeclaredPath(opts.state)); keptErr == nil {
+			if old, parseErr := declaration.ParseTrusted(kept.Declaration); parseErr == nil {
+				for _, f := range moduleFacts(old, apply.Report{}, nil) {
+					before = append(before, f.Module)
+				}
+			}
+		}
 		if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil {
 			report.Failed = map[string]string{"keeping the declaration": err.Error()}
 		}
@@ -2186,6 +2347,10 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 		report.Failed = map[string]string{"apply": applyErr.Error()}
 	}
 	report.Failed = withUnanswered(report.Failed, probe)
+	// And how each module of the declaration came out, for the have keys written beside this report (novox/hq ADR 0429
+	// §4, issue 1132): not on the wire.
+	report.Modules = moduleFacts(declared, outcome, applyErr)
+	report.Before = before
 	// Which resources did not apply, for whether a reconcile's account is news (issue 233): not said on the wire.
 	report.Unapplied = unapplied(declared, outcome, applyErr)
 	return report
@@ -2236,7 +2401,7 @@ func sealOpener(statePath string) apply.Unseal {
 		if err != nil {
 			return nil, err
 		}
-		return key.Unseal(sealed)
+		return key.Open(sealed)
 	}
 }
 
@@ -2438,3 +2603,29 @@ func unhealthyWords(r liveness.State) string {
 	return fmt.Sprintf("%s (%s %s) is unhealthy: %s, %d automatic restart(s) since this engine began watching it",
 		r.ID, r.Kind, r.Target, r.Reason, r.Restarts)
 }
+
+// moduleFacts is how each module of a declaration came out of an apply, with the build the controller sent for it
+// (novox/hq ADR 0429 §4, issue 1132): sorted by module so the facts do not move between applies. A module an adopted
+// node has not taken is held as found and not applied, so it is no fact.
+func moduleFacts(d *declaration.Declaration, outcome apply.Report, applyErr error) []link.ModuleFact {
+	if d == nil {
+		return nil
+	}
+	results := apply.ModuleResults(d, outcome, applyErr)
+	facts := make([]link.ModuleFact, 0, len(results))
+	for module, result := range results {
+		if _, untaken := untakenModules(d)[module]; untaken {
+			continue
+		}
+		facts = append(facts, link.ModuleFact{Module: module, Build: d.Builds[module], Failed: result.Failed, Detail: result.Detail})
+	}
+	sort.Slice(facts, func(i, j int) bool { return facts[i].Module < facts[j].Module })
+	return facts
+}
+
+func untakenModules(d *declaration.Declaration) map[string][]string {
+	if d.Adoption == nil {
+		return nil
+	}
+	return d.Adoption.Untaken
+}

cmd/mesh-host/main_test.go

@@ -12,9 +12,9 @@ import (
 	"testing"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/apply"
 	"github.com/novox/mesh-host/internal/bundle"
-	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/identity"
 	"github.com/novox/mesh-host/internal/link"
 	"github.com/novox/mesh-host/internal/store"

cmd/mesh-host/module_facts_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"errors"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/apply"
+)
+
+// 1132/AC1, AC3: the report carries one fact per module the declaration names, with the build the controller sent for
+// it; a module not yet taken on an adopted node is not one.
+func TestTheReportCarriesOneFactPerModuleWithItsBuild(t *testing.T) {
+	d, err := declaration.Parse([]byte(`{"declaration":1,"builds":{"a":"b-1","b":"b-2"},"resources":[
+		{"id":"a.dir","type":"directory","path":"/srv/a","module":"a"},
+		{"id":"b.dir","type":"directory","path":"/srv/b","module":"b"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	failure := &apply.Error{Resource: "b.dir", Err: errors.New("no space"), Failures: []apply.Failure{{Resource: "b.dir", Err: errors.New("no space")}}}
+	facts := moduleFacts(d, apply.Report{}, failure)
+	if len(facts) != 2 || facts[0].Module != "a" || facts[0].Build != "b-1" || facts[0].Failed ||
+		facts[1].Module != "b" || facts[1].Build != "b-2" || !facts[1].Failed || facts[1].Detail == "" {
+		t.Fatalf("facts %+v", facts)
+	}
+	adopted, err := declaration.Parse([]byte(`{"declaration":1,"adoption":{"taken":["a"],"untaken":{"b":["b.dir"]}},"resources":[
+		{"id":"a.dir","type":"directory","path":"/srv/a","module":"a"},
+		{"id":"b.dir","type":"directory","path":"/srv/b","module":"b"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if facts := moduleFacts(adopted, apply.Report{}, nil); len(facts) != 1 || facts[0].Module != "a" {
+		t.Fatalf("an untaken module is a fact: %+v", facts)
+	}
+	if moduleFacts(nil, apply.Report{}, nil) != nil {
+		t.Fatal("facts without a declaration")
+	}
+}

cmd/mesh-host/move_out_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	osuser "os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/apply"
+)
+
+// TestAMoveOutDirectoryKeepsOnlyWhatItKnows drives the node-engine whole through its command (issue 1024, novox/hq
+// ADR 0422): a directory under an account's home that says move-out keeps the file the mesh declared and moves a
+// file nobody declared into the kept folder, named in a list, and the run says it moved it. It needs no bus: the
+// apply is the one a delivery and the five-minute reconcile both run.
+func TestAMoveOutDirectoryKeepsOnlyWhatItKnows(t *testing.T) {
+	me, err := osuser.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The user database lists an account as a person's only at uid 1000 or more; root, as the forge's merge check
+	// runs, is listed under a number of its own (it can give files to it), by the same name.
+	listedUID := me.Uid
+	if uid, _ := strconv.Atoi(me.Uid); uid < 1000 || uid == 65534 {
+		listedUID = "4242"
+	}
+	root := t.TempDir()
+	home := filepath.Join(root, "home", "agent")
+	dir := filepath.Join(home, ".ssh")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	passwd := filepath.Join(root, "passwd")
+	line := fmt.Sprintf("%s:x:%s:%s::%s:/bin/sh\n", me.Username, listedUID, me.Gid, home)
+	if err := os.WriteFile(passwd, []byte(line), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(apply.UseAccountsFileForTest(passwd))
+	// Independent of the machine: an arch host with a pacman on PATH that answers and holds nothing, as
+	// TestReplay233 puts one there (it answers with an error); nothing here declares a package.
+	bin := t.TempDir()
+	if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
+	wasBuilt := builtFor
+	builtFor = "arch"
+	t.Cleanup(func() { builtFor = wasBuilt })
+
+	stray := filepath.Join(dir, "id_stray")
+	if err := os.WriteFile(stray, []byte("somebody else's key"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	old := time.Now().Add(-time.Hour)
+	if err := os.Chtimes(stray, old, old); err != nil {
+		t.Fatal(err)
+	}
+	pipe := filepath.Join(dir, "agent.sock")
+	if err := syscall.Mkfifo(pipe, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(pipe, old, old); err != nil {
+		t.Fatal(err)
+	}
+	opts := options{state: filepath.Join(root, "state.json")}
+	raw := []byte(`{"declaration":1,"resources":[` +
+		`{"id":"ssh.dir","type":"directory","path":"` + dir + `","owner":"` + me.Username + `","unknown":"move-out"},` +
+		`{"id":"ssh.config","type":"file","path":"` + dir + `/config","content":"Host *\n","owner":"` + me.Username + `"}]}`)
+
+	var said []string
+	report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(s string) { said = append(said, s) })
+
+	if report.Refused != "" || len(report.Failed) > 0 {
+		t.Fatalf("refused or failed: %q %v", report.Refused, report.Failed)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "config")); err != nil {
+		t.Errorf("the declared file is not there: %v", err)
+	}
+	if _, err := os.Lstat(stray); err == nil {
+		t.Error("the file nobody declared stayed in the directory")
+	}
+	lists, _ := filepath.Glob(filepath.Join(home, ".local", "state", "mesh-moved", "ssh", "*.json"))
+	if len(lists) != 1 {
+		t.Fatalf("one list expected: %v", lists)
+	}
+	rawList, err := os.ReadFile(lists[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	var entries []struct{ From, To string }
+	if err := json.Unmarshal(rawList, &entries); err != nil || len(entries) != 1 {
+		t.Fatalf("the list: %s %v", rawList, err)
+	}
+	if got, err := os.ReadFile(entries[0].To); err != nil || string(got) != "somebody else's key" {
+		t.Errorf("the moved file is not whole in the kept folder: %q %v", got, err)
+	}
+	if len(report.MovedOut) != 1 || report.MovedOut[0].From != stray || report.MovedOut[0].To != entries[0].To {
+		t.Errorf("the report does not carry the move: %+v", report.MovedOut)
+	}
+	if len(report.Left) != 1 || report.Left[0].Path != pipe {
+		t.Errorf("the report does not list the pipe left in place: %+v", report.Left)
+	}
+	if !strings.Contains(strings.Join(said, "\n"), "moved "+stray) {
+		t.Errorf("the run does not say it moved the file: %v", said)
+	}
+}

cmd/mesh-host/movesecret.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"golang.org/x/crypto/nacl/box"
+
+	"github.com/novox/mesh-host/internal/identity"
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// The move of a value sealed to this machine into the vault (design 57's V3, issue 676; internal/link
+// movesecret.go). The controller cannot open what it holds; this machine can open what was sealed to it, and so
+// hands it to the vault, sealed to a key the vault made for the call. The value is in this process for that one
+// call and written nowhere.
+
+// vaultAsk asks one of the vault's verbs in this engine's own name and answers its result.
+type vaultAsk func(verb string, args any) (json.RawMessage, error)
+
+// moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over
+// the link open now.
+func moveSecretFrom(statePath string, queue *link.Queue) link.MoveSecretFunc {
+	return func(m link.MoveSecret) (link.MovedSecret, error) {
+		key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath))
+		if err != nil {
+			return link.MovedSecret{}, err
+		}
+		ask := func(verb string, args any) (json.RawMessage, error) {
+			return queue.AskVault(context.Background(), verb, args)
+		}
+		moved, err := moveIntoTheVault(ask, key, m)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
+		}
+		return moved, err
+	}
+}
+
+// moveIntoTheVault opens the row's value with this machine's sealing key, hands it to the vault's move-in in its two
+// steps, and answers what the vault kept once its read-back — sealed to a key made here for this one call — opens
+// to the same value.
+func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret) (link.MovedSecret, error) {
+	if m.SealedTo != "" && m.SealedTo != key.Public {
+		return link.MovedSecret{}, errors.New("the controller's store says this value was sealed to a sealing key this " +
+			"machine no longer has, so it cannot be opened here")
+	}
+	value, err := key.Unseal(m.Sealed)
+	if err != nil {
+		return link.MovedSecret{}, fmt.Errorf("the value does not open with this machine's sealing key: %w", err)
+	}
+	defer wipe(value)
+
+	// This machine's sealing key, stated in its own name: the key the vault seals the core's credentials to for the
+	// controller to carry here (novox/hq ADR 0349 §16, issue 763), never a key the controller names.
+	first := map[string]any{"origin": m.Origin, "generation": m.Generation, "sealing_key": key.Public}
+	if !m.MadeAt.IsZero() {
+		first["made_at"] = m.MadeAt.UTC().Format(time.RFC3339Nano)
+	}
+	if m.Own() {
+		first["node"], first["module"], first["name"] = m.Node, m.Module, m.Name
+	} else {
+		first["provider"], first["provision"] = m.Provider, m.Provision
+		first["consumer"], first["consumer_module"] = m.Consumer, m.ConsumerModule
+		if m.Local != "" {
+			first["local"] = m.Local
+		}
+	}
+	raw, err := ask("move-in", first)
+	if err != nil {
+		return link.MovedSecret{}, fmt.Errorf("the vault did not open the move: %w", err)
+	}
+	var opened struct {
+		Call   string `json:"call"`
+		SealTo string `json:"seal_to"`
+		Secret string `json:"secret"`
+	}
+	if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
+		return link.MovedSecret{}, errors.New("the vault answered no call and no key to seal to")
+	}
+	sealed, err := identity.Seal(opened.SealTo, value)
+	if err != nil {
+		return link.MovedSecret{}, fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
+	}
+	backPub, backPriv, err := box.GenerateKey(rand.Reader)
+	if err != nil {
+		return link.MovedSecret{}, err
+	}
+	defer wipe(backPriv[:])
+	raw, err = ask("move-in", map[string]any{"call": opened.Call, "sealed": sealed,
+		"read_back_to": base64.StdEncoding.EncodeToString(backPub[:])})
+	if err != nil {
+		return link.MovedSecret{}, fmt.Errorf("the vault did not take %s: %w", opened.Secret, err)
+	}
+	var kept struct {
+		Name       string `json:"name"`
+		Generation int    `json:"generation"`
+		Unchanged  bool   `json:"unchanged"`
+		ReadBack   string `json:"read_back"`
+	}
+	if err := json.Unmarshal(raw, &kept); err != nil || kept.Name == "" || kept.ReadBack == "" {
+		return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
+	}
+	blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
+	if err != nil {
+		return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s is not sealed", kept.Name)
+	}
+	back, ok := box.OpenAnonymous(nil, blob, backPub, backPriv)
+	if !ok {
+		return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s does not open with the key made for it", kept.Name)
+	}
+	defer wipe(back)
+	if subtle.ConstantTimeCompare(back, value) != 1 {
+		return link.MovedSecret{}, fmt.Errorf("the vault keeps %s, but it read back another value than the one handed "+
+			"in: the move of it did not hold", kept.Name)
+	}
+	return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
+}
+
+func wipe(b []byte) {
+	for i := range b {
+		b[i] = 0
+	}
+}

cmd/mesh-host/movesecret_test.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+
+	"golang.org/x/crypto/nacl/box"
+
+	"github.com/novox/mesh-host/internal/identity"
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// standInVault answers move-in's two steps as the vault does: a key made for the call, the value opened with it,
+// and the value kept read back sealed to the key the second step sent. lie makes its read-back another value.
+type standInVault struct {
+	t       *testing.T
+	pub     *[32]byte
+	priv    *[32]byte
+	opened  map[string]any
+	kept    string
+	lie     string
+	refuse  string
+	carried []string
+}
+
+func (v *standInVault) ask(verb string, args any) (json.RawMessage, error) {
+	raw, _ := json.Marshal(args)
+	v.carried = append(v.carried, string(raw))
+	if verb != "move-in" {
+		v.t.Fatalf("asked %s", verb)
+	}
+	if v.refuse != "" {
+		return nil, errors.New(v.refuse)
+	}
+	var a map[string]any
+	_ = json.Unmarshal(raw, &a)
+	if a["call"] == nil {
+		v.opened = a
+		pub, priv, _ := box.GenerateKey(rand.Reader)
+		v.pub, v.priv = pub, priv
+		return json.Marshal(map[string]string{"call": "c1", "seal_to": base64.StdEncoding.EncodeToString(pub[:]),
+			"secret": "own/anchor/postgres/superuser-password"})
+	}
+	blob, _ := base64.StdEncoding.DecodeString(a["sealed"].(string))
+	value, ok := box.OpenAnonymous(nil, blob, v.pub, v.priv)
+	if !ok {
+		v.t.Fatal("what the engine sealed does not open with the call's key")
+	}
+	v.kept = string(value)
+	back := v.kept
+	if v.lie != "" {
+		back = v.lie
+	}
+	key, _ := base64.StdEncoding.DecodeString(a["read_back_to"].(string))
+	var to [32]byte
+	copy(to[:], key)
+	boxed, _ := box.SealAnonymous(nil, []byte(back), &to, rand.Reader)
+	return json.Marshal(map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 1,
+		"read_back": base64.StdEncoding.EncodeToString(boxed)})
+}
+
+const theValue = "the-superusers-password"
+
+func aMoveOf(t *testing.T, key identity.SealingKey) link.MoveSecret {
+	t.Helper()
+	sealed, err := identity.Seal(key.Public, []byte(theValue))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return link.MoveSecret{Node: "anchor", By: "the move", MoveSecretAsk: link.MoveSecretAsk{Module: "postgres",
+		Name: "superuser-password", Origin: "made", Generation: "g-1", Sealed: sealed, SealedTo: key.Public}}
+}
+
+// The engine opens what was sealed to its machine, hands it to the vault sealed to the call's key — never in the
+// clear — and answers what the vault kept only once the read-back opens to the same value.
+func TestTheEngineMovesAValueSealedToItAndChecksTheReadBack(t *testing.T) {
+	key, err := identity.GenerateSealingKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	vault := &standInVault{t: t}
+	moved, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if moved.Secret != "own/anchor/postgres/superuser-password" || moved.Generation != 1 || vault.kept != theValue {
+		t.Fatalf("moved %+v, the vault kept %q", moved, vault.kept)
+	}
+	if vault.opened["module"] != "postgres" || vault.opened["name"] != "superuser-password" || vault.opened["node"] != "anchor" ||
+		vault.opened["origin"] != "made" || vault.opened["generation"] != "g-1" ||
+		vault.opened["sealing_key"] != key.Public {
+		t.Errorf("the first step named %v", vault.opened)
+	}
+	if strings.Contains(strings.Join(vault.carried, " "), theValue) {
+		t.Fatal("the value crossed the bus in the clear")
+	}
+}
+
+// A read-back that is not the value handed in is a failed move, said; so is a value this machine cannot open, and
+// the vault's refusal.
+func TestTheEngineSaysAMoveThatDidNotHold(t *testing.T) {
+	key, _ := identity.GenerateSealingKey()
+	vault := &standInVault{t: t, lie: "something-else"}
+	if _, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key)); err == nil ||
+		!strings.Contains(err.Error(), "read back") {
+		t.Fatalf("a read-back of another value: %v", err)
+	}
+	other, _ := identity.GenerateSealingKey()
+	if _, err := moveIntoTheVault((&standInVault{t: t}).ask, key, aMoveOf(t, other)); err == nil ||
+		!strings.Contains(err.Error(), "sealing key") {
+		t.Fatalf("a value sealed to another key: %v", err)
+	}
+	refusing := &standInVault{t: t, refuse: "the move never replaces a value the vault holds"}
+	if _, err := moveIntoTheVault(refusing.ask, key, aMoveOf(t, key)); err == nil ||
+		!strings.Contains(err.Error(), "never replaces") {
+		t.Fatalf("the vault's refusal: %v", err)
+	}
+	// A pair credential names both its ends.
+	pair := aMoveOf(t, key)
+	pair.Module, pair.Name = "", ""
+	pair.Provider, pair.Provision, pair.Consumer, pair.ConsumerModule = "anchor", "database", "shanks", "gitea"
+	ends := &standInVault{t: t}
+	if _, err := moveIntoTheVault(ends.ask, key, pair); err != nil {
+		t.Fatal(err)
+	}
+	if ends.opened["provider"] != "anchor" || ends.opened["consumer"] != "shanks" || ends.opened["consumer_module"] != "gitea" ||
+		ends.opened["node"] != nil {
+		t.Errorf("the pair's first step named %v", ends.opened)
+	}
+}

cmd/mesh-host/network_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/liveness"
 	"github.com/novox/mesh-host/internal/network"
 )

cmd/mesh-host/nodekey.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/novox/mesh-host/internal/identity"
+)
+
+// This machine's sealing key, stated to the vault in the engine's own name (novox/hq ADR 0372, issue 786). The vault
+// records it from the machine's own account alone and, when it is a first or a changed key, says the machine's core
+// secrets changed so the controller asks for them again sealed to it. Without this a machine that never moved a
+// secret in (adopted after feature 57) has no key at the vault. Nothing but the public half crosses.
+
+const (
+	// nodeKeyRetryFrom and nodeKeyRetryUpTo bound the wait between tries while the vault does not answer: it starts
+	// after the bus and may not serve the verb yet.
+	nodeKeyRetryFrom = 5 * time.Second
+	nodeKeyRetryUpTo = 5 * time.Minute
+	// nodeKeyLook is how often the key file is read to see whether it changed.
+	nodeKeyLook = 30 * time.Second
+)
+
+// keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from
+// retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line
+// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends.
+func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string),
+	retryFrom, retryUpTo time.Duration) {
+	stated, lastFailure := "", ""
+	wait := retryFrom
+	fail := func(why string) {
+		if why != lastFailure {
+			say("mesh-host: the sealing key was not stated to the vault, trying again: " + why)
+			lastFailure = why
+		}
+		wait = min(wait*2, retryUpTo)
+	}
+	look := min(nodeKeyLook, retryFrom)
+	for {
+		delay := look
+		key, err := load()
+		switch {
+		case err != nil:
+			fail(err.Error())
+			delay = wait
+		case key.Public != stated:
+			if _, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
+				fail(err.Error())
+				delay = wait
+			} else {
+				stated, lastFailure, wait = key.Public, "", retryFrom
+				say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(delay):
+		}
+	}
+}

cmd/mesh-host/nodekey_test.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/identity"
+	"github.com/novox/mesh-host/internal/link"
+)
+
+// Issue 786 (novox/hq ADR 0372): the engine states its own sealing key to the vault's node-key at start, retries
+// while the vault does not answer, and states it again when the key changes; the same key is not stated twice.
+type keyVault struct {
+	mu     sync.Mutex
+	stated []string
+	fail   int
+	verbs  []string
+}
+
+func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	v.verbs = append(v.verbs, verb)
+	if v.fail > 0 {
+		v.fail--
+		return nil, errors.New("the vault does not answer")
+	}
+	raw, _ := json.Marshal(args)
+	var a struct {
+		Key string `json:"key"`
+	}
+	_ = json.Unmarshal(raw, &a)
+	v.stated = append(v.stated, a.Key)
+	return json.RawMessage(`{"recorded":true}`), nil
+}
+
+func (v *keyVault) said() []string {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return append([]string(nil), v.stated...)
+}
+
+func waitFor(t *testing.T, what string, ok func() bool) {
+	t.Helper()
+	for i := 0; i < 400; i++ {
+		if ok() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("timed out: %s", what)
+}
+
+func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(t *testing.T) {
+	first, _ := identity.GenerateSealingKey()
+	second, _ := identity.GenerateSealingKey()
+	var mu sync.Mutex
+	current := first
+	load := func() (identity.SealingKey, error) {
+		mu.Lock()
+		defer mu.Unlock()
+		return current, nil
+	}
+	vault := &keyVault{fail: 2}
+	var lines []string
+	var lmu sync.Mutex
+	say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() }
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond)
+
+	waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 })
+	if vault.said()[0] != first.Public {
+		t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public)
+	}
+	vault.mu.Lock()
+	verbs := append([]string(nil), vault.verbs...)
+	vault.mu.Unlock()
+	for _, v := range verbs {
+		if v != "node-key" {
+			t.Errorf("asked %s", v)
+		}
+	}
+	// The same key again is not stated again.
+	time.Sleep(60 * time.Millisecond)
+	if n := len(vault.said()); n != 1 {
+		t.Errorf("the same key was stated %d times", n)
+	}
+	mu.Lock()
+	current = second
+	mu.Unlock()
+	waitFor(t, "the changed key stated", func() bool { return len(vault.said()) == 2 })
+	if vault.said()[1] != second.Public {
+		t.Errorf("stated %q after the change, want %q", vault.said()[1], second.Public)
+	}
+	lmu.Lock()
+	defer lmu.Unlock()
+	if len(lines) == 0 {
+		t.Error("the engine said nothing")
+	}
+}
+
+// The statement a host makes names this contract (issue 786), one above the vault's box.
+func TestTheNodeKeyContractIsThirteen(t *testing.T) {
+	if link.NodeKeyContract != 13 {
+		t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
+	}
+}

cmd/mesh-host/probe.go

@@ -6,8 +6,8 @@ import (
 	"os/exec"
 	"sort"
 
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/apply"
-	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/store"
 	"github.com/novox/mesh-host/internal/system"
 )

cmd/mesh-host/probe_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/link"
 	"github.com/novox/mesh-host/internal/system"
 )

cmd/mesh-host/replay233_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

cmd/mesh-host/units_test.go

@@ -5,8 +5,8 @@ import (
 	"testing"
 	"time"
 
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/accounts"
-	"github.com/novox/mesh-host/internal/declaration"
 	"github.com/novox/mesh-host/internal/link"
 	"github.com/novox/mesh-host/internal/liveness"
 	"github.com/novox/mesh-host/internal/units"
@@ -87,8 +87,13 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
 			Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
 	}}
 	h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
-	// At RootContract or later: an engine that also grants an access's group says GrantContract (novox/hq ADR 0312).
-	if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.GrantContract {
+	// At RootContract or later: an engine that states its sealing key to the vault says NodeKeyContract (ADR 0372, issue 786), one above
+	// VaultBoxContract (it opens the vault's box for the core's credentials, ADR 0349 §16, issue 755), one above HandedContract (design 57 P7), one above ProvisionerContract (novox/hq ADR
+	// 0349 §4), one above ADR 0349 §5's OwnershipContract.
+	if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.NodeKeyContract ||
+		link.NodeKeyContract != link.VaultBoxContract+1 || link.VaultBoxContract != link.HandedContract+1 ||
+		link.HandedContract != link.ProvisionerContract+1 || link.ProvisionerContract != link.OwnershipContract+1 || link.OwnershipContract != link.BusyContract+1 ||
+		link.BusyContract != link.OwnGroupContract+1 {
 		t.Fatalf("contract %d", h.Contract)
 	}
 	got := map[string]string{}

cmd/mesh-host/vaultbox_nats_test.go

@@ -0,0 +1,154 @@
+package main
+
+// Issue 755's proof on a real bus (novox/hq ADR 0349 §16): a core credential the vault sealed to this machine crosses
+// the bus inside the declaration, which the controller carries and cannot open, and this engine opens it with its
+// sealing key and writes the value. A listener on every subject keeps every byte that crossed; the value is never
+// among them. Run against the throwaway bus merge-check.sh starts (MESH_TEST_NATS).
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/seal"
+	"github.com/nats-io/nats.go"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/apply"
+	"github.com/novox/mesh-host/internal/identity"
+	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+const coreCredential = "the-provisioner-clients-bus-password-2025"
+
+func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(t *testing.T) {
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	// This machine's sealing key, where the engine reads it.
+	state := filepath.Join(t.TempDir(), "state.json")
+	key, err := identity.GenerateSealingKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	// Every byte that crosses.
+	tap, err := nats.Connect(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tap.Close()
+	var mu sync.Mutex
+	var crossed []string
+	if _, err := tap.Subscribe(">", func(m *nats.Msg) {
+		mu.Lock()
+		crossed = append(crossed, m.Subject+" "+string(m.Data))
+		mu.Unlock()
+	}); err != nil {
+		t.Fatal(err)
+	}
+	_ = tap.Flush()
+
+	// The engine hears its declaration on its own subject.
+	const node = "vaultbox"
+	engine, err := nats.Connect(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer engine.Close()
+	heard, err := engine.SubscribeSync(link.DeclareSubject(node))
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = engine.Flush()
+
+	// The vault's answer to the controller: the value in the ADR 0206 box, sealed to this machine's key. The
+	// controller carries it in the declaration as it is.
+	pub, err := key.PublicPEM()
+	if err != nil {
+		t.Fatal(err)
+	}
+	box, err := seal.Seal(coreCredential, pub)
+	if err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(t.TempDir(), "broker")
+	body, _ := json.Marshal(map[string]any{"declaration": 1, "resources": []map[string]any{
+		{"id": "need-provisioner-broker", "type": "file", "path": path, "vault-sealed": box}}})
+	controller, err := nats.Connect(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer controller.Close()
+	if err := controller.Publish(link.DeclareSubject(node), body); err != nil {
+		t.Fatal(err)
+	}
+	_ = controller.Flush()
+
+	msg, err := heard.NextMsg(5 * time.Second)
+	if err != nil {
+		t.Fatalf("the declaration did not reach the engine: %v", err)
+	}
+	d, err := declaration.Parse(msg.Data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sys, err := system.For("arch")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := apply.Apply(context.Background(), sys, d, store.State{}, store.OriginCarried,
+		func(context.Context, string, ...string) (string, error) { return "", nil }, nil, sealOpener(state)); err != nil {
+		t.Fatal(err)
+	}
+	on, err := os.ReadFile(path)
+	if err != nil || string(on) != coreCredential {
+		t.Fatalf("the engine wrote %q, %v", on, err)
+	}
+
+	_ = tap.Flush()
+	time.Sleep(100 * time.Millisecond)
+	mu.Lock()
+	all := strings.Join(crossed, "\n")
+	mu.Unlock()
+	if !strings.Contains(all, link.DeclareSubject(node)) || !strings.Contains(all, "vault-sealed") {
+		t.Fatalf("the listener heard nothing of the declaration:\n%s", all)
+	}
+	for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} {
+		if strings.Contains(all, form) {
+			t.Fatal("the core credential crossed the bus in the clear")
+		}
+	}
+}
+
+// The engine opens a value the controller sealed (the NaCl box) as before, and the vault's box, with one key.
+func TestTheEnginesOpenerOpensBothBoxes(t *testing.T) {
+	state := filepath.Join(t.TempDir(), "state.json")
+	key, _ := identity.GenerateSealingKey()
+	if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	open := sealOpener(state)
+	nacl, _ := identity.Seal(key.Public, []byte("old"))
+	if got, err := open(nacl); err != nil || string(got) != "old" {
+		t.Errorf("the controller's box opened to %q, %v", got, err)
+	}
+	pub, _ := key.PublicPEM()
+	box, _ := seal.Seal("new", pub)
+	raw, _ := json.Marshal(box)
+	if got, err := open(string(raw)); err != nil || string(got) != "new" {
+		t.Errorf("the vault's box opened to %q, %v", got, err)
+	}
+}

examples/README.md

@@ -2,51 +2,44 @@
 
 ## `foundation-first-node.lock`
 
-What a machine must be before a mesh exists — the bootstrap in
-[novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole:
+The one genesis bundle (novox/hq ADR 0331): what a machine must be before a mesh exists — the
+bootstrap in [novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole, on the bus the
+mesh runs on:
 
 ```
-0  a container runtime
+0  a container runtime, and a filter before anything listens
 1  the store runs
-2  a database per context        `inventory` and `identity`
+2  a database per context        `inventory`, `identity` and `licences`
 3  those contexts' schemas       mesh-controller migrate
-4  the broker runs               with a certificate it generated itself
-5  the control plane runs        mesh-controller serve
+4  the bus's certificate         mesh-controller broker certificate
+5  the bus runs                  NATS as mesh-broker-nats, where the nats module keeps it
+                                 (ADR 0333), with the controller's account as its first user
+6  the control plane runs        mesh-controller serve, given the bus (MESH_BUS_NATS)
 ```
 
 **A machine that applies this is a mesh** — one node, with nothing joined to it yet, which is
 exactly what the first node is (novox/hq ADR 0004). From here it hands out tokens and everything
 else joins the ordinary way.
 
-This file said it stopped at step 4 for longer than that was true, which is its own small lesson:
-a comment about what something does not do is a comment nobody updates.
+It is a **template**: `mesh-bootstrap` reads it, makes the store's superuser and the controller's
+account on the bus, and writes them where the template says `bootstrap` (internal/bootstrap,
+rootsecrets.go). A template whose bus connection it does not recognise is refused at preflight,
+naming the placeholder it looked for. The bundle that raised the previous broker was deleted with
+ADR 0331; there is one bundle, and every lab bed names it.
 
-Build a host carrying it:
+**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container
+`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in
+`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus
+port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once,
+on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest.
+
+Build an installer carrying a builder, and point it at this:
 
 ```
-make host SYSTEM=arch BUNDLE=examples/foundation-first-node.lock
+make bootstrap IMAGE=<builder image>
+mesh-bootstrap --bundle examples/foundation-first-node.lock …
 ```
 
-**The registry address and digests have to be replaced before this is useful.** They are written
-as `192.0.2.250:5000/…@sha256:…` because a digest belongs to whatever registry serves it — here,
-one a lab scenario raises, which reports its digests when it comes up. That is not a placeholder
-to be tidied away: a bundle is built *for a target*, and which registry that target pulls from is
-part of the target.
-
-### What was verified, and how
-
-On a lab machine confirmed sealed — `curl https://example.com` times out, the lab registry answers
-200 — the whole bundle applied from bare: eight resources, `inventory` created and `mesh` nowhere,
-the `node` table present with its indexes, the migration recorded, and LavinMQ answering
-`lavinmqctl status` with AMQP listening on 5672.
-
-Three consecutive reconciles after that: **already matches — 8 resource(s) checked**, each time.
-
-Then the machine was **rebooted**, and everything came back: docker from `boot: enabled`, both
-containers because the host creates every container `--restart unless-stopped`
-(`internal/apply/apply.go`), the schema intact in its named volume, and reconcile still finding
-nothing to do.
-
-The reboot is worth doing rather than assuming. Nothing in the declaration asks for a container to
-return, so that it does is a property of the host, and the only way to know it holds is to take
-the machine away and give it back.
+**The registry address is the lab's.** The images are written as `192.0.2.250:5000/…@sha256:…`; the
+lab rewrites the third-party ones to upstream references by the same digest, and the installer
+replaces the control plane's with the image it built.

examples/bundle_test.go

@@ -7,11 +7,12 @@ package examples
 
 import (
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 func bundle(t *testing.T) *declaration.Declaration {
@@ -48,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin
 	}
 	sort.Strings(images)
 
-	want := []string{"lavinmq", "mesh-controller", "postgres"}
+	want := []string{"mesh-controller", "nats", "pgvector"}
 	if strings.Join(images, ",") != strings.Join(want, ",") {
 		t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+
 			"Adding one is a change to what every first node becomes, and to ADR 0006's "+
@@ -67,3 +68,32 @@ func TestTheControlPlaneIsCarriedToo(t *testing.T) {
 	t.Fatal("nothing in the bundle starts the control plane, so the machine would raise a " +
 		"foundation and stop")
 }
+
+// One genesis bundle, on the bus the mesh runs on (novox/hq ADR 0331 §2). Two bundles let every bed
+// and README name the one that could not raise a mesh, for weeks (issue 146).
+func TestThereIsOneGenesisBundle(t *testing.T) {
+	bundles, err := filepath.Glob("*.lock")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(bundles) != 1 || bundles[0] != "foundation-first-node.lock" {
+		t.Fatalf("examples/ holds %v; it holds one genesis bundle, foundation-first-node.lock", bundles)
+	}
+}
+
+// The control plane the bundle raises is given the bus, with the account the installer fills and
+// the certificate it pins (ADR 0331 §4); without it, it crash-loops for ever (issue 146).
+func TestTheControlPlaneIsGivenTheBus(t *testing.T) {
+	for _, r := range bundle(t).Resources {
+		c, ok := r.(*declaration.Container)
+		if !ok || !strings.Contains(c.Image, "mesh-controller") {
+			continue
+		}
+		if !strings.HasPrefix(c.Env["MESH_BUS_NATS"], "nats://controller:bootstrap@") || c.Env["MESH_BROKER_CERTIFICATE"] == "" {
+			t.Fatalf("the control plane is given the bus %q and the certificate %q",
+				c.Env["MESH_BUS_NATS"], c.Env["MESH_BROKER_CERTIFICATE"])
+		}
+		return
+	}
+	t.Fatal("nothing in the bundle starts the control plane")
+}

examples/foundation-first-node-nats.lock

@@ -1,200 +0,0 @@
-// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being
-// built (novox/hq ADR 0106, design 25).
-//
-// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh
-// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the
-// first one — the controller's own account, at a well-known bootstrap password, exactly as the store
-// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and
-// from the controller's first composition onward the file is the controller's to write.
-//
-// The accounts file is its own file beside the server's configuration, because the server's own
-// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
-// one directory, of necessity: an include path is resolved relative to the including file's own
-// directory, so a server given an absolute one looks for it underneath that directory and refuses to
-// start.
-//
-// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
-// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
-// authenticates with a password (ADR 0004, design 25 §4).
-
-{
-  "declaration": 1,
-  "resources": [
-    {
-      "id": "container-runtime",
-      "type": "package",
-      "package": "docker"
-    },
-    {
-      "id": "container-runtime-running",
-      "type": "service",
-      "unit": "docker.service",
-      "state": "running",
-      "boot": "enabled"
-    },
-    // **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the
-    // broker are adopted as modules later and so bind to every interface from the moment they
-    // start; the packet filter that governs who may reach them is a module too, installed a
-    // dozen steps later. Between the two, a control-node facing the network had its store and
-    // its bus open to anyone who could reach the machine. So the foundation carries a filter of
-    // its own — the same table the filter module will replace wholesale once it can derive one:
-    // drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node
-    // enrols over, the registry a node pulls from), and let the container runtime's own
-    // networks through the forward chain so containers keep working. A published container port
-    // is forwarded, never input (issue 047), which is why the forward chain is where the store's
-    // and broker's ports are refused from outside — and a container on this machine dialling a
-    // port this machine publishes reaches it through the runtime's proxy, which IS input, which
-    // is why the bus and the registry are opened in both chains, exactly as the derived ruleset
-    // does.
-    {
-      "id": "base-filter-package",
-      "type": "package",
-      "package": "nftables"
-    },
-    {
-      "id": "base-filter",
-      "type": "file",
-      "path": "/etc/nftables.conf",
-      "mode": "0644",
-      "content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t\t# the mesh's own, from anywhere: the bus a node enrols over and a container on this machine reaches through the proxy, the registry a node pulls from\n\t\ttcp dport 5671 accept\n\t\ttcp dport 5000 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n"
-    },
-    {
-      "id": "base-filter-loaded",
-      "type": "service",
-      "unit": "nftables.service",
-      "state": "running",
-      "boot": "enabled",
-      "restart-on": ["base-filter"]
-    },
-    {
-      "id": "store",
-      "type": "container",
-      "name": "mesh-store",
-      "image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
-      "env": {
-        "POSTGRES_PASSWORD": "bootstrap",
-        "PGDATA": "/var/lib/postgresql/data/pgdata"
-      },
-      "ports": ["5432:5432"],
-      "volumes": ["mesh-store-data:/var/lib/postgresql/data"]
-    },
-    // Over TCP, not the socket. While the store initialises it runs a temporary server on the
-    // socket ONLY, then stops it and starts the real one — so a socket check passes, the action
-    // exits happy, and the verify a moment later lands in the gap and fails. The action and its
-    // verify must ask the same question, or the action can succeed into a state verify rejects.
-    {
-      "id": "store-ready",
-      "type": "action",
-      "in": "mesh-store",
-      "command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
-"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
-    },
-    {
-      "id": "inventory-database",
-      "type": "action",
-      "in": "mesh-store",
-      "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
-      "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
-    },
-    {
-      "id": "identity-database",
-      "type": "action",
-      "in": "mesh-store",
-      "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
-      "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
-    },
-    // Each context owns its own database (novox/hq ADR 0008). A third one is a third database,
-    // created the same way and named the same way — which is the whole of adding a context to the
-    // bootstrap, and is why the count is not something the foundation has an opinion about.
-    {
-      "id": "licences-database",
-      "type": "action",
-      "in": "mesh-store",
-      "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
-      "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
-    },
-    {
-      "id": "context-schemas",
-      "type": "action",
-      "command": ["docker", "run", "--rm", "--network", "container:mesh-store",
-                  "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
-                  "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
-        "-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
-                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
-                  "migrate"],
-      "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
-    },
-    {
-      "id": "bus-certificate",
-      "type": "action",
-      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
-      // broker's image while the broker was one that carried it; the bus that replaced it has a
-      // shell and no openssl, and no other image the bundle names has one either. So the program
-      // that needs the certificate writes it — already on this machine, since the schema step ran
-      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
-      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
-      // to ask an authority of.
-      // `--user 0:0` because the volume is root's and this image runs as nobody, which is right
-      // for the long-running control plane and wrong for a one-shot writing into a fresh volume.
-      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
-                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
-                  "broker", "certificate", "--into", "/tls"],
-      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
-                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
-                 "broker", "certificate", "--check", "--into", "/tls"]
-    },
-    {
-      "id": "bus-conf-dir",
-      "type": "directory",
-      "path": "/var/lib/mesh-bus-conf",
-      "mode": "0700"
-    },
-    {
-      "id": "bus-conf",
-      "type": "file",
-      "path": "/var/lib/mesh-bus-conf/nats.conf",
-      "mode": "0644",
-      "content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n  cert_file: \"/tls/tls.crt\"\n  key_file: \"/tls/tls.key\"\n}\n\njetstream {\n  store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
-    },
-    {
-      "id": "bus-accounts",
-      "type": "file",
-      "path": "/var/lib/mesh-bus-conf/accounts.conf",
-      "mode": "0600",
-      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
-    },
-    {
-      "id": "broker",
-      "type": "container",
-      "name": "mesh-broker",
-      "image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
-      "ports": ["5671:4222", "127.0.0.1:8222:8222"],
-      "volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
-      "args": ["-c", "/etc/nats/nats.conf", "-js"]
-    },
-    {
-      "id": "broker-ready",
-      "type": "action",
-      "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
-      "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
-    },
-    {
-      "id": "control-plane",
-      "type": "container",
-      "name": "mesh-controller",
-      "image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
-      "network": "host",
-      "args": ["serve"],
-      "volumes": ["mesh-broker-tls:/broker-tls:ro"],
-      "env": {
-        "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
-        "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
-        "MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
-        "MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
-        "MESH_BROKER_ADDRESS": "192.0.2.10:5671",
-        "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
-      }
-    }
-
-  ]
-}

examples/foundation-first-node.lock

@@ -1,36 +1,40 @@
-// foundation-first-node.lock — what a machine must be before a mesh exists.
+// foundation-first-node.lock — what a machine must be before a mesh exists: the one genesis bundle
+// (novox/hq ADR 0331 §2), on the bus the mesh runs on (ADR 0106, ADR 0131, design 25).
 //
 // The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a
-// store, a database per context, those contexts' schemas, the broker, and the control plane
-// running on top of them.
+// filter, a store, a database per context, those contexts' schemas, the bus with its certificate
+// and its first user list, and the control plane running on top of them.
 //
-// It stopped before the control plane once, and this comment said so for longer than it was true.
+// **The mesh writes its own user list, and at genesis there is no mesh yet to write it.** So this
+// carries the first one — the controller's own account — and the control plane's address names the
+// same account. Both say `bootstrap`, and both are placeholders: the installer makes the account's
+// password, puts it in the address and its hash in the list (mesh-host internal/bootstrap
+// rootsecrets.go), and refuses a bundle whose bus connection it does not recognise. Until the first
+// node has enrolled the installer places the list the controller composes; from the bus's adoption
+// as the `nats` module on, the module receives it like any module (ADR 0331 §1, §5).
 //
-// The broker generates its OWN certificate, in its own image, into a volume it then mounts read
-// only. Self-signed, because at this moment there is no mesh to issue one and no public name to
-// obtain one for -- and it does not matter, because what a joining node checks is the fingerprint
-// pinned in its token, not a chain or a name (novox/hq ADR 0004). The subject is decoration.
+// The accounts file is its own file beside the server's configuration, because the server's own
+// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
+// one directory, of necessity: an include path is resolved relative to the including file's own
+// directory, so a server given an absolute one looks for it underneath that directory and refuses to
+// start.
+//
+// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
+// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
+// authenticates with a password (ADR 0004, design 25 §4). The control plane pins it too, through
+// MESH_BROKER_CERTIFICATE, and dials the bus on the machine's own loopback.
 //
 // PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different
-// image, and this file is applied on a machine with no mesh to ask about anything. These digests
-// belong to the registry the lab raises, which is what a real node pulls from anyway — what is
-// required is a reference that is exact and cannot move (novox/hq ADR 0006).
+// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
+// registry prefix is the lab's; the digests are upstream's.
 //
-// The store waits up to three minutes rather than one. A machine that has just pulled the
-// image and is running initdb for the first time can take longer than sixty seconds, and it
-// failed that way three times in the lab -- a flaky bootstrap that a second run always fixed,
-// which is the worst kind because it teaches people to run things twice.
-//
-// It failed a fourth time on 2026-08-30, on a loaded machine, and the report was `docker exited
-// 1:` with nothing after the colon. Raising the timeout again would treat the symptom; what makes
-// a retry the only available response is a timeout that reports nothing. So the wait now says
-// what it saw before giving up.
-//
-// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
-// creates is owned by root, and the database runs as somebody else inside the container — so it
-// could not write, and the container crash-looped. A named volume lets the image set up its own
-// ownership, and outlives the container, which is what you want for the thing holding the mesh's
-// state.
+// **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container
+// `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by
+// the image's postgres user, its superuser password read from the module's own-secret file — so the
+// store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's
+// registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds
+// them to each other against the module's manifest).
+
 {
   "declaration": 1,
   "resources": [
@@ -80,17 +84,28 @@
       "boot": "enabled",
       "restart-on": ["base-filter"]
     },
+    // The store's data is a directory owned by the image's postgres user (999), made before the
+    // container starts: a directory the runtime made would be root's, and the database could not
+    // write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory
+    // made so (novox/hq ADR 0338 §5).
+    {
+      "id": "store-data",
+      "type": "directory",
+      "path": "/var/lib/mesh-store",
+      "mode": "0700",
+      "owner": "999:70"
+    },
     {
       "id": "store",
       "type": "container",
-      "name": "mesh-store",
-      "image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
+      "name": "postgres",
+      "image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f",
       "env": {
         "POSTGRES_PASSWORD": "bootstrap",
         "PGDATA": "/var/lib/postgresql/data/pgdata"
       },
       "ports": ["5432:5432"],
-      "volumes": ["mesh-store-data:/var/lib/postgresql/data"]
+      "volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"]
     },
     // Over TCP, not the socket. While the store initialises it runs a temporary server on the
     // socket ONLY, then stops it and starts the real one — so a socket check passes, the action
@@ -99,21 +114,21 @@
     {
       "id": "store-ready",
       "type": "action",
-      "in": "mesh-store",
+      "in": "postgres",
       "command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
 "verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
     },
     {
       "id": "inventory-database",
       "type": "action",
-      "in": "mesh-store",
+      "in": "postgres",
       "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
       "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
     },
     {
       "id": "identity-database",
       "type": "action",
-      "in": "mesh-store",
+      "in": "postgres",
       "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
       "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
     },
@@ -123,46 +138,112 @@
     {
       "id": "licences-database",
       "type": "action",
-      "in": "mesh-store",
+      "in": "postgres",
       "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
       "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
     },
     {
       "id": "context-schemas",
       "type": "action",
-      "command": ["docker", "run", "--rm", "--network", "container:mesh-store",
+      "command": ["docker", "run", "--rm", "--network", "container:postgres",
                   "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
                   "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
         "-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
                   "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
                   "migrate"],
-      "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
+      "verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
+    },
+    // **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
+    // `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
+    // /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the
+    // catalogue's module declares. So adopting it as that module at the broker step keeps it: the
+    // module's built image takes the place of the upstream one below, once, on the same streams, users
+    // and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other
+    // against the module's own manifest (busLayout), at the broker step and in its tests.
+    //
+    // Directories on the machine, not volumes the runtime names: the module mounts these paths, and the
+    // control plane, once it runs as the mesh's process, reads the certificate at its path on the machine.
+    // The certificate's directory is readable by everyone because the certificate is public and the control
+    // plane runs as nobody; its key is written 0600 by the program that makes it.
+    {
+      "id": "bus-tls-dir",
+      "type": "directory",
+      "path": "/var/lib/mesh-broker-tls",
+      "mode": "0755"
     },
     {
-      "id": "broker-certificate",
-      "type": "action",
-      "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                  "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
-                  "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
-      "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
-                 "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
-                 "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
+      "id": "bus-data-dir",
+      "type": "directory",
+      "path": "/var/lib/mesh-broker-nats",
+      "mode": "0700"
     },
+    {
+      "id": "bus-certificate",
+      "type": "action",
+      // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
+      // broker's image while the broker was one that carried it; the bus that replaced it has a
+      // shell and no openssl, and no other image the bundle names has one either. So the program
+      // that needs the certificate writes it — already on this machine, since the schema step ran
+      // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
+      // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
+      // to ask an authority of.
+      // `--user 0:0` because the directory is root's and this image runs as nobody, which is right
+      // for the long-running control plane and wrong for a one-shot writing into a fresh directory.
+      "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
+                  "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                  "broker", "certificate", "--into", "/tls"],
+      "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
+                 "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
+                 "broker", "certificate", "--check", "--into", "/tls"]
+    },
+    // **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are
+    // the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer
+    // run again after the adoption applies this bundle again, and an ordinary file would put the
+    // bundle's one user back over every user the mesh composed, and leave the node unable to reach its
+    // bus. Until the adoption the installer places the composed list over the seed (enrol.go).
+    {
+      "id": "bus-conf-dir",
+      "type": "directory",
+      "path": "/var/lib/nats-module/conf",
+      "mode": "0700"
+    },
+    {
+      "id": "bus-conf",
+      "type": "file",
+      "path": "/var/lib/nats-module/conf/nats.conf",
+      "mode": "0644",
+      "create-once": true,
+      "content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n  cert_file: \"/tls/tls.crt\"\n  key_file: \"/tls/tls.key\"\n}\n\njetstream {\n  store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
+    },
+    {
+      "id": "bus-accounts",
+      "type": "file",
+      "path": "/var/lib/nats-module/conf/accounts.conf",
+      "mode": "0600",
+      "create-once": true,
+      "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n  MESH {\n    jetstream: enabled\n    users = [\n      { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n        publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.mesh-controller_predefined.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.predefined-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n        subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n        allow_responses: { max: 1, ttl: \"1m\" }\n      } }\n    ]\n  }\n}\n"
+    },
+    // The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
+    // mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned
+    // step (ADR 0236). The release is the one the module's image is built on (its NATS_BASE, 2.11.17),
+    // pinned upstream by that same digest: the adoption changes no server release, so it can be undone
+    // by putting the old image back and is said reversible. The published port is the
+    // node's bus port, 5671 unless given, which every enrolment token names; the installer gives the
+    // module the setting that maps its declared 4222 to it (ADR 0333 §2).
     {
       "id": "broker",
       "type": "container",
-      "name": "mesh-broker",
-      "image": "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
-      "ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"],
-      "volumes": ["mesh-broker-data:/var/lib/lavinmq", "mesh-broker-tls:/tls:ro"],
-      "args": ["--amqps-port=5671", "--cert=/tls/tls.crt", "--key=/tls/tls.key"]
+      "name": "mesh-broker-nats",
+      "image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00",
+      "ports": ["5671:4222", "127.0.0.1:8222:8222"],
+      "volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"],
+      "args": ["-c", "/etc/nats/nats.conf", "-js"]
     },
     {
       "id": "broker-ready",
       "type": "action",
-      "in": "mesh-broker",
-      "command": ["sh", "-c", "for i in $(seq 1 60); do lavinmqctl status >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
-      "verify": ["lavinmqctl", "status"]
+      "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
+      "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671'"]
     },
     {
       "id": "control-plane",
@@ -171,13 +252,12 @@
       "image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
       "network": "host",
       "args": ["serve"],
-      "volumes": ["mesh-broker-tls:/broker-tls:ro"],
+      "volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"],
       "env": {
         "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
         "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
         "MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
-        "MESH_BROKER_AMQP": "amqp://guest:guest@127.0.0.1:5672/",
-        "MESH_BROKER_MANAGEMENT": "http://guest:guest@127.0.0.1:15672",
+        "MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
         "MESH_BROKER_ADDRESS": "192.0.2.10:5671",
         "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
       }

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75
 	github.com/nats-io/nats.go v1.54.0
 	golang.org/x/crypto v0.57.0
 	golang.org/x/sys v0.48.0

go.sum

@@ -1,9 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI=
-git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75 h1:AcYfyJpYaUufhI93VG76+b001msXY4XX7kuRjnf33HY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
 github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=

internal/accounts/accounts.go

@@ -45,7 +45,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // The states, in the words liveness says them (the controller reads them alike).
@@ -119,7 +119,7 @@ func secretsOf(d *declaration.Declaration, account string) []string {
 	var out []string
 	for _, r := range d.Resources {
 		f, ok := r.(*declaration.File)
-		if !ok || f.Path == "" || (f.Sealed == "" && len(f.Secrets) == 0) || f.Owner == account {
+		if !ok || f.Path == "" || (!f.Secret() && len(f.Secrets) == 0) || f.Owner == account {
 			continue
 		}
 		out = append(out, f.Path)

internal/accounts/accounts_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // The account judge (novox/hq ADR 0252, "how it is checked"): an account in a group its running session

internal/accounts/exec.go

@@ -285,3 +285,32 @@ func (e Exec) heldBy(pid string) (map[string]bool, error) {
 	}
 	return held, nil
 }
+
+// IDs is `getent passwd`: an account's uid and primary gid, and whether the user database names it at all. A
+// database that does not answer is an error, never "no such account".
+func (e Exec) IDs(ctx context.Context, account string) (uid, gid int, found bool, err error) {
+	out, err := e.Run(ctx, "getent", "passwd", account)
+	if err != nil {
+		if strings.Contains(err.Error(), "exited 2") || strings.Contains(err.Error(), "exit status 2") {
+			return 0, 0, false, nil
+		}
+		return 0, 0, false, fmt.Errorf("the user database did not answer about %q: %w", account, err)
+	}
+	fields := strings.Split(strings.TrimSpace(out), ":")
+	if len(fields) < 4 {
+		return 0, 0, false, fmt.Errorf("the user database gave %q for %q, which is not a passwd entry",
+			strings.TrimSpace(out), account)
+	}
+	// getent reads an all-digit key as a uid: "1000" answers with whoever has uid 1000. Only the account named is
+	// the one asked for.
+	if fields[0] != account {
+		return 0, 0, false, fmt.Errorf("the user database answered %q with the account %q: an account is named, "+
+			"never numbered", account, fields[0])
+	}
+	uid, uerr := strconv.Atoi(fields[2])
+	gid, gerr := strconv.Atoi(fields[3])
+	if uerr != nil || gerr != nil {
+		return 0, 0, false, fmt.Errorf("the user database gave no numbers for %q", account)
+	}
+	return uid, gid, true, nil
+}

internal/accounts/ids_test.go

@@ -0,0 +1,32 @@
+package accounts
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+)
+
+// An account is read by its name (novox/hq ADR 0325 §4): getent reads an all-digit key as a uid, so "1000" would
+// answer with whoever has uid 1000 — refused, never taken for the account named.
+func TestAnAccountIsReadByItsNameNeverItsNumber(t *testing.T) {
+	db := map[string]string{
+		"nox":  "nox:x:1003:1003::/home/nox:/bin/sh",
+		"1000": "jochen:x:1000:1000::/home/jochen:/bin/zsh", // what getent answers for a uid
+	}
+	e := Exec{Run: func(_ context.Context, name string, args ...string) (string, error) {
+		if line, ok := db[args[len(args)-1]]; ok {
+			return line + "\n", nil
+		}
+		return "", errors.New("getent exited 2")
+	}}
+	if uid, gid, found, err := e.IDs(context.Background(), "nox"); err != nil || !found || uid != 1003 || gid != 1003 {
+		t.Fatalf("nox: %d %d %v %v", uid, gid, found, err)
+	}
+	if _, _, found, err := e.IDs(context.Background(), "1000"); found || err == nil || !strings.Contains(err.Error(), "jochen") {
+		t.Fatalf("an all-digit name was read as a uid: found=%v err=%v", found, err)
+	}
+	if _, _, found, err := e.IDs(context.Background(), "ghost"); found || err != nil {
+		t.Fatalf("an absent account: found=%v err=%v", found, err)
+	}
+}

internal/accounts/root_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // An account declared never to become root without a person (novox/hq ADR 0266, "how it is checked"): each

internal/apply/access_test.go

@@ -3,7 +3,6 @@ package apply
 import (
 	"context"
 	"os"
-	"strings"
 	"testing"
 
 	"github.com/novox/mesh-host/internal/store"
@@ -32,28 +31,6 @@ func TestAnAccessPresentIsConfirmedAndNothingIsChanged(t *testing.T) {
 	}
 }
 
-// Absent is refused, not created. A bind mount whose source does not exist is made by the
-// container runtime as root, with whatever mode it picks — the silent wrong-ownership
-// 04-ISSUES/026 records. So the host says plainly that the operator must provide the path, rather
-// than conjuring a directory it does not own.
-func TestAnAccessThatIsAbsentIsRefusedClearlyAndNotCreated(t *testing.T) {
-	missing := t.TempDir() + "/media/library" // named, never created
-	d := declare(t, `{"id":"lib","type":"access","path":"`+missing+`"}`)
-
-	_, _, err := Apply(context.Background(), archHost(t), d, store.State{},
-		store.OriginDeclared, noServices, nil, nil)
-	if err == nil {
-		t.Fatal("an absent operator-owned path was accepted, and would be created as root by the runtime")
-	}
-	if !strings.Contains(err.Error(), "the operator must provide") ||
-		!strings.Contains(err.Error(), "does not own") {
-		t.Fatalf("the refusal does not say whose the path is: %v", err)
-	}
-	if _, statErr := os.Stat(missing); statErr == nil {
-		t.Fatal("the host created the path it does not own")
-	}
-}
-
 // Undeclaring an access never removes the path. Unassigning the module that reached the media
 // library must not delete the library — that is the data loss ADR 0030 exists to prevent, on a
 // directory the mesh never made. The record is dropped; the operator's data is left exactly as it

internal/apply/agent_home_guard.go

@@ -14,7 +14,7 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // AgentHomeRefusedError is an archive or a written-into file the engine will not place below an agent's home.

internal/apply/agent_home_guard_test.go

@@ -6,7 +6,7 @@ package apply
 import (
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 func TestNothingIsUnpackedOrWrittenIntoBelowAnAgentsHome(t *testing.T) {

internal/apply/apply_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 	"github.com/novox/mesh-host/internal/system"
 )

internal/apply/archive.go

@@ -13,7 +13,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/block.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/busy.go

@@ -0,0 +1,153 @@
+package apply
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335).
+//
+// **Asked, never waited on.** The apply asks the container once, and while it says busy leaves it running
+// as it is and goes on with everything else of the declaration: the machine's lock is held for one ask,
+// never for the wait. The wait is carried between passes in the container's record (store.Applied's
+// BusySince), so the next pass — the node-engine asks again sooner than its reconcile while anything waits
+// — asks again, and so does a node-engine started again in between: the bound runs from the first apply
+// that would have replaced the container, whoever applied it.
+
+// waitsBusy is the action of a container left running because it said busy (novox/hq ADR 0335 §2). Its own
+// word rather than "unchanged": its declaration moved, and the machine is not yet what was declared.
+const waitsBusy = "waits-busy"
+
+// What a report says of a busy container (novox/hq ADR 0335 §2, §3).
+const (
+	// BusyWaiting is a container left running because it said busy, waiting to be replaced.
+	BusyWaiting = "waiting"
+	// BusyReplacedIdle is a container replaced on the first pass it said idle, after it had waited.
+	BusyReplacedIdle = "replaced-when-idle"
+	// BusyReplacedAtBound is a container still busy at the bound, replaced anyway.
+	BusyReplacedAtBound = "replaced-at-bound"
+)
+
+// BusyWait is one container a busy command held, as a report says it.
+type BusyWait struct {
+	// ID is the resource's id (`<module>.<resource>`), Container its runtime name.
+	ID        string `json:"id"`
+	Container string `json:"container"`
+	// State is BusyWaiting, BusyReplacedIdle or BusyReplacedAtBound.
+	State string `json:"state"`
+	// Since is when the first apply that would have replaced it ran; Until when the bound replaces it.
+	Since time.Time `json:"since"`
+	Until time.Time `json:"until"`
+	// Said is the last output its busy command gave.
+	Said string `json:"said,omitempty"`
+}
+
+// Words is the wait in a sentence, for an outcome's detail and the console.
+func (b BusyWait) Words() string {
+	since := b.Since.UTC().Format(time.RFC3339)
+	said := ""
+	if b.Said != "" {
+		said = "; its busy command last said: " + b.Said
+	}
+	switch b.State {
+	case BusyReplacedAtBound:
+		return fmt.Sprintf("replaced while still busy: it said busy since %s and the bound of %s ran out "+
+			"(novox/hq ADR 0335 §3)%s", since, declaration.BusyBound, said)
+	case BusyReplacedIdle:
+		return fmt.Sprintf("replaced once it said idle, after waiting since %s (novox/hq ADR 0335 §2)%s", since, said)
+	}
+	return fmt.Sprintf("busy: left running and replaced on the first pass it says idle, or at %s at the latest; "+
+		"waiting since %s (novox/hq ADR 0335 §2)%s", b.Until.UTC().Format(time.RFC3339), since, said)
+}
+
+// owedSinceBusy is the reason a container still owes the replacement it was left running for, or empty: its
+// record keeps when its busy wait began, under the same name.
+func owedSinceBusy(r *declaration.Container, previous store.Applied) string {
+	if previous.BusySince == nil || previous.Target != r.Name {
+		return ""
+	}
+	return "the replacement owed since its busy wait began at " + previous.BusySince.UTC().Format(time.RFC3339)
+}
+
+// busyClock is the clock the bound is read on. A test moves it.
+var busyClock = time.Now
+
+// busySaidMost is how much of a busy command's output a report carries: its last lines, enough to say what
+// it held.
+const busySaidMost = 400
+
+// askBeforeReplacing asks a running container that declares busy whether it may be replaced now, and says
+// in out what came of it. True when it waits: out is then the whole outcome of this pass. False when the
+// caller replaces it — idle, or past the bound — with out carrying the wait it ended, if there was one.
+func askBeforeReplacing(ctx context.Context, r *declaration.Container, run Runner, cri string,
+	previous store.Applied, out *Outcome) bool {
+	now := busyClock().UTC()
+	busy, said := askBusy(ctx, r, run, cri)
+	var since time.Time
+	waited := previous.BusySince != nil && previous.Target == r.Name
+	if waited {
+		since = previous.BusySince.UTC()
+		// A wait kept from a clock that has since gone back reads as starting now, never as later: the bound
+		// holds from no later than this pass.
+		if since.After(now) {
+			since = now
+		}
+	}
+	wait := BusyWait{ID: r.ID, Container: r.Name, Since: since, Said: said}
+	switch {
+	case !busy && !waited:
+		// Idle at the first ask: replaced at once, as a container that declares nothing is.
+		return false
+	case !busy:
+		wait.State, wait.Until = BusyReplacedIdle, since.Add(declaration.BusyBound)
+		out.busy = &wait
+		return false
+	}
+	if !waited {
+		since = now
+		wait.Since = now
+	}
+	wait.Until = since.Add(declaration.BusyBound)
+	if !now.Before(wait.Until) {
+		wait.State = BusyReplacedAtBound
+		out.busy = &wait
+		return false
+	}
+	wait.State = BusyWaiting
+	out.Action, out.Detail = waitsBusy, wait.Words()
+	out.busySince, out.busy = &since, &wait
+	return true
+}
+
+// askBusy runs a container's busy command inside it, by its shell, as an exec health check is run: busy is
+// an exit of 0 and nothing else. An ask that could not be made, or did not answer within its timeout, is
+// not busy: the bound is what keeps a container whose command never answers from being held.
+func askBusy(ctx context.Context, r *declaration.Container, run Runner, cri string) (bool, string) {
+	ask, cancel := context.WithTimeout(ctx, r.Busy.Within())
+	defer cancel()
+	output, err := run(ask, cri, "exec", r.Name, "/bin/sh", "-c", r.Busy.Command)
+	said := strings.TrimSpace(output)
+	if err != nil && said == "" {
+		said = err.Error()
+	}
+	return err == nil, lastOf(said, busySaidMost)
+}
+
+// lastOf is the end of a text, at most n bytes, on one line.
+func lastOf(s string, n int) string {
+	s = strings.Join(strings.Fields(s), " ")
+	if len(s) <= n {
+		return s
+	}
+	from := len(s) - n
+	for from < len(s) && !utf8.RuneStart(s[from]) {
+		from++
+	}
+	return "…" + s[from:]
+}

internal/apply/busy_test.go

@@ -0,0 +1,433 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335). The
+// machine of these tests runs two containers, a build agent that declares `busy` and a web server that
+// does not; every call that replaces a container is counted.
+
+type aBusyMachine struct {
+	mu sync.Mutex
+	// containers is each container's name and the spec it was created with; running whether it runs.
+	containers map[string]string
+	busy       bool   // what the agent's busy command answers
+	said       string // what it prints
+	asked      int
+	calls      []string
+}
+
+func newBusyMachine() *aBusyMachine {
+	return &aBusyMachine{containers: map[string]string{}}
+}
+
+func (m *aBusyMachine) run(_ context.Context, _ string, args ...string) (string, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	switch args[0] {
+	case "info", "image":
+		return "27.0\n", nil
+	case "container": // inspect
+		name := args[len(args)-1]
+		spec, there := m.containers[name]
+		if !there {
+			return "", errors.New("no such container")
+		}
+		return "true\t" + spec, nil
+	case "exec":
+		m.asked++
+		m.calls = append(m.calls, "ask "+args[1])
+		if strings.Join(args[2:], " ") != "/bin/sh -c mesh-builder busy" {
+			return "", errors.New("asked something else: " + strings.Join(args[2:], " "))
+		}
+		if m.busy {
+			return m.said + "\n", nil
+		}
+		return m.said + "\n", errors.New("exited 1")
+	case "rm":
+		name := args[len(args)-1]
+		m.calls = append(m.calls, "rm "+name)
+		delete(m.containers, name)
+	case "run":
+		name := args[slices.Index(args, "--name")+1]
+		m.calls = append(m.calls, "create "+name)
+		for i, a := range args {
+			if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") {
+				m.containers[name] = strings.TrimPrefix(args[i+1], specLabel+"=")
+			}
+		}
+	}
+	return "", nil
+}
+
+func (m *aBusyMachine) mark() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return len(m.calls)
+}
+
+// since is every call since mark that replaced a container, or asked one.
+func (m *aBusyMachine) since(mark int) []string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return append([]string(nil), m.calls[mark:]...)
+}
+
+func (m *aBusyMachine) answer(busy bool, said string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.busy, m.said = busy, said
+}
+
+// twoContainers is the declaration at a version: v moves the agent's image tag through its env, w the web's.
+func twoContainers(t *testing.T, agent, web string) *declaration.Declaration {
+	t.Helper()
+	return parse(t, `{"declaration":1,"resources":[
+	  {"id":"build-agent.server","type":"container","name":"mesh-build-agent","image":"`+pinned+`",
+	   "env":{"V":"`+agent+`"},"busy":{"command":"mesh-builder busy","timeout":"5s"}},
+	  {"id":"web.server","type":"container","name":"web","image":"`+pinned+`","env":{"V":"`+web+`"}}
+	]}`)
+}
+
+// atClock moves the bound's clock for one test.
+func atClock(t *testing.T, at *time.Time) {
+	t.Helper()
+	was := busyClock
+	busyClock = func() time.Time { return *at }
+	t.Cleanup(func() { busyClock = was })
+}
+
+func busyOf(r Report, id string) (BusyWait, bool) {
+	for _, b := range r.Busy {
+		if b.ID == id {
+			return b, true
+		}
+	}
+	return BusyWait{}, false
+}
+
+func applyBusy(t *testing.T, m *aBusyMachine, d *declaration.Declaration, known store.State) (Report, store.State) {
+	t.Helper()
+	report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil)
+	if err != nil {
+		t.Fatalf("the apply failed: %v", err)
+	}
+	return report, state
+}
+
+// The decision's first check: an apply that would replace a container whose busy command exits 0 leaves it
+// running, applies the rest, and replaces it on the first pass the command exits non-zero.
+func TestABusyContainerIsLeftRunningAndReplacedOnTheFirstPassItSaysIdle(t *testing.T) {
+	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
+	atClock(t, &now)
+	m := newBusyMachine()
+	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
+
+	// A merge moves both: the agent holds a build, the web server holds nothing.
+	m.answer(true, "building novox/mesh-controller (check of #280) since 00:58")
+	mark := m.mark()
+	report, state := applyBusy(t, m, twoContainers(t, "2", "2"), state)
+	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm web", "create web"}) {
+		t.Fatalf("a busy agent was not left running while the rest applied: %v", got)
+	}
+	o := outcomeOf(report, "build-agent.server")
+	if o.Action != waitsBusy {
+		t.Fatalf("a busy container was reported %q, want %q: %+v", o.Action, waitsBusy, o)
+	}
+	for _, says := range []string{"busy", "first pass it says idle", "check of #280", "ADR 0335"} {
+		if !strings.Contains(o.Detail, says) {
+			t.Errorf("the outcome does not say %q: %s", says, o.Detail)
+		}
+	}
+	if (Report{Outcomes: []Outcome{o}}).Changed() {
+		t.Errorf("a container left running reads as a change: %+v", o)
+	}
+	b, said := busyOf(report, "build-agent.server")
+	if !said || b.State != BusyWaiting || !b.Since.Equal(now) || !b.Until.Equal(now.Add(15*time.Minute)) ||
+		b.Container != "mesh-build-agent" || !strings.Contains(b.Said, "check of #280") {
+		t.Fatalf("the report does not say the agent waits, since when and until when: %+v", report.Busy)
+	}
+	if !report.Waiting() {
+		t.Error("a report with a container waiting does not say so")
+	}
+	if rec, _ := state.Find("build-agent.server"); rec.BusySince == nil || !rec.BusySince.Equal(now) {
+		t.Fatalf("the wait is not kept in the container's record: %+v", rec)
+	}
+
+	// Asked again a minute later, still busy: still left running, the wait counted from the first apply.
+	now = now.Add(time.Minute)
+	mark = m.mark()
+	report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
+	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent"}) {
+		t.Fatalf("a second pass while busy did more than ask: %v", got)
+	}
+	if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-time.Minute)) {
+		t.Fatalf("the second pass does not keep the wait from the first apply: %+v", report.Busy)
+	}
+
+	// The build ends: the next pass replaces it, and says it waited.
+	now = now.Add(time.Minute)
+	m.answer(false, "idle")
+	mark = m.mark()
+	report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
+	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
+		t.Fatalf("the agent was not replaced on the first pass it said idle: %v", got)
+	}
+	if o := outcomeOf(report, "build-agent.server"); o.Action != "updated" || !strings.Contains(o.Detail, "once it said idle") {
+		t.Errorf("the replacement does not say it waited: %+v", o)
+	}
+	if b, _ := busyOf(report, "build-agent.server"); b.State != BusyReplacedIdle {
+		t.Errorf("the report does not say the agent was replaced when idle: %+v", report.Busy)
+	}
+	if report.Waiting() {
+		t.Error("a report with nothing waiting says something waits")
+	}
+	if rec, _ := state.Find("build-agent.server"); rec.BusySince != nil {
+		t.Errorf("the wait is still kept after the replacement: %+v", rec)
+	}
+
+	// And the pass after that has nothing to say about it.
+	report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
+	if len(report.Busy) != 0 {
+		t.Errorf("a container replaced is still said: %+v", report.Busy)
+	}
+}
+
+// An idle container is replaced at once, as one that declares nothing is: asked once, nothing waits.
+func TestAnIdleContainerIsReplacedAtOnce(t *testing.T) {
+	m := newBusyMachine()
+	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
+	m.answer(false, "")
+	mark := m.mark()
+	report, _ := applyBusy(t, m, twoContainers(t, "2", "1"), state)
+	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
+		t.Fatalf("an idle agent was not replaced at once: %v", got)
+	}
+	if len(report.Busy) != 0 {
+		t.Errorf("an idle container replaced at once is said as a wait: %+v", report.Busy)
+	}
+	// And a container whose declaration did not move is never asked.
+	mark = m.mark()
+	applyBusy(t, m, twoContainers(t, "2", "1"), state)
+	if got := m.since(mark); slices.Contains(got, "ask mesh-build-agent") {
+		t.Errorf("a container nothing replaces was asked: %v", got)
+	}
+}
+
+// Past the bound it is replaced anyway, the report carrying the command's last output — and the bound holds
+// across a restart of the node-engine: the wait is read back from the state on disk.
+func TestABusyContainerIsReplacedAtTheBoundEvenAfterARestart(t *testing.T) {
+	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
+	atClock(t, &now)
+	m := newBusyMachine()
+	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
+	m.answer(true, "building novox/mesh-host (a check that never ends)")
+	_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
+
+	// The node-engine stops and starts again: what it knows is what it saved.
+	path := filepath.Join(t.TempDir(), "state.json")
+	if err := store.Save(path, state); err != nil {
+		t.Fatal(err)
+	}
+	reloaded, err := store.Load(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	now = now.Add(14 * time.Minute)
+	mark := m.mark()
+	report, reloaded := applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
+	if got := m.since(mark); slices.Contains(got, "rm mesh-build-agent") {
+		t.Fatalf("a container busy within the bound was replaced after a restart: %v", got)
+	}
+	if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-14*time.Minute)) {
+		t.Fatalf("a restart started the wait again: %+v", report.Busy)
+	}
+
+	now = now.Add(time.Minute)
+	mark = m.mark()
+	report, _ = applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
+	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
+		t.Fatalf("a container still busy at the bound was not replaced: %v", got)
+	}
+	b, _ := busyOf(report, "build-agent.server")
+	if b.State != BusyReplacedAtBound || !strings.Contains(b.Said, "never ends") {
+		t.Fatalf("the report does not say it replaced a busy container, with its last output: %+v", report.Busy)
+	}
+	if o := outcomeOf(report, "build-agent.server"); !strings.Contains(o.Detail, "still busy") ||
+		!strings.Contains(o.Detail, "never ends") {
+		t.Errorf("the outcome does not say a busy container was replaced, with what it said: %s", o.Detail)
+	}
+}
+
+// While one container waits, a send that changes another module on the same machine is applied at once: the
+// apply returns after one ask, and holds nothing for the wait.
+func TestASendToAnotherModuleIsAppliedWhileOneWaits(t *testing.T) {
+	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
+	atClock(t, &now)
+	m := newBusyMachine()
+	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
+	m.answer(true, "building")
+	_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
+
+	// The next send moves only the web server.
+	mark := m.mark()
+	done := make(chan struct{})
+	var report Report
+	go func() {
+		defer close(done)
+		report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("an apply while a container waits did not return")
+	}
+	got := m.since(mark)
+	if !slices.Contains(got, "create web") || slices.Contains(got, "rm mesh-build-agent") {
+		t.Fatalf("the other module's send was not applied at once beside the wait: %v", got)
+	}
+	if o := outcomeOf(report, "web.server"); o.Action != "updated" {
+		t.Errorf("the web server was reported %q: %+v", o.Action, o)
+	}
+}
+
+// A container's busy is held to its shape where it arrives.
+func TestBusyIsReadAndHeldToItsShape(t *testing.T) {
+	d := twoContainers(t, "1", "1")
+	c := d.Resources[0].(*declaration.Container)
+	if c.Busy == nil || c.Busy.Command != "mesh-builder busy" || c.Busy.Within() != 5*time.Second {
+		t.Fatalf("busy was not read: %+v", c.Busy)
+	}
+	for name, raw := range map[string]string{
+		"no command":     `"busy":{"command":" "}`,
+		"a long timeout": `"busy":{"command":"x","timeout":"5m"}`,
+		"not a duration": `"busy":{"command":"x","timeout":"soon"}`,
+		"an unknown key": `"busy":{"command":"x","every":"1m"}`,
+		"on a step":      `"run-once":true,"busy":{"command":"x"}`,
+	} {
+		_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"a.b","type":"container",` +
+			`"name":"b","image":"` + pinned + `",` + raw + `}]}`))
+		if err == nil {
+			t.Errorf("%s: a busy of that shape was accepted", name)
+		}
+	}
+}
+
+// A replacement owed to a restart-on trigger is still owed after the wait: a trigger that is no file or
+// directory (a service, an identity's or a library's change) is in no container's spec and counts as changed only
+// in the pass it changed, so the container must not read unchanged at the next pass and drop from the report as if
+// replaced (novox/hq ADR 0335).
+func TestAReplacementOwedToARestartTriggerIsMadeOnceIdle(t *testing.T) {
+	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
+	atClock(t, &now)
+	d := parse(t, `{"declaration":1,"resources":[
+	  {"id":"build-agent.server","type":"container","name":"mesh-build-agent","image":"`+pinned+`",
+	   "restart-on":["identity:build-agent"],"busy":{"command":"mesh-builder busy"}}
+	]}`)
+	r := d.Resources[0].(*declaration.Container)
+	in := inputs{declares: map[string]string{}}
+	m := newBusyMachine()
+	m.containers["mesh-build-agent"] = containerSpec(r, in)
+	ctx := context.Background()
+
+	// The trigger moved this pass, and the agent holds a build: left running.
+	m.answer(true, "building")
+	out, err := applyContainer(ctx, r, m.run, map[string]bool{"identity:build-agent": true}, in, store.Applied{})
+	if err != nil || out.Action != waitsBusy || out.busySince == nil {
+		t.Fatalf("a busy container was not left running for its trigger: %+v %v", out, err)
+	}
+	record := store.Applied{ID: r.ID, Target: r.Name, BusySince: out.busySince}
+
+	// The next pass: nothing triggers now, and the replacement is still owed.
+	now = now.Add(time.Minute)
+	m.answer(false, "idle")
+	mark := m.mark()
+	out, err = applyContainer(ctx, r, m.run, map[string]bool{}, in, record)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
+		t.Fatalf("the replacement owed to the trigger was dropped: %v (%+v)", got, out)
+	}
+	if out.busy == nil || out.busy.State != BusyReplacedIdle || out.busySince != nil {
+		t.Fatalf("the replacement does not say it was made once idle: %+v", out)
+	}
+}
+
+// A wait kept from a clock that later went back is never longer than the bound from now.
+func TestAWaitFromAClockThatWentBackIsClampedToNow(t *testing.T) {
+	now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
+	atClock(t, &now)
+	m := newBusyMachine()
+	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
+	m.answer(true, "building")
+	_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
+	now = now.Add(-time.Hour) // the clock went back an hour
+	report, _ := applyBusy(t, m, twoContainers(t, "2", "1"), state)
+	b, _ := busyOf(report, "build-agent.server")
+	if b.State != BusyWaiting || b.Since.After(now) || !b.Until.Equal(now.Add(15*time.Minute)) {
+		t.Fatalf("a wait from a later clock is not clamped to now: %+v (now %s)", b, now)
+	}
+}
+
+// An ask that fails — no shell in the image, a timeout, a paused container — is not busy: replaced at once.
+func TestAnAskThatFailsIsIdle(t *testing.T) {
+	m := newBusyMachine()
+	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
+	failing := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "exec" {
+			m.mu.Lock()
+			m.calls = append(m.calls, "ask "+args[1])
+			m.mu.Unlock()
+			return "", errors.New("OCI runtime exec failed: exec: \"/bin/sh\": no such file or directory")
+		}
+		return m.run(ctx, name, args...)
+	}
+	mark := m.mark()
+	report, _, err := Apply(context.Background(), archHost(t), twoContainers(t, "2", "1"), state,
+		store.OriginDeclared, failing, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
+		t.Fatalf("an ask that failed held the container: %v", got)
+	}
+	if len(report.Busy) != 0 {
+		t.Errorf("an ask that failed is said as a wait: %+v", report.Busy)
+	}
+}
+
+// A stopped container holds no work: it is replaced without being asked.
+func TestAStoppedContainerIsNotAsked(t *testing.T) {
+	m := newBusyMachine()
+	_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
+	m.answer(true, "would say busy")
+	stopped := func(ctx context.Context, name string, args ...string) (string, error) {
+		if args[0] == "container" && args[len(args)-1] == "mesh-build-agent" {
+			out, err := m.run(ctx, name, args...)
+			return strings.Replace(out, "true\t", "false\t", 1), err
+		}
+		return m.run(ctx, name, args...)
+	}
+	mark := m.mark()
+	if _, _, err := Apply(context.Background(), archHost(t), twoContainers(t, "2", "1"), state,
+		store.OriginDeclared, stopped, nil, nil); err != nil && !strings.Contains(err.Error(), "not running") {
+		t.Fatal(err)
+	}
+	got := m.since(mark)
+	if slices.Contains(got, "ask mesh-build-agent") || !slices.Contains(got, "rm mesh-build-agent") {
+		t.Fatalf("a stopped container was asked, or not replaced: %v", got)
+	}
+}

internal/apply/container_groups_test.go

@@ -0,0 +1,54 @@
+package apply
+
+import (
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+)
+
+// Issue 1637 (novox/hq ADR 0480 clause 7): a container's supplementary groups reach the runtime as --group-add.
+
+func groupsContainer(groups ...int64) *declaration.Container {
+	return &declaration.Container{ID: "n8n.app", Name: "n8n", Image: pinned, Groups: groups}
+}
+
+// 1637/AC1: both run commands carry one --group-add per group, and none without groups.
+func TestAContainerIsAddedItsSupplementaryGroups(t *testing.T) {
+	with := groupsContainer(2000, 2001)
+	for name, args := range map[string][]string{
+		"detached":   detachedRunArgs(with, "x"),
+		"foreground": foregroundRunArgs(with, "x"),
+	} {
+		got := " " + strings.Join(args, " ") + " "
+		if !strings.Contains(got, " --group-add 2000 ") || !strings.Contains(got, " --group-add 2001 ") {
+			t.Errorf("1637/AC1: the %s run does not add both groups: %s", name, got)
+		}
+	}
+	without := groupsContainer()
+	for name, args := range map[string][]string{
+		"detached":   detachedRunArgs(without, "x"),
+		"foreground": foregroundRunArgs(without, "x"),
+	} {
+		if strings.Contains(strings.Join(args, " "), "--group-add") {
+			t.Errorf("1637/AC1: the %s run adds a group none was declared", name)
+		}
+	}
+}
+
+// 1637/AC2: a changed set of groups recreates the container; no groups leaves the spec as it was.
+func TestAChangeOfGroupsRecreatesTheContainer(t *testing.T) {
+	none, a, b := groupsContainer(), groupsContainer(2000), groupsContainer(2000, 2001)
+	if containerSpec(none, inputs{}) == containerSpec(a, inputs{}) || containerSpec(a, inputs{}) == containerSpec(b, inputs{}) {
+		t.Error("1637/AC2: a change of groups does not change the container's spec")
+	}
+	same := groupsContainer(2000)
+	if containerSpec(a, inputs{}) != containerSpec(same, inputs{}) {
+		t.Error("1637/AC2: the same groups changed the spec")
+	}
+	plain := *none
+	plain.Groups = nil
+	if containerSpec(none, inputs{}) != containerSpec(&plain, inputs{}) {
+		t.Error("1637/AC2: a container with no groups has its spec moved")
+	}
+}

internal/apply/declared_home_test.go

@@ -0,0 +1,158 @@
+package apply
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+)
+
+// A module that declares its own system account and a .ssh in that account's home is placed (novox/hq issue 1542):
+// the account is not in the user database yet when the declaration is judged, and its home is under /var/lib.
+func declaredHomeFixture(t *testing.T) (home string, user *declaration.User) {
+	t.Helper()
+	dir := t.TempDir()
+	passwd := filepath.Join(dir, "passwd")
+	if err := os.WriteFile(passwd, []byte("root:x:0:0::/root:/bin/sh\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	before := passwdFile
+	passwdFile = passwd
+	t.Cleanup(func() { passwdFile = before })
+	home = filepath.Join(dir, "lib", "mesh-build-go")
+	return home, &declaration.User{ID: "mesh-build-go.account", Type: declaration.TypeUser, Name: "mesh-build-go",
+		Shell: "/usr/bin/nologin", Home: home}
+}
+
+// 1542/AC1: the .ssh of a declared account, owned by it, is refused by nothing.
+func TestIssue1542ADeclaredAccountsSSHIsPlaced(t *testing.T) {
+	home, user := declaredHomeFixture(t)
+	ssh := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
+		Mode: "0700", Owner: "mesh-build-go"}
+	refused := refusedPlaces([]declaration.Resource{user, ssh})
+	if len(refused) != 0 {
+		t.Fatalf("a declared account's own .ssh was refused: %v", refused)
+	}
+	if err := (inputs{refused: refused}).refused[filepath.Clean(ssh.Path)]; err != nil {
+		t.Fatalf("the apply-time check refuses it: %v", err)
+	}
+	if line := JudgeDirectories(&declaration.Declaration{Resources: []declaration.Resource{user, ssh}}); strings.Contains(line, "refused already by the placement guard: ") {
+		t.Fatalf("the read-only judgement refuses it: %s", line)
+	}
+}
+
+// 1542/AC2: another account's directory below the declared home is refused as below any home, and a .ssh below
+// no home of the machine or the declaration is refused with today's words.
+func TestIssue1542OnlyTheDeclaredAccountsOwnIsPlaced(t *testing.T) {
+	home, user := declaredHomeFixture(t)
+	other := &declaration.Directory{ID: "notes.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
+		Mode: "0700", Owner: "intruder"}
+	refused := refusedPlaces([]declaration.Resource{user, other})
+	err := refused[filepath.Clean(other.Path)]
+	if err == nil || !strings.Contains(err.Error(), "below mesh-build-go's home and declared intruder's") {
+		t.Fatalf("another account's directory below a declared home: %v", err)
+	}
+	stray := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory,
+		Path: filepath.Join(filepath.Dir(home), "elsewhere", ".ssh"), Mode: "0700", Owner: "mesh-build-go"}
+	refused = refusedPlaces([]declaration.Resource{user, stray})
+	err = refused[filepath.Clean(stray.Path)]
+	if err == nil || !strings.Contains(err.Error(), "a .ssh directory is placed only below its account's home, as that account's") {
+		t.Fatalf("a .ssh below no home: %v", err)
+	}
+	// Not a .ssh: a root-owned directory below the declared home is placed exactly as on main.
+	data := &declaration.Directory{ID: "mesh-build-go.data", Type: declaration.TypeDirectory, Path: home + "/data",
+		Mode: "0755", Owner: "root"}
+	if refused := refusedPlaces([]declaration.Resource{user, data}); len(refused) != 0 {
+		t.Fatalf("a root-owned directory below a declared home, not a .ssh, was refused: %v", refused)
+	}
+	// And without the account declared, the same .ssh is refused as it was.
+	own := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
+		Mode: "0700", Owner: "mesh-build-go"}
+	if refusedPlaces([]declaration.Resource{own})[filepath.Clean(own.Path)] == nil {
+		t.Fatal("a .ssh below an account nobody makes was placed")
+	}
+}
+
+// 1542/AC3: a declared account is no home to anyone but the directory placement guard.
+func TestIssue1542DeclaredAccountsAreNotHomesToTheOtherGuards(t *testing.T) {
+	home, user := declaredHomeFixture(t)
+	_ = refusedPlaces([]declaration.Resource{user})
+	if _, ok := accountsOfHomes()[home]; ok {
+		t.Fatal("a declared account became a home of the machine")
+	}
+	if homeAbove(home+"/.ssh") != "" {
+		t.Fatal("a declared account became a home for the links and the move-out")
+	}
+}
+
+// A declared account is added as a home and never takes one over (reviewer, issue 1542): the person's own .ssh
+// stays refused to every other account, whatever a declaration says about a user.
+func TestIssue1542ADeclaredAccountTakesNoExistingHome(t *testing.T) {
+	dir := t.TempDir()
+	passwd := filepath.Join(dir, "passwd")
+	body := "root:x:0:0::/root:/bin/sh\njochen:x:1000:1000::/home/jochen:/bin/sh\n" +
+		"svc:x:998:998::/var/lib/svc:/usr/bin/nologin\n"
+	if err := os.WriteFile(passwd, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	before := passwdFile
+	passwdFile = passwd
+	t.Cleanup(func() { passwdFile = before })
+	newHome := filepath.Join(dir, "lib", "mesh-build-go")
+	ssh := func(path, owner string) *declaration.Directory {
+		return &declaration.Directory{ID: "x.ssh", Type: declaration.TypeDirectory, Path: path, Mode: "0700", Owner: owner}
+	}
+	user := func(name, home string) *declaration.User {
+		return &declaration.User{ID: "x.user", Type: declaration.TypeUser, Name: name, Home: home}
+	}
+	// A declared user over a person's home, whoever it is named for.
+	for _, name := range []string{"evil", "postgres", "root"} {
+		d := ssh("/home/jochen/.ssh", name)
+		if refusedPlaces([]declaration.Resource{user(name, "/home/jochen"), d})[d.Path] == nil {
+			t.Errorf("%s, declared over jochen's home, was given jochen's .ssh", name)
+		}
+	}
+	// A declared home below a person's home.
+	d := ssh("/home/jochen/sub/.ssh", "evil")
+	if refusedPlaces([]declaration.Resource{user("evil", "/home/jochen/sub"), d})[d.Path] == nil {
+		t.Error("a home declared below jochen's home was counted")
+	}
+	// A name the machine has already, declared with another home: that is not its home.
+	d = ssh(newHome+"/e/.ssh", "svc")
+	if refusedPlaces([]declaration.Resource{user("svc", newHome+"/e"), d})[d.Path] == nil {
+		t.Error("an existing account's declared home was counted")
+	}
+	// A number is no name: chown -1 leaves the owner as it was.
+	for _, owner := range []string{"-1", "-1:0", "0", "1001"} {
+		d := ssh(newHome+"/.ssh", owner)
+		if refusedPlaces([]declaration.Resource{user("mesh-build-go", newHome), d})[d.Path] == nil {
+			t.Errorf("owner %q was placed below a declared home", owner)
+		}
+	}
+	// And the account that is new, by name, still is.
+	d = ssh(newHome+"/.ssh", "mesh-build-go")
+	if got := refusedPlaces([]declaration.Resource{user("mesh-build-go", newHome), d}); len(got) != 0 {
+		t.Errorf("a new account's own .ssh was refused: %v", got)
+	}
+}
+
+// The account the mesh made, as declared, is judged the same at the next apply (issue 1542): once it is in the
+// user database with exactly the declared home, its .ssh is still placed.
+func TestIssue1542TheAccountTheMeshMadeIsStillPlacedAtTheNextApply(t *testing.T) {
+	home, user := declaredHomeFixture(t)
+	ssh := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
+		Mode: "0700", Owner: "mesh-build-go"}
+	resources := []declaration.Resource{user, ssh}
+	if got := refusedPlaces(resources); len(got) != 0 {
+		t.Fatalf("first apply, the account not yet made: %v", got)
+	}
+	body := "root:x:0:0::/root:/bin/sh\nmesh-build-go:x:998:998::" + home + ":/usr/bin/nologin\n"
+	if err := os.WriteFile(passwdFile, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if got := refusedPlaces(resources); len(got) != 0 {
+		t.Fatalf("second apply, the account made as declared: %v", got)
+	}
+}

internal/apply/filled.go

@@ -0,0 +1,234 @@
+package apply
+
+import (
+	"context"
+	"fmt"
+	"slices"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// WriteFilled writes what the provisioner's client filled into one file the declaration names as filled (novox/hq
+// ADR 0349 §4), and makes what reads the file take the change, exactly as an apply that changed it would.
+//
+// **The declared file says everything but the content.** Its path, mode, owner and who owns it at runtime are the
+// declaration's; the client's request carries the content alone. So the file is applied as the declared one with
+// that content in place of the declaration's: an owned file whole through applyFile, a seed only when absent, a
+// managed file's keys through applyInto or its block through applyBlock — the same guards on where nothing is
+// placed, the same Keep of an original found there, the same record. What the declaration seals or fills with
+// `${secret:}` the client has already filled, so the content is written as it came; a file that carried a secret
+// keeps the mode a secret is written with when the declaration gives none.
+//
+// **Then what reads it, in the apply's order** (afterWhatTheyRead): a service naming it under restart-on is
+// restarted and under reload-on reloaded, a process or container naming it under restart-on is restarted, and a
+// running container that reads it at creation — an env-file, a file mounted into it — is recreated, because what
+// it read is recorded by digest and this write changed the digest. Each of those is applied and recorded as an
+// apply does, and what moves goes on moving what names it. Nothing else of the declaration is touched.
+//
+// The caller holds the apply's lock: a write and an apply never write one file at once. log is said the path, the
+// action and the digest, never the content.
+//
+// **It does not refresh the runtimes' data roots.** A filled write is one file, not an apply; the placement guard
+// judges it with the roots the last apply read (runtimeDataRoots), falling back to the runtimes' configuration
+// files when no apply has read them yet — the runtimes themselves are asked once, at an apply's start, not once
+// per file (novox/hq issue 1138).
+func WriteFilled(
+	ctx context.Context,
+	sys system.System,
+	d *declaration.Declaration,
+	known store.State,
+	path string,
+	content []byte,
+	run Runner,
+	log func(string),
+	keep Keep,
+	windows *Windows,
+) (report Report, _ store.State, _ error) {
+	if log == nil {
+		log = func(string) {}
+	}
+	if !d.Fills(path) {
+		return report, known, &Error{Resource: path, Err: fmt.Errorf("this node's declaration does not name %s "+
+			"as a file the provisioner fills, so nothing was written", path)}
+	}
+	var declared []*declaration.File
+	for _, r := range d.Resources {
+		if f, ok := r.(*declaration.File); ok && f.Path == path {
+			declared = append(declared, f)
+		}
+	}
+	if len(declared) != 1 {
+		// The parser refuses a filled path no file declares; two at one path would need the request to say whose
+		// keys or block it carries, and it carries the content only.
+		return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
+			"is written for exactly one, so nothing was written", len(declared), path)}
+	}
+	file := filledAs(declared[0], content)
+	if held, isHeld := known.HeldAt(file.ID); isHeld {
+		return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
+			"yet taken (%s), so nothing was written", path, held.Why)}
+	}
+
+	in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
+		groups: groupsWanted(d.Resources), handed: Handed{}}
+	in.refused = refusedPlaces(d.Resources)
+	in.agentHomes = rootNeverHomes(d)
+	in.dirsBefore = map[string]bool{}
+	// A container that reads the file and mounts a library whose disk or mount is not there is held, as a full apply
+	// holds it: not created over the missing disk (novox/hq ADR 0328 §9).
+	in.held = heldLibraries(ctx, d.Libraries)
+	for _, r := range d.Resources {
+		in.declares[r.Identity()] = declaredDigest(r)
+	}
+	changed := map[string]bool{}
+
+	// The file, as the apply loop applies it.
+	was, _ := known.Find(file.ID)
+	var keepFound Keep
+	if (was.ID == "" || was.Target != file.Path) && !known.Recorded(string(declaration.TypeFile), file.Path) {
+		keepFound = keep
+	}
+	outcome, err := applyOne(ctx, sys, file, run, changed, in, was, nil, keepFound)
+	if err != nil {
+		log(fmt.Sprintf("  failed %s (%s): %v", file.ID, file.Path, err))
+		return report, known, &Error{Resource: file.ID, Err: err, Done: report}
+	}
+	recordKeeping(&known, file, outcome, was)
+	report.Outcomes = append(report.Outcomes, outcome)
+	if !moved(outcome) {
+		return report, known, nil
+	}
+	changed[file.ID] = true
+	line := fmt.Sprintf("  %s %s (%s), filled: %s", outcome.Action, file.ID, file.Path, outcome.wrote)
+	if outcome.Detail != "" {
+		line += ": " + outcome.Detail
+	}
+	log(line)
+
+	// What reads it, in the apply's order, each moving what names it in turn.
+	var failures []*Error
+	for _, r := range afterWhatTheyRead(d.Resources) {
+		if r.Identity() == file.ID || !readsChange(r, path, changed) {
+			continue
+		}
+		if _, isHeld := known.HeldAt(r.Identity()); isHeld {
+			log(fmt.Sprintf("  held %s (%s): held as found for a module not yet taken, so it was not restarted "+
+				"for %s", r.Identity(), r.Target(), file.ID))
+			continue
+		}
+		was, _ := known.Find(r.Identity())
+		o, err := applyOne(ctx, sys, r, run, changed, in, was, nil, nil)
+		if err != nil {
+			if o.found != nil {
+				known.KeepFound(r.Identity(), store.OriginDeclared, *o.found)
+			}
+			failures = append(failures, &Error{Resource: r.Identity(), Err: err})
+			log(fmt.Sprintf("  failed %s (%s): %v", r.Identity(), o.Target, err))
+			continue
+		}
+		if o.Action == "waiting" {
+			if o.found != nil {
+				known.KeepFound(r.Identity(), store.OriginDeclared, *o.found)
+			}
+			report.Outcomes = append(report.Outcomes, o)
+			log(fmt.Sprintf("  waiting %s (%s): %s", o.ID, o.Target, o.Detail))
+			continue
+		}
+		recordKeeping(&known, r, o, was)
+		report.Outcomes = append(report.Outcomes, o)
+		if o.busy != nil {
+			report.Busy = append(report.Busy, *o.busy)
+		}
+		if moved(o) {
+			changed[r.Identity()] = true
+			line := fmt.Sprintf("  %s %s (%s)", o.Action, o.ID, o.Target)
+			if o.Detail != "" {
+				line += ": " + o.Detail
+			}
+			log(line)
+		} else if o.Action != "unchanged" {
+			log(fmt.Sprintf("  %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail))
+		}
+	}
+	if len(failures) > 0 {
+		first := failures[0]
+		first.Done = report
+		first.Others = len(failures) - 1
+		return report, known, first
+	}
+	return report, known, nil
+}
+
+// filledAs is the declared file with the filled content in place of the declaration's.
+func filledAs(declared *declaration.File, content []byte) *declaration.File {
+	f := *declared
+	f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
+	if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) {
+		// Written as the declaration would have written it: a file carrying a credential is not world-readable.
+		f.Mode = "0600"
+	}
+	return &f
+}
+
+// readsChange reports whether applying r is what makes it take a change of what changed: a service's restart-on or
+// reload-on, a process's or container's restart-on, or a running container that reads path at creation.
+func readsChange(r declaration.Resource, path string, changed map[string]bool) bool {
+	switch res := r.(type) {
+	case *declaration.Service:
+		return len(restartedBy(res.RestartOn, changed)) > 0 || len(restartedBy(res.ReloadOn, changed)) > 0
+	case *declaration.Process:
+		return len(restartedBy(res.RestartOn, changed)) > 0
+	case *declaration.Container:
+		if len(restartedBy(res.RestartOn, changed)) > 0 {
+			return true
+		}
+		if res.RunOnce || res.Schedule != "" {
+			return false
+		}
+		if slices.Contains(res.EnvFile, path) {
+			return true
+		}
+		for _, v := range res.Volumes {
+			if mountSource(v) == path {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// recordKeeping records r as applied, carrying where the original it replaced was kept from its record when this
+// write kept none — for as long as the resource is recorded at the same target, as the apply loop carries it.
+func recordKeeping(known *store.State, r declaration.Resource, o Outcome, was store.Applied) {
+	kept, keptMode, keptOwner := o.kept, o.keptMode, o.keptOwner
+	if kept == "" && was.Target == o.Target {
+		kept, keptMode, keptOwner = was.Kept, was.KeptMode, was.KeptOwner
+	}
+	recordApplied(known, store.OriginDeclared, r, o, kept, keptMode, keptOwner)
+}
+
+// leftToTheClient is a filled file as an apply leaves it (novox/hq ADR 0349 §5, design 57 P7): untouched, and said
+// so. A file the mesh wrote at the same path under another id, going in this apply, is handed to it as an apply hands
+// a whole file (novox/hq ADR 0223): its record — what the mesh wrote there, and the original it kept — is the filled
+// file's now, so the verb judges what is there against it, and the file is neither removed nor written.
+func leftToTheClient(f *declaration.File, known *store.State, handedFiles map[string]store.Applied, report *Report,
+	log func(string)) Outcome {
+	if from, handed := handedFiles[f.ID]; handed {
+		if _, recorded := known.Find(f.ID); !recorded {
+			rec := from
+			rec.ID = f.ID
+			known.Record(rec)
+		}
+		delete(handedFiles, f.ID)
+		known.Forget(from.ID)
+		detail := "no longer declared; " + f.ID + " declares the file now, filled by the provisioner's client, so " +
+			"it was handed to it, not removed"
+		report.Outcomes = append(report.Outcomes, Outcome{ID: from.ID, Type: from.Type, Target: from.Target,
+			Action: "forgotten", Detail: detail})
+		log(fmt.Sprintf("  forgotten %s (%s): %s", from.ID, from.Target, detail))
+	}
+	return Outcome{ID: f.ID, Type: string(declaration.TypeFile), Target: f.Path, Action: "unchanged",
+		Detail: "filled by the provisioner's client, which alone has it written"}
+}

internal/apply/filled_apply_test.go

@@ -0,0 +1,218 @@
+package apply
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Defends novox/hq ADR 0349 §5, one writer per file (design 57 P7, issue 673): a file the declaration names as
+// filled is written only through the write verb (WriteFilled), on the provisioner client's word. An apply — the
+// controller's send and the five-minute re-apply alike — never writes it, never records it, and removes it as an
+// orphan only when the verb placed it.
+
+// An apply leaves a filled file alone, whatever who owns it at runtime says: absent stays absent and is not
+// recorded, and what the verb wrote stays as it wrote it through every re-apply.
+func TestAnApplyNeverWritesAFilledFile(t *testing.T) {
+	for _, c := range []struct{ name, fields, content string }{
+		{"owned", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n"},
+		{"seed", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n"},
+		{"managed block", `"into":"block","content":"","owner-at-runtime":"managed"`, "10.42.0.1 ace\n"},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			path := filepath.Join(t.TempDir(), "the-file")
+			d := filledDecl(t, path, c.fields, "")
+			report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
+				noServices, nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := os.Stat(path); err == nil {
+				t.Fatalf("an apply wrote a filled file the client has not filled: %q", readText(t, path))
+			}
+			if _, ok := state.Find("m.conf"); ok {
+				t.Errorf("an apply recorded a filled file it did not write")
+			}
+			if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" || !strings.Contains(o.Detail, "provisioner") {
+				t.Errorf("an apply did not say it left the file to the provisioner's client: %+v", o)
+			}
+
+			_, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte(c.content),
+				noServices, nil, nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			placed, _ := state.Find("m.conf")
+			want, wantMode := readText(t, path), modeAt(t, path)
+
+			// The re-apply, twice over: nothing written, the verb's record kept as it wrote it.
+			for i := 0; i < 2; i++ {
+				var said []string
+				report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
+					noServices, func(line string) { said = append(said, line) }, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if got := readText(t, path); got != want || modeAt(t, path) != wantMode {
+					t.Fatalf("re-apply %d wrote the filled file again: %q, want %q", i+1, got, want)
+				}
+				if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" {
+					t.Errorf("re-apply %d: %+v", i+1, o)
+				}
+				if rec, _ := state.Find("m.conf"); rec.Wrote != placed.Wrote || rec.Target != path {
+					t.Errorf("re-apply %d changed the verb's record: %+v, was %+v", i+1, rec, placed)
+				}
+				if len(said) != 0 {
+					t.Errorf("re-apply %d said something of an untouched file: %q", i+1, said)
+				}
+			}
+		})
+	}
+}
+
+// What reads a filled file is moved by the verb's write, once, never again by an apply: a service under
+// restart-on is not restarted and a container reading it is not recreated by the re-apply.
+func TestAnApplyMovesNothingForAFilledFile(t *testing.T) {
+	env := filepath.Join(t.TempDir(), "forge.env")
+	d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`,
+		`{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`)
+	m := &machine{containers: map[string]*fakeContainer{}}
+	_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"),
+		m.run, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m.asked = nil
+	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil,
+		nil); err != nil {
+		t.Fatal(err)
+	}
+	if m.removed("forge") || m.did("docker run") {
+		t.Errorf("the re-apply recreated the container reading the filled file: %v", m.asked)
+	}
+	if got := readText(t, env); got != "PORT=5432\n" {
+		t.Errorf("the re-apply wrote the filled file: %q", got)
+	}
+
+	conf := filepath.Join(t.TempDir(), "daemon.conf")
+	d = filledDecl(t, conf, `"content":"","owner-at-runtime":"owned"`,
+		`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`)
+	var commands []string
+	_, state, err = WriteFilled(context.Background(), archHost(t), d, store.State{}, conf, []byte("level = 1\n"),
+		recordingServices(&commands), nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	commands = nil
+	if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
+		recordingServices(&commands), nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if n := count(commands, "systemctl stop m.service"); n != 0 {
+		t.Errorf("the re-apply restarted what reads the filled file: %v", commands)
+	}
+}
+
+// A filled file the verb never placed is not the mesh's to remove: one found there, or put there by hand, stays
+// when the file leaves the declaration.
+func TestAFilledFileTheVerbNeverPlacedIsNotRemoved(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "the-file")
+	if err := os.WriteFile(path, []byte("the machine's own\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices,
+		nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	gone := elsewhere(t)
+	report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices,
+		nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := readText(t, path); got != "the machine's own\n" {
+		t.Errorf("a filled file the verb never placed was touched when it left: %q", got)
+	}
+	if o := outcomeOf(report, "m.conf"); o.Action == "removed" {
+		t.Errorf("a filled file the verb never placed was removed: %+v", o)
+	}
+	if _, ok := state.Find("m.conf"); ok {
+		t.Errorf("a record of the filled file was left")
+	}
+}
+
+// What the verb placed is the mesh's: when the file leaves the declaration it is removed as an orphan, as a file
+// an apply wrote is.
+func TestAFilledFileTheVerbPlacedIsRemovedWhenItLeaves(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "the-file")
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices,
+		nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte("port = 8080\n"),
+		noServices, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	gone := elsewhere(t)
+	report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices,
+		nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(path); err == nil {
+		t.Errorf("a filled file the verb placed stayed when it left the declaration: %q", readText(t, path))
+	}
+	if o := outcomeOf(report, "m.conf"); o.Action != "removed" {
+		t.Errorf("not removed: %+v", o)
+	}
+	if _, ok := state.Find("m.conf"); ok {
+		t.Errorf("its record stayed")
+	}
+}
+
+// A file the mesh wrote the old way and that is now filled keeps its record for the verb to judge against, and is
+// not written over by the apply that hands it to the client; a file at the same path recorded under another id
+// is handed to the filled one, not removed.
+func TestAFileThatMovesToTheClientIsHandedNotRewritten(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "the-file")
+	old := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.old","type":"file","path":%q,`+
+		`"content":"port = 1\n"}]}`, path))
+	_, state, err := Apply(context.Background(), archHost(t), old, store.State{}, store.OriginDeclared, noServices,
+		nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	was, _ := state.Find("m.old")
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
+	_, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := readText(t, path); got != "port = 1\n" {
+		t.Errorf("the file handed to the client was rewritten or removed: %q", got)
+	}
+	if _, ok := state.Find("m.old"); ok {
+		t.Errorf("the record going was kept beside the filled file's")
+	}
+	if rec, ok := state.Find("m.conf"); !ok || rec.Wrote != was.Wrote || rec.Target != path {
+		t.Errorf("the filled file did not take the record of what the mesh wrote there: %+v", rec)
+	}
+}
+
+// elsewhere is a declaration with nothing at the filled path: one unrelated file.
+func elsewhere(t *testing.T) *declaration.Declaration {
+	t.Helper()
+	return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"n.other","type":"file","path":%q,`+
+		`"content":"x\n"}]}`, filepath.Join(t.TempDir(), "other")))
+}

internal/apply/filled_library_test.go

@@ -0,0 +1,57 @@
+//go:build linux
+
+package apply
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A container that reads a filled file and mounts a library whose disk is not there is held by the write verb as a
+// full apply holds it (novox/hq ADR 0328 §9, issue 668): never created or recreated over the missing disk.
+func TestAWriteDoesNotRecreateAContainerHeldForItsLibrary(t *testing.T) {
+	// No folder is given the group: the library's root is absent, and nothing is made.
+	gid := 61000
+	libraries(t, nil, nil)
+	base := t.TempDir()
+	root := filepath.Join(base, "gone", "media")
+	env := filepath.Join(base, "radarr.env")
+	raw := fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":[%q],
+	  "groups":[{"name":"mesh-media","gid":%d}],"libraries":[%s],"resources":[
+	  {"id":"radarr.env","type":"file","path":%q,"content":"","mode":"0600","owner-at-runtime":"owned"},
+	  {"id":"radarr.app","type":"container","name":"radarr","image":%q,"env-file":[%q],"volumes":[%q]}]}`,
+		env, gid, libJSON("media", root, gid, true, false, "movies"), env, pinned, env,
+		filepath.Join(root, "movies")+":/movies")
+	d, err := declaration.Parse([]byte(raw))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		return "", nil
+	}
+	var state store.State
+	for _, content := range []string{"PORT=1\n", "PORT=2\n"} {
+		var report Report
+		report, state, err = WriteFilled(context.Background(), archHost(t), d, state, env, []byte(content), run,
+			nil, nil, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" {
+			t.Errorf("after writing %q the container is %+v, not held for its library", content, o)
+		}
+	}
+	for _, line := range ran {
+		if strings.HasPrefix(line, "docker run") || strings.HasPrefix(line, "docker rm") {
+			t.Errorf("the container was created or removed over a missing disk: %v", ran)
+		}
+	}
+}

internal/apply/filled_runtime_roots_test.go

@@ -0,0 +1,168 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A filled write is one file, not an apply: it asks no container runtime, and judges where nothing is placed
+// (issue 339) with the roots the last apply read, falling back to the runtimes' configuration files when no apply
+// has read them yet (novox/hq issue 1138). These tests set the package's runtimeRoots and AskRuntimes, so none
+// runs in parallel with another that reads or sets them, and each restores what it changed.
+
+// setRuntimeRoots makes runtimeDataRoots answer roots, as if an apply had just read them (or, with nil, as if none
+// ever had), restoring what a test before it left there once this one is done.
+func setRuntimeRoots(t *testing.T, roots []string) {
+	t.Helper()
+	runtimeRootsMu.Lock()
+	before := runtimeRoots
+	runtimeRoots = roots
+	runtimeRootsMu.Unlock()
+	t.Cleanup(func() {
+		runtimeRootsMu.Lock()
+		runtimeRoots = before
+		runtimeRootsMu.Unlock()
+	})
+}
+
+// askNoRuntime makes AskRuntimes record the name of every runtime it is asked for, answering as one that is not
+// running does (nothing, no error): were it consulted, that is what would say so. Restored once the test is done.
+func askNoRuntime(t *testing.T, asked *[]string) {
+	t.Helper()
+	before := AskRuntimes
+	AskRuntimes = func(_ context.Context, name string, _ ...string) (string, error) {
+		*asked = append(*asked, name)
+		return "", nil
+	}
+	t.Cleanup(func() { AskRuntimes = before })
+}
+
+// aRuntimeConfig points the runtimes' configuration at a fresh, unmounted root below a temp directory, restoring
+// the engine's real configuration once the test is done.
+func aRuntimeConfig(t *testing.T) (dir, root string) {
+	t.Helper()
+	dir = t.TempDir()
+	root = filepath.Join(dir, "docker")
+	daemon := filepath.Join(dir, "daemon.json")
+	if err := os.WriteFile(daemon, []byte(`{"data-root":"`+root+`"}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	before := runtimeConfigs
+	runtimeConfigs = runtimeFiles{daemonJSON: daemon, storageConf: filepath.Join(dir, "none.conf")}
+	t.Cleanup(func() { runtimeConfigs = before })
+	return dir, root
+}
+
+// refusedAsPlacement fails the test unless err is a PlacementRefusedError.
+func refusedAsPlacement(t *testing.T, err error) {
+	t.Helper()
+	var refusal *PlacementRefusedError
+	if err == nil || !errors.As(err, &refusal) {
+		t.Fatalf("a filled file below a refused runtime root was not refused: %v", err)
+	}
+}
+
+// 1138/AC1: a filled write asks no runtime, and judges with the roots the last apply read.
+func TestAFilledWriteAsksNoRuntimeAndUsesTheLastApplysRoots(t *testing.T) {
+	root := filepath.Join(t.TempDir(), "docker-root")
+	setRuntimeRoots(t, []string{root}) // as if an apply had just read it
+
+	var asked []string
+	askNoRuntime(t, &asked)
+
+	path := filepath.Join(root, "app.conf")
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
+	_, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("x\n"), noServices,
+		nil, nil, nil)
+
+	refusedAsPlacement(t, err)
+	if len(asked) != 0 {
+		t.Errorf("a filled write asked the container runtimes: %v", asked)
+	}
+}
+
+// 1138/AC2: given a real apply that read a runtime data root, a filled write below it is refused as today, with
+// no apply run again in between.
+func TestAFilledWriteBelowARootARealApplyReadIsRefused(t *testing.T) {
+	dir, root := aRuntimeConfig(t)
+	t.Cleanup(func() {
+		runtimeRootsMu.Lock()
+		runtimeRoots = nil
+		runtimeRootsMu.Unlock()
+	})
+
+	// A full apply of something unrelated reads the runtimes' configuration once, at its start.
+	elsewhere := filepath.Join(dir, "elsewhere.conf")
+	if _, _, err := Apply(context.Background(), archHost(t), declare(t,
+		fmt.Sprintf(`{"id":"m.other","type":"file","path":%q,"content":"x\n"}`, elsewhere)),
+		store.State{}, store.OriginDeclared, noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	// A filled write below the runtime's root, with no apply run since, is still refused.
+	path := filepath.Join(root, "sub", "app.conf")
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
+	_, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("x\n"), noServices,
+		nil, nil, nil)
+	refusedAsPlacement(t, err)
+}
+
+// 1138/AC3: with no apply having read the roots yet, a filled write judges by the runtimes' configuration files,
+// asking no runtime.
+func TestAFilledWriteWithNoApplyYetUsesTheRuntimesConfigFiles(t *testing.T) {
+	setRuntimeRoots(t, nil) // no apply has read the roots yet
+	_, root := aRuntimeConfig(t)
+
+	var asked []string
+	askNoRuntime(t, &asked)
+
+	path := filepath.Join(root, "sub", "app.conf")
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
+	_, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("x\n"), noServices,
+		nil, nil, nil)
+
+	refusedAsPlacement(t, err)
+	if len(asked) != 0 {
+		t.Errorf("a filled write with no apply yet asked the container runtimes: %v", asked)
+	}
+}
+
+// 1138/AC4: a full apply still asks each runtime once, at its start, whatever the number of files it writes.
+func TestAFullApplyAsksEachRuntimeOnceRegardlessOfFileCount(t *testing.T) {
+	dir := t.TempDir()
+	var asked []string
+	before := AskRuntimes
+	AskRuntimes = func(_ context.Context, name string, _ ...string) (string, error) {
+		asked = append(asked, name)
+		return "", errors.New("not running in this test")
+	}
+	t.Cleanup(func() {
+		AskRuntimes = before
+		runtimeRootsMu.Lock()
+		runtimeRoots = nil
+		runtimeRootsMu.Unlock()
+	})
+
+	var resources []string
+	for i := 0; i < 3; i++ {
+		resources = append(resources, fmt.Sprintf(`{"id":"m.f%d","type":"file","path":%q,"content":"x\n"}`, i,
+			filepath.Join(dir, fmt.Sprintf("f%d.conf", i))))
+	}
+	if _, _, err := Apply(context.Background(), archHost(t), declare(t, strings.Join(resources, ",")),
+		store.State{}, store.OriginDeclared, noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if got := count(asked, "docker"); got != 1 {
+		t.Errorf("docker was asked %d times for one apply of 3 files; want 1: %v", got, asked)
+	}
+	if got := count(asked, "podman"); got != 1 {
+		t.Errorf("podman was asked %d times for one apply of 3 files; want 1: %v", got, asked)
+	}
+}

internal/apply/filled_test.go

@@ -0,0 +1,271 @@
+package apply
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Defends novox/hq ADR 0349 §4 (issue 668): the provisioner's client has the node-engine write a file it filled,
+// and the node-engine writes it as an apply of the declared file with that content would — through applyFile,
+// applyInto or applyBlock by who owns it at runtime — and makes what reads it take the change as an apply does.
+
+// filledDecl is a declaration naming path as filled, with the file's fields after id, type and path, and more
+// resources after it.
+func filledDecl(t *testing.T, path, fields, more string) *declaration.Declaration {
+	t.Helper()
+	if more != "" {
+		more = "," + more
+	}
+	return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner",
+	  "filled_files":[%q],"resources":[{"id":"m.conf","type":"file","path":%q,%s}%s]}`, path, path, fields, more))
+}
+
+func TestAWriteOfAPathNotFilledIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	filled := filepath.Join(dir, "filled.conf")
+	other := filepath.Join(dir, "other.conf")
+	d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[
+	  {"id":"m.conf","type":"file","path":%q,"content":"x\n","owner-at-runtime":"owned"},
+	  {"id":"m.other","type":"file","path":%q,"content":"y\n"}
+	]}`, filled, filled, other))
+	for _, path := range []string{other, filepath.Join(dir, "nothing.conf")} {
+		_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("evil\n"),
+			noServices, nil, nil, nil)
+		if err == nil || !strings.Contains(err.Error(), "does not name") {
+			t.Errorf("%s: a write of a path the declaration does not fill was not refused: %v", path, err)
+		}
+		if _, err := os.Stat(path); err == nil && path != other {
+			t.Errorf("%s was written", path)
+		}
+		if len(state.Resources) != 0 {
+			t.Errorf("a refused write recorded something: %+v", state.Resources)
+		}
+	}
+	if _, err := os.Stat(other); err == nil {
+		t.Errorf("%s was written", other)
+	}
+}
+
+// Each ownership is written by the write verb exactly as an apply of the declared file with the same content: the
+// same action, the same bytes on disk, the same mode.
+func TestAFilledFileIsWrittenAsItsOwnershipSays(t *testing.T) {
+	for _, c := range []struct {
+		name, fields, content, before, action string
+	}{
+		{"owned, absent", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n", "", "created"},
+		{"owned, over what is there", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n",
+			"port = 1\n", "updated"},
+		{"seed, absent", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n", "", "created"},
+		{"seed, present", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n",
+			"user default on\nuser app-one on\n", "kept"},
+		{"managed json", `"into":"json","content":"{}","owner-at-runtime":"managed"`,
+			`{"insecure-registries":["10.42.0.1:5000"]}`, machinesOwn, "updated"},
+		{"managed block", `"into":"block","content":"x\n","owner-at-runtime":"managed"`, "10.42.0.1 ace\n",
+			workstationHosts, "updated"},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			at := func() string {
+				path := filepath.Join(t.TempDir(), "the-file")
+				if c.before != "" {
+					if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil {
+						t.Fatal(err)
+					}
+				}
+				return path
+			}
+			written := at()
+			report, state, err := WriteFilled(context.Background(), archHost(t), filledDecl(t, written, c.fields, ""),
+				store.State{}, written, []byte(c.content), noServices, nil, nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			applied := at()
+			withContent := strings.Replace(c.fields, fmt.Sprintf(`"content":%q`, contentOf(t, c.fields)),
+				fmt.Sprintf(`"content":%q`, c.content), 1)
+			// The same file declared the old way: an apply writes a filled file only on the client's word (P7).
+			plain := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.conf","type":"file",`+
+				`"path":%q,%s}]}`, applied, withContent))
+			want, _, err := Apply(context.Background(), archHost(t), plain, store.State{}, store.OriginDeclared,
+				noServices, nil, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := report.Outcomes[0].Action; got != c.action || got != want.Outcomes[0].Action {
+				t.Errorf("written %s; an apply of the same content %s; want %s", got, want.Outcomes[0].Action, c.action)
+			}
+			if got, apply := readText(t, written), readText(t, applied); got != apply {
+				t.Errorf("written, the file holds\n%q\nan apply of the same content leaves\n%q", got, apply)
+			}
+			if a, b := modeAt(t, written), modeAt(t, applied); a != b {
+				t.Errorf("written mode %o, an apply's %o", a, b)
+			}
+			if rec, ok := state.Find("m.conf"); !ok || rec.Target != written || rec.Origin != store.OriginDeclared {
+				t.Errorf("the write is not recorded as the file's: %+v", rec)
+			}
+		})
+	}
+}
+
+func contentOf(t *testing.T, fields string) string {
+	t.Helper()
+	var holder struct {
+		Content string `json:"content"`
+	}
+	if err := json.Unmarshal([]byte("{"+fields+"}"), &holder); err != nil {
+		t.Fatal(err)
+	}
+	return holder.Content
+}
+
+func modeAt(t *testing.T, path string) os.FileMode {
+	t.Helper()
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return info.Mode().Perm()
+}
+
+// A file that carried a secret keeps the mode a secret is written with: the client filled the value, and the file
+// is no more readable for it.
+func TestAFilledFileThatCarriedASecretIsNotWorldReadable(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "token.env")
+	d := filledDecl(t, path, `"content":"TOKEN=${secret:token}\n","secrets":{"token":"c2VhbGVk"},"owner-at-runtime":"owned"`, "")
+	if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("TOKEN=abc\n"),
+		noServices, nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if m := modeAt(t, path); m != 0o600 {
+		t.Errorf("a filled file that carried a secret is mode %o", m)
+	}
+}
+
+func TestAWriteUnderReloadOnReloadsItsProgramOnce(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.conf")
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`,
+		`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","reload-on":["m.conf"]},
+		 {"id":"m.other","type":"service","unit":"other.service","state":"running"}`)
+	var commands []string
+	var said []string
+	_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"),
+		recordingServices(&commands), func(line string) { said = append(said, line) }, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n := count(commands, "systemctl reload m.service"); n != 1 {
+		t.Errorf("the program was reloaded %d times; commands were %v", n, commands)
+	}
+	joined := strings.Join(commands, "\n")
+	if strings.Contains(joined, "restart m.service") || strings.Contains(joined, "stop m.service") {
+		t.Errorf("the program was restarted, not reloaded: %v", commands)
+	}
+	if strings.Contains(joined, "other.service") {
+		t.Errorf("a service that does not read the file was touched: %v", commands)
+	}
+	if _, ok := state.Find("m.daemon"); !ok {
+		t.Errorf("the reloaded service was not recorded")
+	}
+	// The log says the path and the digest, never the content.
+	for _, line := range said {
+		if strings.Contains(line, "level = 1") {
+			t.Errorf("the content was said: %q", line)
+		}
+	}
+	if !strings.Contains(strings.Join(said, "\n"), digestOf("level = 1\n")) {
+		t.Errorf("the digest was not said: %q", said)
+	}
+
+	// The same content again moves nothing, and reloads nothing.
+	commands = nil
+	report, _, err := WriteFilled(context.Background(), archHost(t), d, state, path, []byte("level = 1\n"),
+		recordingServices(&commands), nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.Outcomes[0].Action != "unchanged" || count(commands, "reload") != 0 {
+		t.Errorf("an unchanged write reloaded: %v %+v", commands, report.Outcomes)
+	}
+}
+
+func TestAWriteUnderRestartOnRestartsItsProgram(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "daemon.conf")
+	d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`,
+		`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`)
+	var commands []string
+	if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"),
+		recordingServices(&commands), nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	// A restart is a stop and a start, once.
+	stops, starts := count(commands, "systemctl stop m.service"), count(commands, "systemctl start m.service")
+	if stops != 1 || starts != 1 {
+		t.Errorf("the program was stopped %d and started %d times; commands were %v", stops, starts, commands)
+	}
+}
+
+// A running container that reads the file at creation is recreated, as an apply recreates it for a changed file.
+func TestAWriteOfAFileAContainerReadsRecreatesIt(t *testing.T) {
+	env := filepath.Join(t.TempDir(), "forge.env")
+	d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`,
+		`{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`)
+	m := &machine{containers: map[string]*fakeContainer{}}
+	_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"),
+		m.run, nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !m.did("docker run") {
+		t.Fatalf("the container was not created: %v", m.asked)
+	}
+	m.asked = nil
+	report, _, err := WriteFilled(context.Background(), archHost(t), d, state, env, []byte("PORT=5433\n"), m.run,
+		nil, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !m.removed("forge") || !m.did("docker run") {
+		t.Fatalf("the container kept what it read before the write: %v", m.asked)
+	}
+	if o := outcomeOf(report, "forge.server"); o.Action != "updated" || !strings.Contains(o.Detail, env) {
+		t.Errorf("the recreate did not say which file changed: %+v", o)
+	}
+}
+
+func count(lines []string, prefix string) int {
+	n := 0
+	for _, l := range lines {
+		if strings.Contains(l, prefix) {
+			n++
+		}
+	}
+	return n
+}
+
+// Two resources declaring one filled path — two modules' blocks in one file — are refused: the request carries the
+// content only, so it cannot say whose block it is, and nothing is written.
+func TestAFilledPathTwoResourcesDeclareIsRefused(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "hosts")
+	if err := os.WriteFile(path, []byte(workstationHosts), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[
+	  {"id":"a.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.1 ace\n","owner-at-runtime":"managed"},
+	  {"id":"b.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.2 g14\n","owner-at-runtime":"managed"}
+	]}`, path, path, path))
+	_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("6.6.6.6 evil\n"),
+		noServices, nil, nil, nil)
+	if err == nil || !strings.Contains(err.Error(), "2 resources declare") {
+		t.Fatalf("a path two resources declare was not refused: %v", err)
+	}
+	if got := readText(t, path); got != workstationHosts || len(state.Resources) != 0 {
+		t.Errorf("something was written or recorded: %q %+v", got, state.Resources)
+	}
+}

internal/apply/grants.go

@@ -1,519 +0,0 @@
-package apply
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/exec"
-	"os/user"
-	"path/filepath"
-	"slices"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/novox/mesh-host/internal/accounts"
-	"github.com/novox/mesh-host/internal/declaration"
-	"github.com/novox/mesh-host/internal/store"
-	"github.com/novox/mesh-host/internal/system"
-)
-
-// A read-write access grants the path's own group (novox/hq ADR 0312, issue 537).
-//
-// **An access is not owning.** ADR 0051 left the operator's shared paths exactly as they are: the engine
-// confirms one is there and mounts it, and sets no owner, group or mode on it or below it. Nothing decided how
-// a `read-write` access then becomes writable, and a module whose identity was not the path's owner could not
-// write it at all — a download client could not remove a folder it had filed (issue 535).
-//
-// **The path's group is the operator's statement of who may share it.** So the engine reads, at apply, which
-// group owns the accessed path — only the machine knows — and makes the identities the controller names for
-// the access members of it: each container that mounts it gains the group as a supplementary group of its
-// process (`--group-add`, by number, because a container's own group database is not the machine's), and the
-// account that runs the module's tools is put in it as an account group (ADR 0252 §1). The module then writes
-// the path as its group may, and as nothing else.
-//
-// **The path is never changed** (§2). A group that cannot write the path is the operator's choice, as the data
-// is: the access grants nothing, is stated unhealthy on every look, saying so and that the operator decides, and
-// nothing is done to make it writable. **A read access grants nothing** (§3). **The root group is never granted** (§4): a
-// path owned by group 0, or by a group whose members act as root (accounts.RootGroups, the account judge's own
-// closed list, judged by the numbers those groups have here), or by a number the machine names nothing for, is
-// refused by name.
-//
-// **What was granted is taken back as ADR 0252 takes back** (§5): the access records each group it put the
-// account in that the account was not in before; when no declared resource asks for it any more, the account
-// leaves it, and a group the account held before is never recorded and never taken back. A container needs no
-// record: the group is part of what it is created as, so one no longer granted is recreated without it.
-//
-// **And what runs the code is restarted to hold it** (§6). A container is recreated with the group, because
-// its spec carries it. The tool runner — a process of the machine's own service manager with `User=` — is
-// restarted because the controller names the access under its `restart-on`, and the access is reported
-// updated exactly when the account's groups moved: that manager reads the account's groups from the database
-// each time it starts the unit, so the restarted process holds the group with no new login. A unit of the
-// account's own manager, or a login session, keeps the groups its login began with: the account judge
-// (internal/accounts) says *relogin needed* there, for the access's account as for a user resource's.
-
-// PathGroup is the group that owns an accessed path, as the engine reads it.
-type PathGroup struct {
-	GID int
-	// Name is the group's name in the machine's group database; empty when the database names nothing for
-	// the number.
-	Name string
-	// Writable is whether the group's write bit is set on the path itself.
-	Writable bool
-	// UID is the path's owner, and OwnerWritable whether the owner's write bit is set: a writer whose uid owns
-	// the path writes it as its owner (novox/hq ADR 0322 §2).
-	UID           int
-	OwnerWritable bool
-}
-
-// Words names the group for a person: its name and number, or its number alone.
-func (g PathGroup) Words() string {
-	if g.Name == "" {
-		return fmt.Sprintf("gid %d", g.GID)
-	}
-	return fmt.Sprintf("%s (gid %d)", g.Name, g.GID)
-}
-
-// GroupOfPath reads which group owns a path, by its number, its name and whether it may write. Reads only. A
-// variable so a test can stand in for a machine whose groups it cannot make.
-var GroupOfPath = func(path string) (PathGroup, error) {
-	info, err := os.Stat(path)
-	if err != nil {
-		return PathGroup{}, err
-	}
-	uid, gid, ok := statOwner(info)
-	if !ok {
-		return PathGroup{}, fmt.Errorf("the group that owns %s cannot be read on this machine", path)
-	}
-	g := PathGroup{GID: gid, Writable: info.Mode().Perm()&0o020 != 0, UID: uid,
-		OwnerWritable: info.Mode().Perm()&0o200 != 0}
-	if found, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil {
-		g.Name = found.Name
-	}
-	return g, nil
-}
-
-// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine
-// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine
-// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number
-// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and
-// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can
-// stand in.
-var RootGroupIDs = func() map[int]string {
-	rootIDs.Lock()
-	defer rootIDs.Unlock()
-	if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute {
-		rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now()
-	}
-	return rootIDs.ids
-}
-
-var rootIDs struct {
-	sync.Mutex
-	ids map[int]string
-	at  time.Time
-}
-
-func readRootGroupIDs() map[int]string {
-	out := map[int]string{}
-	if raw, err := os.ReadFile("/etc/group"); err == nil {
-		for name, gid := range groupEntries(string(raw)) {
-			if slices.Contains(accounts.RootGroups, name) {
-				out[gid] = name
-			}
-		}
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	for _, name := range accounts.RootGroups {
-		raw, err := exec.CommandContext(ctx, "getent", "group", name).Output()
-		if err != nil {
-			continue
-		}
-		for found, gid := range groupEntries(string(raw)) {
-			if found == name {
-				out[gid] = name
-			}
-		}
-	}
-	return out
-}
-
-// groupEntries reads group database lines, name:password:gid:members, as name to number.
-func groupEntries(raw string) map[string]int {
-	out := map[string]int{}
-	for _, line := range strings.Split(raw, "\n") {
-		fields := strings.Split(strings.TrimSpace(line), ":")
-		if len(fields) < 3 || strings.HasPrefix(fields[0], "#") {
-			continue
-		}
-		if gid, err := strconv.Atoi(fields[2]); err == nil {
-			out[fields[0]] = gid
-		}
-	}
-	return out
-}
-
-// GrantRefusal is why a path's group is never granted (novox/hq ADR 0312 §4), or nil when it may be: group 0, a
-// group whose members act as root by membership alone — judged by its NUMBER against every number those groups
-// have here, and by its name — or a group this machine names nothing for, which cannot be judged at all.
-func GrantRefusal(g PathGroup) error {
-	const never = "a shared path is never a way to root (novox/hq ADR 0312 §4); nothing was granted, and nothing " +
-		"was changed on it"
-	if g.GID == 0 {
-		return fmt.Errorf("it is owned by group 0 (%s), and a read-write access never grants the root group: %s",
-			orRoot(g.Name), never)
-	}
-	if root, is := RootGroupIDs()[g.GID]; is {
-		return fmt.Errorf("it is owned by gid %d, which is %s here, whose members act as root, and a read-write "+
-			"access never grants such a group: %s", g.GID, root, never)
-	}
-	if slices.Contains(accounts.RootGroups, g.Name) {
-		return fmt.Errorf("it is owned by the group %s, whose members act as root, and a read-write access never "+
-			"grants such a group: %s", g.Words(), never)
-	}
-	if g.Name == "" {
-		return fmt.Errorf("it is owned by gid %d, which this machine's group database names nothing, so whether "+
-			"its members act as root cannot be judged, and an unjudged group is never granted: %s", g.GID, never)
-	}
-	return nil
-}
-
-// CannotWriteWords is the reason an access whose group may not write its path is stated unhealthy with (novox/hq
-// ADR 0312 §2). Never the path: the controller says a reason on the operator's channel, which withholds a path
-// (ADR 0234 §6), and keeps the target as evidence.
-func CannotWriteWords(g PathGroup) string {
-	return fmt.Sprintf("its group cannot write it: %s owns it without group write, and the mesh changes nothing on "+
-		"an accessed path (novox/hq ADR 0312 §2) — whether its group may write is the operator's to decide",
-		g.Words())
-}
-
-func orRoot(name string) string {
-	if name == "" {
-		return "root"
-	}
-	return name
-}
-
-// grant is what one read-write access grants, read once per apply.
-type grant struct {
-	path  string
-	group PathGroup
-	// refused is why nothing is granted: the path could not be read, or its group is never granted.
-	refused error
-}
-
-// grants says the group is given: it may be granted, and it may write the path. A group that cannot write is
-// granted nothing (novox/hq ADR 0312 §2) — the access is stated unhealthy for the operator to decide.
-func (g grant) grants() bool { return g.refused == nil && g.group.Writable }
-
-// grantsOf reads the group of every read-write access that names a grant, by access id. One that is refused
-// grants nothing — the access itself fails, saying why — and one whose path the engine refuses to place
-// anything at (issue 339) is not read at all.
-func grantsOf(resources []declaration.Resource, refused map[string]error) map[string]grant {
-	out := map[string]grant{}
-	for _, r := range resources {
-		a, ok := r.(*declaration.Access)
-		if !ok || a.Grant == nil || a.Mode != declaration.AccessReadWrite {
-			continue
-		}
-		if why := refused[filepath.Clean(a.Path)]; why != nil {
-			out[a.ID] = grant{path: a.Path, refused: why}
-			continue
-		}
-		g, err := GroupOfPath(a.Path)
-		if err == nil {
-			err = GrantRefusal(g)
-		}
-		out[a.ID] = grant{path: a.Path, group: g, refused: err}
-	}
-	return out
-}
-
-// containerGroups is, for each container, the number of every group a read-write access grants it, sorted and
-// once each: what it is created with as supplementary groups.
-func containerGroups(resources []declaration.Resource, grants map[string]grant) map[string][]int {
-	out := map[string][]int{}
-	for _, r := range resources {
-		a, ok := r.(*declaration.Access)
-		if !ok || a.Grant == nil {
-			continue
-		}
-		g, granted := grants[a.ID]
-		if !granted || !g.grants() {
-			continue
-		}
-		for _, id := range a.Grant.Containers {
-			if !slices.Contains(out[id], g.group.GID) {
-				out[id] = append(out[id], g.group.GID)
-			}
-		}
-	}
-	for id := range out {
-		sort.Ints(out[id])
-	}
-	return out
-}
-
-// ContainerGroupsOf is containerGroups for a whole declaration, read now: what the scheduler arms a scheduled
-// step with, so a fire runs with the groups an apply would have created it with. A path issue 339's guard refuses
-// is not read, as the apply reads none.
-func ContainerGroupsOf(d *declaration.Declaration) map[string][]int {
-	if d == nil {
-		return nil
-	}
-	return containerGroups(d.Resources, grantsOf(d.Resources, refusedPlaces(d.Resources)))
-}
-
-// withGrants adds to what the declaration asks of each account the groups its read-write accesses grant (novox/hq
-// ADR 0312 §5): one user resource or access going leaves an account in a group another still asks for.
-func (w Wanted) withGrants(resources []declaration.Resource, grants map[string]grant) Wanted {
-	for _, r := range resources {
-		a, ok := r.(*declaration.Access)
-		if !ok || a.Grant == nil || a.Grant.Account == "" {
-			continue
-		}
-		g, granted := grants[a.ID]
-		if !granted || !g.grants() {
-			continue
-		}
-		if w[a.Grant.Account] == nil {
-			w[a.Grant.Account] = map[string][]string{}
-		}
-		w[a.Grant.Account][g.group.Name] = append(w[a.Grant.Account][g.group.Name], a.ID)
-	}
-	return w
-}
-
-// GrantedAccounts is every account a read-write access puts in its path's group, as the account judge reads
-// them (novox/hq ADR 0312 §6): judged like a user resource's account groups, so a session of the account's own
-// manager that began before is said *relogin needed*, and only there. Read now; a path whose group is refused
-// or cannot be read asks nothing of the judge — the access says that itself.
-func GrantedAccounts(d *declaration.Declaration, held map[string]bool) []accounts.Account {
-	if d == nil {
-		return nil
-	}
-	grants := grantsOf(d.Resources, refusedPlaces(d.Resources))
-	var out []accounts.Account
-	for _, r := range d.Resources {
-		a, ok := r.(*declaration.Access)
-		if !ok || a.Grant == nil || a.Grant.Account == "" || held[a.ID] {
-			continue
-		}
-		g, granted := grants[a.ID]
-		if !granted || !g.grants() {
-			continue
-		}
-		at := strings.LastIndex(a.ID, ".")
-		if at <= 0 || strings.HasPrefix(a.ID, declaration.AdoptionPrefix) {
-			continue
-		}
-		out = append(out, accounts.Account{Module: a.ID[:at], ID: a.ID, Name: a.Grant.Account,
-			Groups: []string{g.group.Name}})
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
-	return out
-}
-
-// grantAccess is applyAccess's grant (novox/hq ADR 0312): the account put in the path's group, what an earlier
-// apply of this access put an account in and nobody asks for any more taken back, and the outcome said. The
-// path itself is only read.
-func grantAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
-	in inputs, out *Outcome) error {
-	var g grant
-	granting := r.Grant != nil && r.Mode == declaration.AccessReadWrite
-	if granting {
-		var read bool
-		g, read = in.grants[r.ID]
-		if !read {
-			// An apply that read nothing ahead — a caller with no inputs — reads it here.
-			g = grantsOf([]declaration.Resource{r}, in.refused)[r.ID]
-		}
-		if g.refused != nil {
-			return fmt.Errorf("%s grants nothing: %w", r.Path, g.refused)
-		}
-	}
-	// A group that cannot write is granted nothing (§2): what an earlier apply granted is taken back below.
-	account := ""
-	if granting && g.group.Writable {
-		account = r.Grant.Account
-	}
-	if account != "" && g.group.Name == "" {
-		return fmt.Errorf("%s is owned by %s, which this machine's group database names nothing, and an account "+
-			"is put in a group by name: %s was not (novox/hq ADR 0312 §1)", r.Path, g.group.Words(), account)
-	}
-
-	var said []string
-	moved := false
-	// What this access put an account in before. An access whose account changed gives the old one back
-	// everything it put it in; the new one starts with nothing recorded.
-	recorded := previous.Groups
-	if previous.User != "" && previous.User != account {
-		gave, words := takeBack(ctx, sys, previous.User, r.ID, previous.Groups, nil, run, in.groups)
-		moved = moved || gave
-		said = append(said, words...)
-		recorded = nil
-	}
-	var kept []string
-	if account != "" {
-		groups, err := system.GroupsOf(ctx, system.Runner(run), account)
-		if err != nil {
-			return err
-		}
-		inGroup := setOf(groups)[g.group.Name]
-		if !inGroup {
-			if err := sys.AddUserToGroup(ctx, system.Runner(run), account, g.group.Name); err != nil {
-				return err
-			}
-			moved = true
-			said = append(said, fmt.Sprintf("put %s in %s, the group that owns it; the tool runner holds it once "+
-				"restarted, a session that began before only after a new login", account, g.group.Name))
-		}
-		// Recorded only when the mesh put it there, now or before, or a record that went handed it over: a group
-		// held before is never taken back.
-		if !inGroup || slices.Contains(recorded, g.group.Name) || in.handed.has(account, g.group.Name) {
-			kept = append(kept, g.group.Name)
-		}
-		var others []string
-		for _, x := range recorded {
-			if x != g.group.Name {
-				others = append(others, x)
-			}
-		}
-		gave, words := takeBack(ctx, sys, account, r.ID, others, &kept, run, in.groups)
-		moved = moved || gave
-		said = append(said, words...)
-	}
-	if len(kept) > 0 {
-		out.groups, out.user = kept, account
-	}
-
-	if granting {
-		var to []string
-		if len(r.Grant.Containers) > 0 {
-			to = append(to, "its containers "+strings.Join(r.Grant.Containers, ", "))
-		}
-		if account != "" {
-			to = append(to, account)
-		}
-		switch {
-		case !g.group.Writable:
-			out.Detail = joinDetail(out.Detail, fmt.Sprintf("its group %s cannot write it, so nothing is granted: "+
-				"stated unhealthy, and nothing was changed on it — the operator decides", g.group.Words()))
-		case len(to) > 0:
-			out.Detail = joinDetail(out.Detail, fmt.Sprintf("grants %s to %s", g.group.Words(), strings.Join(to, " and ")))
-		}
-	}
-	if len(said) > 0 {
-		out.Detail = joinDetail(out.Detail, strings.Join(said, "; "))
-	}
-	if moved {
-		// Updated exactly when the account's groups moved: what names this access under restart-on — the
-		// tool runner — is restarted to hold them (novox/hq ADR 0312 §6).
-		out.Action = "updated"
-	}
-	return nil
-}
-
-// takeBack takes account out of every group of groups the mesh put it in for the resource id, unless another
-// declared resource still asks for it — then it stays, and stays recorded on keep when keep is given, so it is
-// taken back once nobody asks. A group it is no longer in is nothing to give back. Never fatal: a failure is
-// said, and the group stays recorded where it can be. Whether any group was taken back, and what was said.
-func takeBack(ctx context.Context, sys system.System, account, id string, groups []string, keep *[]string,
-	run Runner, wanted Wanted) (bool, []string) {
-	if len(groups) == 0 {
-		return false, nil
-	}
-	in, err := system.GroupsOf(ctx, system.Runner(run), account)
-	if err != nil {
-		if keep != nil {
-			*keep = append(*keep, groups...)
-		}
-		return false, []string{fmt.Sprintf("the groups the mesh put %s in (%s) were not given back: %v", account,
-			strings.Join(groups, ", "), err)}
-	}
-	already := setOf(in)
-	gave := false
-	var said []string
-	for _, g := range groups {
-		if !already[g] {
-			continue
-		}
-		if others := wanted.othersAsk(account, g, id); len(others) > 0 {
-			said = append(said, fmt.Sprintf("left %s in %s, which %s still asks for", account, g, strings.Join(others, ", ")))
-			if keep != nil {
-				*keep = append(*keep, g)
-			}
-			continue
-		}
-		gone, why := leaveGroup(ctx, sys, account, g, run)
-		if !gone && keep != nil {
-			*keep = append(*keep, g)
-		}
-		gave = gave || gone
-		said = append(said, account+" "+why)
-	}
-	return gave, said
-}
-
-// removeAccess is an access no longer declared: the path left exactly as it is (novox/hq ADR 0051), and every
-// group it put an account in taken back unless another declared resource still asks for it (ADR 0312 §5) — then
-// handed to that resource, which records it as the mesh's and takes it back when it stops asking.
-func removeAccess(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted,
-	handed Handed) (string, string) {
-	const left = "an operator-owned path is never the host's to remove"
-	if a.User == "" || len(a.Groups) == 0 {
-		return "forgotten", left
-	}
-	var stays []string
-	gave, said := takeBack(ctx, sys, a.User, a.ID, a.Groups, &stays, run, wanted)
-	for _, g := range stays {
-		if len(wanted.othersAsk(a.User, g, a.ID)) > 0 {
-			handed.hand(a.User, g)
-		}
-	}
-	action := "forgotten"
-	if gave {
-		action = "restored"
-	}
-	return action, joinDetail(left, strings.Join(said, "; "))
-}
-
-// groupAddArgs is the runtime's arguments that give a container's process each granted group as a supplementary
-// group, by number: the container's own group database is not the machine's, and a name could mean another
-// group there, or none.
-func groupAddArgs(groups []int) []string {
-	var args []string
-	for _, gid := range groups {
-		args = append(args, "--group-add", strconv.Itoa(gid))
-	}
-	return args
-}
-
-func gidWords(groups []int) string {
-	words := make([]string, 0, len(groups))
-	for _, gid := range groups {
-		words = append(words, strconv.Itoa(gid))
-	}
-	return strings.Join(words, ", ")
-}
-
-// Handed is every group a record that went in this apply put an account in, and another declared resource still
-// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group
-// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it
-// there went first (novox/hq ADR 0252 §3, ADR 0312 §5).
-type Handed map[string]map[string]bool
-
-func (h Handed) hand(account, group string) {
-	if h == nil {
-		return
-	}
-	if h[account] == nil {
-		h[account] = map[string]bool{}
-	}
-	h[account][group] = true
-}
-
-func (h Handed) has(account, group string) bool { return h[account][group] }

internal/apply/grants_test.go

@@ -1,559 +0,0 @@
-package apply
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"os/user"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/novox/mesh-host/internal/accounts"
-	"github.com/novox/mesh-host/internal/declaration"
-	"github.com/novox/mesh-host/internal/store"
-)
-
-// Defends novox/hq ADR 0312 and issue 537: a read-write access grants its path's own group to the containers and
-// the account the controller names, and the path is never changed; a read access grants nothing; the root group,
-// and a group whose members act as root, are refused by name; what was granted is taken back as ADR 0252 takes
-// back; a container is recreated with the group, and a unit of the machine's own manager running as the account
-// is restarted once the account holds it.
-
-// grantMachine is the group database of groupDB, a container runtime that keeps each container's spec label and
-// the arguments it was run with, and a service manager that notes, at each restart, the groups the account holds
-// in the database then — which is what the machine's own manager gives the unit's new process.
-type grantMachine struct {
-	*groupDB
-	containers map[string]string
-	runs       map[string][]string
-	restarts   []string
-}
-
-func newGrantMachine(account string, in []string, exists ...string) *grantMachine {
-	g := newGroupDB(in, exists...)
-	g.account = account
-	return &grantMachine{groupDB: g, containers: map[string]string{}, runs: map[string][]string{}}
-}
-
-func (m *grantMachine) run(ctx context.Context, name string, args ...string) (string, error) {
-	a := oldShape(name, args)
-	switch name {
-	case "docker":
-		m.asked = append(m.asked, name+" "+strings.Join(a, " "))
-		switch a[0] {
-		case "info":
-			return "27.0\n", nil
-		case "container":
-			spec, there := m.containers[a[len(a)-1]]
-			if !there {
-				return "", errors.New("no such container")
-			}
-			return "true\t" + spec, nil
-		case "rm":
-			delete(m.containers, a[len(a)-1])
-		case "run":
-			var named, spec string
-			for i := 0; i+1 < len(a); i++ {
-				switch {
-				case a[i] == "--name":
-					named = a[i+1]
-				case a[i] == "--label" && strings.HasPrefix(a[i+1], specLabel+"="):
-					spec = strings.TrimPrefix(a[i+1], specLabel+"=")
-				}
-			}
-			m.containers[named] = spec
-			m.runs[named] = a
-			return "deadbeef\n", nil
-		}
-		return "", nil
-	case "systemctl":
-		m.asked = append(m.asked, name+" "+strings.Join(a, " "))
-		if a[0] == "restart" {
-			m.restarts = append(m.restarts, a[1]+": "+m.groups())
-		}
-		return "", nil
-	}
-	return m.groupDB.run(ctx, name, args...)
-}
-
-// groupOfPaths stands in for the machine's group database and the paths' groups for the length of a test.
-func groupOfPaths(t *testing.T, groups map[string]PathGroup) {
-	t.Helper()
-	was := GroupOfPath
-	GroupOfPath = func(path string) (PathGroup, error) {
-		if _, err := os.Stat(path); err != nil {
-			return PathGroup{}, err
-		}
-		g, ok := groups[path]
-		if !ok {
-			return PathGroup{}, errors.New("no group in this test for " + path)
-		}
-		return g, nil
-	}
-	t.Cleanup(func() { GroupOfPath = was })
-	rootGroupsAre(t, map[int]string{0: "root", 10: "wheel", 968: "docker"})
-}
-
-// rootGroupsAre stands in for the numbers this machine gives the groups whose members act as root.
-func rootGroupsAre(t *testing.T, ids map[int]string) {
-	t.Helper()
-	was := RootGroupIDs
-	RootGroupIDs = func() map[int]string { return ids }
-	t.Cleanup(func() { RootGroupIDs = was })
-}
-
-func accessGranting(id, path, account string, containers ...string) string {
-	grant := `"grant":{"account":"` + account + `"`
-	if len(containers) > 0 {
-		grant += `,"containers":["` + strings.Join(containers, `","`) + `"]`
-	}
-	return `{"id":"` + id + `","type":"access","path":"` + path + `","mode":"read-write",` + grant + `}}`
-}
-
-func mounting(id, path string) string {
-	return `{"id":"` + id + `","type":"container","name":"` + strings.ReplaceAll(id, ".", "-") + `","image":"` + pinned +
-		`","volumes":["` + path + `:/library"]}`
-}
-
-func applyGrants(t *testing.T, m *grantMachine, known store.State, resources ...string) (Report, store.State, error) {
-	t.Helper()
-	return Apply(context.Background(), archHost(t), parse(t, `{"declaration":1,"resources":[`+
-		strings.Join(resources, ",")+`]}`), known, store.OriginDeclared, m.run, nil, nil)
-}
-
-func statOf(t *testing.T, path string) string {
-	t.Helper()
-	info, err := os.Stat(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	uid, gid, _ := OwnerOf(info)
-	return fmt.Sprintf("%s %d:%d %s", info.Mode(), uid, gid, info.ModTime())
-}
-
-func TestAReadWriteAccessGrantsItsPathsGroupToTheContainerAndTheAccountAndLeavesThePath(t *testing.T) {
-	library := t.TempDir()
-	if err := os.WriteFile(filepath.Join(library, "album"), []byte("x"), 0o640); err != nil {
-		t.Fatal(err)
-	}
-	groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
-	before, beforeInside := statOf(t, library), statOf(t, filepath.Join(library, "album"))
-	m := newGrantMachine("operator", []string{"operator"}, "media")
-
-	report, state, err := applyGrants(t, m, store.State{},
-		accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !strings.Contains(" "+m.groups()+" ", " media ") {
-		t.Fatalf("the account was not put in the path's group: %q", m.groups())
-	}
-	if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") {
-		t.Fatalf("the container was not created with the path's group: %s", run)
-	}
-	o := outcomeOf(report, "importer.music")
-	if o.Action != "updated" || !strings.Contains(o.Detail, "grants media (gid 1001)") ||
-		!strings.Contains(o.Detail, "put operator in media") {
-		t.Errorf("the access did not say what it granted: %+v", o)
-	}
-	if rec, _ := state.Find("importer.music"); rec.User != "operator" || strings.Join(rec.Groups, " ") != "media" {
-		t.Errorf("the access did not record the group it put the account in: %+v", rec)
-	}
-	// The path, and what is under it, exactly as they were (ADR 0312 §2).
-	if after := statOf(t, library); after != before {
-		t.Errorf("the accessed path changed: %s, was %s", after, before)
-	}
-	if after := statOf(t, filepath.Join(library, "album")); after != beforeInside {
-		t.Errorf("a file under the accessed path changed: %s, was %s", after, beforeInside)
-	}
-	for _, asked := range m.asked {
-		for _, verb := range []string{"chown", "chmod", "chgrp", "setfacl"} {
-			if strings.HasPrefix(asked, verb) || strings.Contains(asked, " "+verb+" ") {
-				t.Errorf("the engine asked the machine to change something: %q", asked)
-			}
-		}
-	}
-
-	// Applied again: nothing moves, and nothing is recreated.
-	m.runs = map[string][]string{}
-	report, _, err = applyGrants(t, m, state,
-		accessGranting("importer.music", library, "operator", "importer.app"), mounting("importer.app", library))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if o := outcomeOf(report, "importer.music"); o.Action != "unchanged" {
-		t.Errorf("a second apply moved the access: %+v", o)
-	}
-	if len(m.runs) != 0 {
-		t.Errorf("a second apply recreated the container: %v", m.runs)
-	}
-}
-
-func TestAReadAccessGrantsNothing(t *testing.T) {
-	library := t.TempDir()
-	groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
-
-	// A grant on a read access is refused whole: the controller asked for something the decision forbids.
-	_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"player.music","type":"access","path":"` +
-		library + `","mode":"read","grant":{"account":"operator"}}]}`))
-	if err == nil || !strings.Contains(err.Error(), "a read access grants nothing") {
-		t.Fatalf("a grant on a read access was not refused by name: %v", err)
-	}
-
-	m := newGrantMachine("operator", []string{"operator"}, "media")
-	report, state, err := applyGrants(t, m, store.State{},
-		`{"id":"player.music","type":"access","path":"`+library+`","mode":"read"}`, mounting("player.app", library))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if strings.Contains(m.groups(), "media") {
-		t.Errorf("a read access put the account in a group: %q", m.groups())
-	}
-	if run := strings.Join(m.runs["player-app"], " "); strings.Contains(run, "--group-add") {
-		t.Errorf("a read access gave its container a group: %s", run)
-	}
-	if o := outcomeOf(report, "player.music"); o.Action != "unchanged" {
-		t.Errorf("a read access moved something: %+v", o)
-	}
-	if rec, _ := state.Find("player.music"); len(rec.Groups) != 0 || rec.User != "" {
-		t.Errorf("a read access recorded a grant: %+v", rec)
-	}
-}
-
-func TestAnAccessWhoseGroupCannotWriteGrantsNothingAndSaysTheOperatorDecides(t *testing.T) {
-	spool := t.TempDir()
-	groupOfPaths(t, map[string]PathGroup{spool: {GID: 1002, Name: "downloads", Writable: false}})
-	before := statOf(t, spool)
-	m := newGrantMachine("operator", nil, "downloads")
-
-	report, _, err := applyGrants(t, m, store.State{},
-		accessGranting("nzbget.downloads", spool, "operator", "nzbget.app"), mounting("nzbget.app", spool))
-	if err != nil {
-		t.Fatal(err)
-	}
-	o := outcomeOf(report, "nzbget.downloads")
-	if !strings.Contains(o.Detail, "cannot write it, so nothing is granted") || !strings.Contains(o.Detail, "the operator decides") ||
-		!strings.Contains(o.Detail, "nothing was changed") {
-		t.Errorf("the access did not say its group cannot write and that the operator decides: %+v", o)
-	}
-	// Nothing is given (ADR 0312 §2): no account group, no container group.
-	if strings.Contains(m.groups(), "downloads") {
-		t.Errorf("a group that cannot write was granted to the account: %q", m.groups())
-	}
-	if run := strings.Join(m.runs["nzbget-app"], " "); strings.Contains(run, "--group-add") {
-		t.Errorf("a group that cannot write was granted to the container: %s", run)
-	}
-	if after := statOf(t, spool); after != before {
-		t.Errorf("the engine changed a path whose group cannot write: %s, was %s", after, before)
-	}
-	// The words the look states it unhealthy with: the remedy is the operator's, and the path is never in them.
-	words := CannotWriteWords(PathGroup{GID: 1002, Name: "downloads"})
-	if !strings.Contains(words, "the operator's to decide") || strings.Contains(words, spool) {
-		t.Errorf("the unhealthy reason is %q", words)
-	}
-}
-
-func TestTheRootGroupIsNeverGranted(t *testing.T) {
-	for _, g := range []PathGroup{{GID: 0, Name: "root", Writable: true}, {GID: 968, Name: "docker", Writable: true},
-		{GID: 10, Name: "wheel", Writable: true}, {GID: 42, Name: "shadow", Writable: true},
-		// A number the root groups have here under another name — judged by the number, never the name.
-		{GID: 968, Name: "containers", Writable: true},
-		// A number the machine names nothing for: unjudged, so never granted, to a container either.
-		{GID: 4242, Name: "", Writable: true}} {
-		want := g.Name
-		if want == "" || want == "containers" {
-			want = fmt.Sprintf("gid %d", g.GID)
-		}
-		t.Run(fmt.Sprintf("%d %s", g.GID, g.Name), func(t *testing.T) {
-			shared := t.TempDir()
-			groupOfPaths(t, map[string]PathGroup{shared: g})
-			m := newGrantMachine("operator", nil, g.Name)
-
-			report, _, err := applyGrants(t, m, store.State{},
-				accessGranting("importer.shared", shared, "operator", "importer.app"), mounting("importer.app", shared))
-			if err == nil {
-				t.Fatal("a read-write access to a path of a root-granting group applied")
-			}
-			var failed *Error
-			if !errors.As(err, &failed) || failed.Resource != "importer.shared" ||
-				!strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "never") {
-				t.Errorf("the refusal does not name the access and the group: %v", err)
-			}
-			if g.Name != "" && strings.Contains(m.groups(), g.Name) {
-				t.Errorf("the account was put in %s", g.Name)
-			}
-			if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") {
-				t.Errorf("the container was given a root-granting group: %s", run)
-			}
-			_ = report
-		})
-	}
-}
-
-func TestAGrantedGroupIsTakenBackUnlessStillAskedForOrHeldBefore(t *testing.T) {
-	music, movies := t.TempDir(), t.TempDir()
-	groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true},
-		movies: {GID: 1001, Name: "media", Writable: true}})
-
-	t.Run("no longer asked", func(t *testing.T) {
-		m := newGrantMachine("operator", []string{"operator"}, "media")
-		_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"))
-		if err != nil {
-			t.Fatal(err)
-		}
-		report, state, err := applyGrants(t, m, state, mounting("other.app", t.TempDir()))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if strings.Contains(m.groups(), "media") {
-			t.Errorf("the group the mesh granted was not taken back: %q", m.groups())
-		}
-		if o := outcomeOf(report, "importer.music"); o.Action != "restored" || !strings.Contains(o.Detail, "taken out of media") {
-			t.Errorf("the removal did not say it took the group back: %+v", o)
-		}
-		if _, still := state.Find("importer.music"); still {
-			t.Error("the access is still recorded")
-		}
-	})
-
-	t.Run("still asked by another access", func(t *testing.T) {
-		m := newGrantMachine("operator", []string{"operator"}, "media")
-		_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"),
-			accessGranting("sonarr.movies", movies, "operator"))
-		if err != nil {
-			t.Fatal(err)
-		}
-		report, _, err := applyGrants(t, m, state, accessGranting("sonarr.movies", movies, "operator"))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !strings.Contains(m.groups(), "media") {
-			t.Errorf("a group another access still grants was taken back: %q", m.groups())
-		}
-		if o := outcomeOf(report, "importer.music"); !strings.Contains(o.Detail, "still asks for") {
-			t.Errorf("the removal did not say why the group stayed: %+v", o)
-		}
-	})
-
-	t.Run("still asked by a user resource", func(t *testing.T) {
-		m := newGrantMachine("operator", []string{"operator"}, "media")
-		_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if _, _, err := applyGrants(t, m, state,
-			`{"id":"player.account","type":"user","name":"operator","groups":["media"]}`); err != nil {
-			t.Fatal(err)
-		}
-		if !strings.Contains(m.groups(), "media") {
-			t.Errorf("a group a user resource asks for was taken back: %q", m.groups())
-		}
-	})
-
-	t.Run("held before", func(t *testing.T) {
-		m := newGrantMachine("operator", []string{"operator", "media"})
-		_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"))
-		if err != nil {
-			t.Fatal(err)
-		}
-		if rec, _ := state.Find("importer.music"); len(rec.Groups) != 0 {
-			t.Fatalf("a group the account held before was recorded as the mesh's: %+v", rec)
-		}
-		if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil {
-			t.Fatal(err)
-		}
-		if !strings.Contains(m.groups(), "media") {
-			t.Errorf("a group the account held before the mesh was taken back: %q", m.groups())
-		}
-	})
-}
-
-func TestAContainerIsRecreatedWithTheGroupItIsGranted(t *testing.T) {
-	library := t.TempDir()
-	groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
-	m := newGrantMachine("operator", nil, "media")
-
-	// Running before its access granted anything: a controller older than the grant.
-	_, state, err := applyGrants(t, m, store.State{},
-		`{"id":"importer.music","type":"access","path":"`+library+`","mode":"read-write"}`, mounting("importer.app", library))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if run := strings.Join(m.runs["importer-app"], " "); strings.Contains(run, "--group-add") {
-		t.Fatalf("a container granted nothing was given a group: %s", run)
-	}
-	report, _, err := applyGrants(t, m, state,
-		accessGranting("importer.music", library, "", "importer.app"), mounting("importer.app", library))
-	if err != nil {
-		t.Fatal(err)
-	}
-	o := outcomeOf(report, "importer.app")
-	if o.Action != "updated" || !strings.Contains(o.Detail, "recreated to hold the groups") {
-		t.Errorf("the container was not recreated for its group: %+v", o)
-	}
-	if run := strings.Join(m.runs["importer-app"], " "); !strings.Contains(run, "--group-add 1001") {
-		t.Errorf("the recreated container does not hold the group: %s", run)
-	}
-}
-
-func TestTheToolRunnerIsRestartedOnceItsAccountHoldsTheGroup(t *testing.T) {
-	me, err := user.Current()
-	if err != nil {
-		t.Skip("no current account to run the tool runner as")
-	}
-	units, bundles := t.TempDir(), t.TempDir()
-	wasUnits, wasBundles := unitDir, daemonRoot
-	unitDir, daemonRoot = units, bundles
-	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
-	library := t.TempDir()
-	groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
-	m := newGrantMachine(me.Username, nil, "media")
-
-	body, digest := anArchive(t, map[string]string{"main.js": "console.log(1)\n"})
-	runner := `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
-		`","digest":"` + digest + `","run":["node","main.js"],"user":"` + me.Username +
-		`","restart-on":["importer.music"]}`
-	// Running before the grant.
-	_, state, err := applyGrants(t, m, store.State{}, runner)
-	if err != nil {
-		t.Fatal(err)
-	}
-	m.restarts = nil
-	report, state, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(m.restarts) != 1 || !strings.HasPrefix(m.restarts[0], "node-tools.service: ") ||
-		!strings.Contains(m.restarts[0], "media") {
-		t.Fatalf("the tool runner was not restarted once its account held the group: %v", m.restarts)
-	}
-	if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "importer.music") {
-		t.Errorf("the restart does not say what it was for: %+v", o)
-	}
-	// And not again while nothing moves.
-	m.restarts = nil
-	if _, _, err := applyGrants(t, m, state, accessGranting("importer.music", library, me.Username), runner); err != nil {
-		t.Fatal(err)
-	}
-	if len(m.restarts) != 0 {
-		t.Errorf("the tool runner was restarted with nothing moved: %v", m.restarts)
-	}
-}
-
-// sessionReader is an account whose own service manager runs, holding the groups it began with.
-type sessionReader struct{ database, session []string }
-
-func (s sessionReader) InDatabase(context.Context, string) ([]string, error) { return s.database, nil }
-func (s sessionReader) Session(_ context.Context, _ string, groups []string) (accounts.Session, error) {
-	has := map[string]bool{}
-	for _, g := range s.session {
-		has[g] = true
-	}
-	return accounts.Session{Running: true, Has: has}, nil
-}
-func (s sessionReader) Escalation(context.Context, string, []string) ([]string, error) {
-	return nil, nil
-}
-
-func TestAnAccountsOwnSessionIsSaidReloginNeededForAGrantedGroup(t *testing.T) {
-	library := t.TempDir()
-	groupOfPaths(t, map[string]PathGroup{library: {GID: 1001, Name: "media", Writable: true}})
-	d := parse(t, `{"declaration":1,"resources":[`+accessGranting("importer.music", library, "operator")+`]}`)
-
-	as := GrantedAccounts(d, nil)
-	if len(as) != 1 || as[0].Module != "importer" || as[0].Name != "operator" || strings.Join(as[0].Groups, " ") != "media" {
-		t.Fatalf("the account the access grants is not judged: %+v", as)
-	}
-	j := accounts.New(sessionReader{database: []string{"operator", "media"}, session: []string{"operator"}})
-	j.Set(as)
-	st, _ := j.Look(context.Background())
-	if len(st.Accounts) != 1 || !strings.HasPrefix(st.Accounts[0].Reason, accounts.ReasonRelogin) {
-		t.Errorf("a session of the account's own manager that began before the grant is not said relogin needed: %+v",
-			st.Accounts)
-	}
-	// A read access, or one refused, asks nothing of the judge.
-	groupOfPaths(t, map[string]PathGroup{library: {GID: 0, Name: "root", Writable: true}})
-	if as := GrantedAccounts(d, nil); len(as) != 0 {
-		t.Errorf("a refused grant is judged: %+v", as)
-	}
-}
-
-// An access going while a user resource still asks for its group hands the group over: the user resource records it
-// as the mesh's, and takes it back when it stops asking too (novox/hq ADR 0312 §5, ADR 0252 §3).
-func TestAGroupStillAskedForIsHandedOverAndTakenBackWhenTheLastAskerGoes(t *testing.T) {
-	music := t.TempDir()
-	groupOfPaths(t, map[string]PathGroup{music: {GID: 1001, Name: "media", Writable: true}})
-	player := `{"id":"player.account","type":"user","name":"operator","groups":["media"]}`
-	m := newGrantMachine("operator", []string{"operator"}, "media")
-
-	_, state, err := applyGrants(t, m, store.State{}, accessGranting("importer.music", music, "operator"), player)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if rec, _ := state.Find("player.account"); len(rec.Groups) != 0 {
-		t.Fatalf("the user resource recorded a group the access put the account in: %+v", rec)
-	}
-	_, state, err = applyGrants(t, m, state, player)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !strings.Contains(m.groups(), "media") {
-		t.Fatalf("a group the user resource still asks for was taken back: %q", m.groups())
-	}
-	if rec, _ := state.Find("player.account"); strings.Join(rec.Groups, " ") != "media" {
-		t.Fatalf("the group was not handed to the user resource that still asks for it: %+v", rec)
-	}
-	if _, _, err := applyGrants(t, m, state, mounting("other.app", t.TempDir())); err != nil {
-		t.Fatal(err)
-	}
-	if strings.Contains(m.groups(), "media") {
-		t.Errorf("the group stayed after the last resource asking for it went: %q", m.groups())
-	}
-}
-
-// A grant's account is an argument to id, usermod and gpasswd, and a name that starts with a dash would be read as
-// an option by each.
-func TestAGrantsAccountMayNotReadAsAnOption(t *testing.T) {
-	for _, name := range []string{"-ogroups", "--help", "a b", "a:b"} {
-		_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` +
-			`"mode":"read-write","grant":{"account":"` + name + `"}}]}`))
-		if err == nil || !strings.Contains(err.Error(), "not starting with a dash") {
-			t.Errorf("the account %q was accepted: %v", name, err)
-		}
-	}
-	if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"m.lib","type":"access","path":"/srv/lib",` +
-		`"mode":"read-write","grant":{"account":"jochen"}}]}`)); err != nil {
-		t.Errorf("an ordinary account was refused: %v", err)
-	}
-}
-
-// The scheduler reads no path issue 339's guard refuses, as the apply reads none.
-func TestAScheduledStepIsArmedWithoutReadingARefusedPath(t *testing.T) {
-	var read []string
-	was := GroupOfPath
-	GroupOfPath = func(path string) (PathGroup, error) {
-		read = append(read, path)
-		return PathGroup{GID: 1001, Name: "media", Writable: true}, nil
-	}
-	t.Cleanup(func() { GroupOfPath = was })
-	rootGroupsAre(t, map[int]string{0: "root"})
-	d := parse(t, `{"declaration":1,"resources":[`+accessGranting("m.etc", "/etc", "", "m.step")+`,`+
-		`{"id":"m.step","type":"container","name":"step","image":"`+pinned+`","schedule":"0 3 * * *","volumes":["/etc:/x"]}]}`)
-	if got := ContainerGroupsOf(d); len(got["m.step"]) != 0 {
-		t.Errorf("a step was armed with the group of a refused path: %v", got)
-	}
-	if len(read) != 0 {
-		t.Errorf("the scheduler read a path the guard refuses: %v", read)
-	}
-}
-
-// The group database is read as name to number, so a number carrying a root group's name under any other name
-// is still that number.
-func TestGroupEntriesAreReadByNumber(t *testing.T) {
-	got := groupEntries("root:x:0:\n# a comment\ndocker:x:968:operator\ncontainers:x:968:\nbroken\n")
-	if got["root"] != 0 || got["docker"] != 968 || got["containers"] != 968 || len(got) != 3 {
-		t.Errorf("read %v", got)
-	}
-}

internal/apply/groups.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 	"github.com/novox/mesh-host/internal/system"
 )

internal/apply/health_check_test.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/hold.go

@@ -13,7 +13,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 	"github.com/novox/mesh-host/internal/system"
 )

internal/apply/hold_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/homes_linux.go

@@ -137,7 +137,7 @@ func chmodUnderAs(home, path string, mode os.FileMode, want int) error {
 		return err
 	}
 	defer unix.Close(fd)
-	if err := unix.Fchmod(fd, uint32(mode.Perm())); err != nil {
+	if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
 		return &os.PathError{Op: "chmod", Path: path, Err: err}
 	}
 	return nil
@@ -256,8 +256,8 @@ func giveHandedOver(path string, mode os.FileMode, uid, gid int, chown bool) (be
 	if err := unix.Fstat(fd, &st); err != nil {
 		return 0, 0, 0, &os.PathError{Op: "fstat", Path: path, Err: err}
 	}
-	before, beforeUID, beforeGID = os.FileMode(st.Mode&0o777), int(st.Uid), int(st.Gid)
-	if err := unix.Fchmod(fd, uint32(mode.Perm())); err != nil {
+	before, beforeUID, beforeGID = statDirMode(st.Mode), int(st.Uid), int(st.Gid)
+	if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
 		return 0, 0, 0, &os.PathError{Op: "chmod", Path: path, Err: err}
 	}
 	if chown && (beforeUID != uid || beforeGID != gid) {

internal/apply/homes_other.go

@@ -9,7 +9,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 func chmodUnder(home, path string, mode os.FileMode) error {

internal/apply/identities.go

@@ -0,0 +1,382 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// Every module is given an identity of its own (novox/hq ADR 0328 §3, §4, §15, issue 567).
+//
+// The controller allocates each module a user and a group number once, from a range of its own, the same on every
+// machine, and declares them as `mesh-<module>`. The engine makes the group and the account with exactly those
+// numbers — a system account: no login shell, no home, no password — before anything of the declaration is
+// applied, so a directory the module owns can be given to it and a container told to run as it.
+//
+// **Never root** (§11): a number that is 0, or a gid a root group has here (by its number and by its name), is
+// refused for the whole declaration before anything is made or started, naming why — for an account and for an
+// image alike. **Never another's**: a name that exists with other numbers, or numbers another name holds, is
+// refused for that identity alone, naming whose, and its module is not attempted; the rest of the machine is.
+//
+// **What it made, and only that, goes** when the identity is no longer declared: the record says whether the
+// engine made the account, the group or both, and an account or a group that was there before is never removed.
+
+// accountsTool is how the engine reads and changes the machine's accounts: the system's own tools through the
+// runner, which is what a test stands in for.
+type accountsTool interface {
+	CreateSystemAccount(ctx context.Context, run system.Runner, name string, uid, gid int, groups []string) error
+	ModifyAccount(ctx context.Context, run system.Runner, name string, gid int, groups []string) error
+	CreateSystemGroup(ctx context.Context, run system.Runner, name string, gid int) error
+	RemoveAccount(ctx context.Context, run system.Runner, name string) error
+	RemoveGroup(ctx context.Context, run system.Runner, name string) error
+}
+
+// entry is one line the user or group database gives for a key: its name and its numbers.
+type entry struct {
+	name     string
+	id, prim int
+}
+
+// lookUp asks the database (passwd or group) for key, a name or a number. Absent is (entry{}, false, nil); a
+// database that does not answer is an error, never read as absent (system.LookUpUser's rule).
+func lookUp(ctx context.Context, run Runner, database, key string) (entry, bool, error) {
+	out, err := run(ctx, "getent", database, key)
+	if err != nil {
+		if code, ok := system.ExitCode(err); ok && code == 2 {
+			return entry{}, false, nil
+		}
+		return entry{}, false, fmt.Errorf("the %s database did not answer about %s: %w", database, key, err)
+	}
+	fields := strings.Split(strings.TrimSpace(strings.SplitN(out, "\n", 2)[0]), ":")
+	if len(fields) < 3 {
+		return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
+	}
+	e := entry{name: fields[0], prim: -1}
+	if e.id, err = strconv.Atoi(fields[2]); err != nil {
+		return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
+	}
+	if database == "passwd" && len(fields) >= 4 {
+		if e.prim, err = strconv.Atoi(fields[3]); err != nil {
+			e.prim = -1
+		}
+	}
+	return e, true, nil
+}
+
+// refuseRootIdentities is every identity and every container's run-as that is root's (§11), judged before
+// anything of the declaration is applied: one is enough to refuse it whole, so nothing is made or started.
+func refuseRootIdentities(d *declaration.Declaration) error {
+	var problems []string
+	for _, id := range d.Identities {
+		// Its user's number, its own group's (the same number), its primary group and every added group.
+		if err := rootIdentity(id.UID, id.UID); err != nil {
+			problems = append(problems, fmt.Sprintf("the identity %s: %v", id.Name, err))
+		}
+		for _, gid := range append([]int{id.GID}, id.Groups...) {
+			if why := rootGroup(gid); why != "" {
+				problems = append(problems, fmt.Sprintf("the identity %s is given gid %s, and a module's identity "+
+					"is never in a root group (novox/hq ADR 0328 §15)", id.Name, why))
+			}
+		}
+	}
+	for _, g := range d.Groups {
+		if why := rootGroup(g.GID); why != "" {
+			problems = append(problems, fmt.Sprintf("the library group %s is gid %s, and a library's group is never a "+
+				"root group (novox/hq ADR 0328 §15)", g.Name, why))
+		}
+	}
+	for _, r := range d.Resources {
+		if c, ok := r.(*declaration.Container); ok && c.RunAs != nil {
+			if err := rootIdentity(c.RunAs.UID, c.RunAs.GID); err != nil {
+				problems = append(problems, fmt.Sprintf("the container %s is told to run as uid %d, gid %d: %v",
+					c.ID, c.RunAs.UID, c.RunAs.GID, err))
+			}
+		}
+	}
+	if len(problems) == 0 {
+		return nil
+	}
+	sort.Strings(problems)
+	return fmt.Errorf("%s; nothing of this declaration was made or started", strings.Join(problems, "; "))
+}
+
+// applyIdentity makes one identity as declared (ADR 0328 §4): its own group `mesh-<module>` with the number of its
+// uid, and its user with that uid, its primary group gid (its own, or its first library's) and its other libraries'
+// groups as added groups — or refuses it naming whose the name or the numbers are. A user the engine made is given
+// a changed primary or added groups; one that was there before is never changed, and is refused if it is not as
+// declared. What it made is recorded on known; an identity already right is unchanged.
+func applyIdentity(ctx context.Context, tool accountsTool, id declaration.Identity, names map[int]string,
+	known *store.State, run Runner) (Outcome, error) {
+	out := Outcome{ID: "identity " + id.Name, Type: "identity", Target: id.Name, Action: "unchanged"}
+	user, hasUser, err := lookUp(ctx, run, "passwd", id.Name)
+	if err != nil {
+		return out, err
+	}
+	group, hasGroup, err := lookUp(ctx, run, "group", id.Name)
+	if err != nil {
+		return out, err
+	}
+	byUID, uidUsed, err := lookUp(ctx, run, "passwd", strconv.Itoa(id.UID))
+	if err != nil {
+		return out, err
+	}
+	byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(id.UID))
+	if err != nil {
+		return out, err
+	}
+	rec := madeIdentity(known, id.Name)
+	const never = "; nothing was made, and nothing of the module was attempted (novox/hq ADR 0328 §4)"
+	switch {
+	case hasUser && user.id != id.UID:
+		return out, fmt.Errorf("the account %s is here with uid %d, and the mesh allocated uid %d%s", id.Name,
+			user.id, id.UID, never)
+	case hasUser && user.prim != id.GID && !rec.MadeUser:
+		return out, fmt.Errorf("the account %s is here with group %d, was not made by the mesh, and the mesh "+
+			"gives it group %d%s", id.Name, user.prim, id.GID, never)
+	case hasGroup && group.id != id.UID:
+		return out, fmt.Errorf("the group %s is here with gid %d, and a module's own group has its uid, %d%s",
+			id.Name, group.id, id.UID, never)
+	case uidUsed && byUID.name != id.Name:
+		return out, fmt.Errorf("uid %d is the account %s's here, and the mesh allocated it to %s%s", id.UID,
+			byUID.name, id.Name, never)
+	case gidUsed && byGID.name != id.Name:
+		return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s%s", id.UID,
+			byGID.name, id.Name, never)
+	}
+	added := make([]string, 0, len(id.Groups))
+	for _, gid := range id.Groups {
+		added = append(added, names[gid])
+	}
+	sort.Strings(added)
+	rec.Module, rec.Name, rec.UID, rec.GID = id.Module, id.Name, id.UID, id.GID
+	var made []string
+	if !hasGroup {
+		if err := tool.CreateSystemGroup(ctx, run, id.Name, id.UID); err != nil {
+			return out, err
+		}
+		rec.MadeGroup = true
+		made = append(made, fmt.Sprintf("its own group (gid %d)", id.UID))
+	}
+	if !hasUser {
+		if err := tool.CreateSystemAccount(ctx, run, id.Name, id.UID, id.GID, added); err != nil {
+			recordIdentity(known, rec)
+			return out, err
+		}
+		rec.MadeUser = true
+		made = append(made, fmt.Sprintf("the account (uid %d, group %d%s), with no login, no home and no password",
+			id.UID, id.GID, alsoIn(added)))
+	} else if rec.MadeUser {
+		// The mesh's own account: its primary and added groups follow what is declared.
+		in, err := system.GroupsOf(ctx, system.Runner(run), id.Name)
+		if err != nil {
+			return out, err
+		}
+		have := map[string]bool{}
+		for _, g := range in {
+			if g != id.Name && g != names[user.prim] {
+				have[g] = true
+			}
+		}
+		same := user.prim == id.GID && len(have) == len(added)
+		for _, g := range added {
+			same = same && have[g]
+		}
+		if !same {
+			if err := tool.ModifyAccount(ctx, run, id.Name, id.GID, added); err != nil {
+				return out, err
+			}
+			out.Action = "updated"
+			out.Detail = fmt.Sprintf("gave the account group %d%s", id.GID, alsoIn(added))
+		}
+	}
+	if rec.At.IsZero() {
+		rec.At = time.Now().UTC()
+	}
+	recordIdentity(known, rec)
+	if len(made) > 0 {
+		out.Action, out.Detail = "created", "made "+strings.Join(made, " and ")
+	} else if out.Detail == "" {
+		out.Detail = "here with its numbers"
+	}
+	return out, nil
+}
+
+func alsoIn(groups []string) string {
+	if len(groups) == 0 {
+		return ""
+	}
+	return ", also in " + strings.Join(groups, ", ")
+}
+
+// applyGroup makes one library's group with the number the controller allocated (ADR 0328 §3), or refuses it naming
+// whose the name or the number is, or that the number is a root group's (§15). What it made is recorded on known.
+func applyGroup(ctx context.Context, tool accountsTool, g declaration.Group, known *store.State,
+	run Runner) (Outcome, error) {
+	out := Outcome{ID: "group " + g.Name, Type: "group", Target: g.Name, Action: "unchanged"}
+	if why := rootGroup(g.GID); why != "" {
+		return out, fmt.Errorf("its gid is %s, and a library's group is never a root group (novox/hq ADR 0328 §15); "+
+			"nothing was made", why)
+	}
+	byName, hasName, err := lookUp(ctx, run, "group", g.Name)
+	if err != nil {
+		return out, err
+	}
+	byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(g.GID))
+	if err != nil {
+		return out, err
+	}
+	switch {
+	case hasName && byName.id != g.GID:
+		return out, fmt.Errorf("the group %s is here with gid %d, and the mesh allocated gid %d; nothing was made",
+			g.Name, byName.id, g.GID)
+	case gidUsed && byGID.name != g.Name:
+		return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s; nothing was made",
+			g.GID, byGID.name, g.Name)
+	case hasName:
+		out.Detail = "here with its number"
+		return out, nil
+	}
+	if err := tool.CreateSystemGroup(ctx, run, g.Name, g.GID); err != nil {
+		return out, err
+	}
+	known.Groups = append(known.Groups, store.MadeGroup{Name: g.Name, GID: g.GID, At: time.Now().UTC()})
+	out.Action, out.Detail = "created", fmt.Sprintf("made the library's group (gid %d)", g.GID)
+	return out, nil
+}
+
+// removeGroups removes every library group the engine made that is no longer declared, while it still has the
+// number the engine gave it. Never fatal: one that could not be removed (still a user's primary group) stays
+// recorded, said, for the next apply.
+func removeGroups(ctx context.Context, tool accountsTool, declared []declaration.Group, known *store.State,
+	run Runner) []Outcome {
+	wanted := map[string]bool{}
+	for _, g := range declared {
+		wanted[g.Name] = true
+	}
+	var out []Outcome
+	var keep []store.MadeGroup
+	for _, m := range known.Groups {
+		if wanted[m.Name] {
+			keep = append(keep, m)
+			continue
+		}
+		o := Outcome{ID: "group " + m.Name, Type: "group", Target: m.Name, Action: "removed",
+			Detail: "no longer declared: removed the group"}
+		g, has, err := lookUp(ctx, run, "group", m.Name)
+		switch {
+		case err != nil || (has && g.id != m.GID):
+			o.Action, o.Detail = "forgotten", "no longer declared: the group was left, it no longer has the gid the "+
+				"mesh gave it, or the database did not answer"
+		case has:
+			if err := tool.RemoveGroup(ctx, run, m.Name); err != nil {
+				o.Action, o.Detail = "waiting", "no longer declared: the group could not be removed: "+err.Error()
+				keep = append(keep, m)
+			}
+		default:
+			o.Action, o.Detail = "forgotten", "no longer declared, and no longer here"
+		}
+		out = append(out, o)
+	}
+	known.Groups = keep
+	return out
+}
+
+// madeIdentity is what the engine recorded of name, or an empty record.
+func madeIdentity(known *store.State, name string) store.MadeIdentity {
+	for _, m := range known.Identities {
+		if m.Name == name {
+			return m
+		}
+	}
+	return store.MadeIdentity{}
+}
+
+// recordIdentity records what was made of one identity, in place of any record before.
+func recordIdentity(known *store.State, rec store.MadeIdentity) {
+	for i, m := range known.Identities {
+		if m.Name == rec.Name {
+			known.Identities[i] = rec
+			return
+		}
+	}
+	if rec.MadeUser || rec.MadeGroup {
+		known.Identities = append(known.Identities, rec)
+	}
+}
+
+// removeIdentities removes what the engine made of every identity no longer declared — the account, then the
+// group — and only while it still has the numbers the engine gave it. Never fatal: what could not be removed
+// stays recorded, said, for the next apply.
+func removeIdentities(ctx context.Context, tool accountsTool, declared []declaration.Identity, known *store.State,
+	run Runner) []Outcome {
+	wanted := map[string]bool{}
+	for _, id := range declared {
+		wanted[id.Name] = true
+	}
+	var out []Outcome
+	var keep []store.MadeIdentity
+	for _, m := range known.Identities {
+		if wanted[m.Name] {
+			keep = append(keep, m)
+			continue
+		}
+		o := Outcome{ID: "identity " + m.Name, Type: "identity", Target: m.Name, Action: "removed"}
+		var said []string
+		if m.MadeUser {
+			if u, has, err := lookUp(ctx, run, "passwd", m.Name); err != nil || (has && u.id != m.UID) {
+				said = append(said, "the account was left: it no longer has the uid the mesh gave it, or the "+
+					"database did not answer")
+				m.MadeUser = false
+			} else if has {
+				if err := tool.RemoveAccount(ctx, run, m.Name); err != nil {
+					said = append(said, "the account could not be removed: "+err.Error())
+				} else {
+					m.MadeUser = false
+					said = append(said, "removed the account")
+				}
+			} else {
+				m.MadeUser = false
+			}
+		}
+		if m.MadeGroup && !m.MadeUser {
+			if g, has, err := lookUp(ctx, run, "group", m.Name); err != nil || (has && g.id != m.GID) {
+				said = append(said, "the group was left: it no longer has the gid the mesh gave it, or the "+
+					"database did not answer")
+				m.MadeGroup = false
+			} else if has {
+				if err := tool.RemoveGroup(ctx, run, m.Name); err != nil {
+					said = append(said, "the group could not be removed: "+err.Error())
+				} else {
+					m.MadeGroup = false
+					said = append(said, "removed the group")
+				}
+			} else {
+				m.MadeGroup = false
+			}
+		}
+		if m.MadeUser || m.MadeGroup {
+			o.Action = "waiting"
+			keep = append(keep, m)
+		}
+		o.Detail = "no longer declared: " + strings.Join(said, "; ")
+		out = append(out, o)
+	}
+	known.Identities = keep
+	return out
+}
+
+// toolFor is the accounts tool of a system, or why it has none.
+func toolFor(sys system.System) (accountsTool, error) {
+	if t, ok := sys.(accountsTool); ok {
+		return t, nil
+	}
+	return nil, errors.New("this system makes no module identity: it has no way to make a system account")
+}

internal/apply/identities_test.go

@@ -0,0 +1,385 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// The engine's half of ADR 0325 §3 and §11 (issue 567): every module given an account and a group of its own with
+// the numbers the controller allocated, a container told to run as it, and never a root number.
+
+// accountsMachine is a user and group database the runner answers from, and every command that changed it.
+type accountsMachine struct {
+	users, groups map[string][2]int // name → id, primary gid (a group's second is unused)
+	// in is each account's added groups, by name; homes each account's home, "/" when not set.
+	in    map[string][]string
+	homes map[string]string
+	ran   []string
+}
+
+// groupNamed is the name of gid in m, or the number.
+func (m *accountsMachine) groupNamed(gid int) string {
+	for n, ids := range m.groups {
+		if ids[0] == gid {
+			return n
+		}
+	}
+	return strconv.Itoa(gid)
+}
+
+func (m *accountsMachine) run(_ context.Context, name string, args ...string) (string, error) {
+	line := name + " " + strings.Join(args, " ")
+	switch name {
+	case "getent":
+		db, key := args[0], args[1]
+		table := m.users
+		if db == "group" {
+			table = m.groups
+		}
+		for n, ids := range table {
+			if n == key || strconv.Itoa(ids[0]) == key {
+				if db == "group" {
+					return n + ":x:" + strconv.Itoa(ids[0]) + ":\n", nil
+				}
+				home := m.homes[n]
+				if home == "" {
+					home = "/"
+				}
+				return n + ":x:" + strconv.Itoa(ids[0]) + ":" + strconv.Itoa(ids[1]) + "::" + home + ":/usr/bin/nologin\n", nil
+			}
+		}
+		return "", errors.New("getent exited 2")
+	case "id":
+		name := args[len(args)-1]
+		u, ok := m.users[name]
+		if !ok {
+			return "", errors.New("id exited 1")
+		}
+		return strings.Join(append([]string{m.groupNamed(u[1])}, m.in[name]...), " ") + "\n", nil
+	case "groupadd":
+		m.groups[args[len(args)-1]] = [2]int{mustAtoi(args[2]), 0}
+	case "useradd":
+		name := args[len(args)-1]
+		m.users[name] = [2]int{mustAtoi(args[2]), mustAtoi(args[4])}
+		if args[5] == "--groups" {
+			m.setIn(name, strings.Split(args[6], ","))
+		}
+		if m.homes == nil {
+			m.homes = map[string]string{}
+		}
+		m.homes[name] = "/nonexistent"
+	case "usermod":
+		name := args[len(args)-1]
+		if args[0] == "--append" {
+			m.setIn(name, append(m.in[name], args[2]))
+		} else if args[0] == "--home" {
+			if m.homes == nil {
+				m.homes = map[string]string{}
+			}
+			m.homes[name] = args[1]
+		} else {
+			u := m.users[name]
+			m.users[name] = [2]int{u[0], mustAtoi(args[1])}
+			m.setIn(name, strings.Split(args[3], ","))
+		}
+	case "gpasswd":
+		var kept []string
+		for _, g := range m.in[args[1]] {
+			if g != args[2] {
+				kept = append(kept, g)
+			}
+		}
+		m.setIn(args[1], kept)
+	case "userdel":
+		delete(m.users, args[0])
+	case "groupdel":
+		delete(m.groups, args[0])
+	default:
+		return "", nil
+	}
+	m.ran = append(m.ran, line)
+	return "", nil
+}
+
+func (m *accountsMachine) setIn(name string, groups []string) {
+	if m.in == nil {
+		m.in = map[string][]string{}
+	}
+	var out []string
+	for _, g := range groups {
+		if g != "" {
+			out = append(out, g)
+		}
+	}
+	m.in[name] = out
+}
+
+func mustAtoi(s string) int {
+	n, _ := strconv.Atoi(s)
+	return n
+}
+
+// withGroups is a declaration with library groups, identities and resources.
+func withGroups(t *testing.T, groups, identities, resources string) *declaration.Declaration {
+	t.Helper()
+	nothing := ""
+	if resources == "" {
+		nothing = `"owns_nothing":true,`
+	}
+	d, err := declaration.Parse([]byte(`{"declaration":1,"groups":[` + groups + `],"identities":[` + identities +
+		`],` + nothing + `"resources":[` + resources + `]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d
+}
+
+func withIdentities(t *testing.T, identities, resources string) *declaration.Declaration {
+	t.Helper()
+	d, err := declaration.Parse([]byte(`{"declaration":1,"identities":[` + identities + `],"resources":[` +
+		resources + `]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d
+}
+
+// Row 3: the account made as a system account with exactly its numbers, recorded, and removed — only what was
+// made — when no longer declared.
+func TestAModulesIdentityIsMadeWithItsNumbersAndRemovedWhenUndeclared(t *testing.T) {
+	m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
+	dir := t.TempDir()
+	d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
+		`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`)
+	report, known, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{
+		"groupadd --system --gid 61001 mesh-radarr",
+		"useradd --system --uid 61001 --gid 61001 --no-create-home --home-dir /nonexistent --shell /usr/bin/nologin mesh-radarr",
+	}
+	if strings.Join(m.ran, "\n") != strings.Join(want, "\n") {
+		t.Fatalf("made with:\n%s\nwant:\n%s", strings.Join(m.ran, "\n"), strings.Join(want, "\n"))
+	}
+	if o := outcomeOf(report, "identity mesh-radarr"); o.Action != "created" {
+		t.Errorf("the identity's outcome: %+v", o)
+	}
+	if len(known.Identities) != 1 || !known.Identities[0].MadeUser || !known.Identities[0].MadeGroup {
+		t.Fatalf("what was made is not recorded: %+v", known.Identities)
+	}
+
+	// Applied again: nothing is made twice.
+	m.ran = nil
+	if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if len(m.ran) != 0 {
+		t.Fatalf("an identity already here was made again: %v", m.ran)
+	}
+
+	// Undeclared, with a group the machine had before for another identity: only what was made goes.
+	m.groups["mesh-sonarr"] = [2]int{61002, 0}
+	other := withIdentities(t, `{"module":"sonarr","name":"mesh-sonarr","uid":61002,"gid":61002}`,
+		`{"id":"x.dir","type":"directory","path":"`+dir+`/x"}`)
+	if _, known, err = Apply(context.Background(), archHost(t), other, known, store.OriginDeclared, m.run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, has := m.users["mesh-radarr"]; has {
+		t.Error("an identity no longer declared kept its account")
+	}
+	if _, has := m.groups["mesh-radarr"]; has {
+		t.Error("an identity no longer declared kept its group")
+	}
+	if rec := madeIdentity(&known, "mesh-sonarr"); rec.MadeGroup {
+		t.Error("a group that was there before is recorded as the mesh's")
+	}
+}
+
+// Row 3: a name that exists with other numbers, or numbers another name holds, is refused naming whose, and the
+// module is not attempted; the rest of the machine is.
+func TestAnIdentityWhoseNameOrNumbersAreAnothersIsRefused(t *testing.T) {
+	dir := t.TempDir()
+	for _, tc := range []struct {
+		users, groups map[string][2]int
+		says          string
+	}{
+		{users: map[string][2]int{"mesh-radarr": {1500, 1500}}, groups: map[string][2]int{}, says: "uid 1500"},
+		{users: map[string][2]int{"someone": {61001, 100}}, groups: map[string][2]int{}, says: "someone"},
+		{users: map[string][2]int{}, groups: map[string][2]int{"media": {61001, 0}}, says: "media"},
+	} {
+		m := &accountsMachine{users: tc.users, groups: tc.groups}
+		d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
+			`{"id":"radarr.config","type":"directory","path":"`+dir+`/radarr"},`+
+				`{"id":"plex.config","type":"directory","path":"`+dir+`/plex"}`)
+		report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
+		if err == nil || !strings.Contains(err.Error(), tc.says) {
+			t.Errorf("not refused naming %s: %v", tc.says, err)
+		}
+		if len(m.ran) != 0 {
+			t.Errorf("something was made for an identity that is another's: %v", m.ran)
+		}
+		if o := outcomeOf(report, "radarr.config"); o.Action != "skipped" {
+			t.Errorf("the module of a refused identity was attempted: %+v", o)
+		}
+		if _, err := os.Stat(filepath.Join(dir, "plex")); err != nil {
+			t.Errorf("the rest of the machine was not applied: %v", err)
+		}
+		os.RemoveAll(filepath.Join(dir, "plex"))
+	}
+}
+
+// Row 11: a declaration naming a root group for an account or an image is refused with its reason, and nothing is
+// made or started. A number 0 is refused on arrival, by the parser.
+func TestNoModuleIdentityIsRoot(t *testing.T) {
+	was := RootGroupIDs
+	RootGroupIDs = func() map[int]string { return map[int]string{10: "wheel"} }
+	defer func() { RootGroupIDs = was }()
+	dir := t.TempDir()
+	for _, d := range []*declaration.Declaration{
+		withGroups(t, `{"name":"mesh-media","gid":10}`, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":10}`,
+			`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`),
+		withGroups(t, `{"name":"mesh-media","gid":10}`, ``,
+			`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`),
+		withIdentities(t, ``, `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"},`+
+			`{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`",`+
+			`"identity":{"mode":"run","uid":61001,"gid":10}}`),
+	} {
+		m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
+		_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
+		if err == nil || !strings.Contains(err.Error(), "wheel") || !strings.Contains(err.Error(), "never") {
+			t.Errorf("a root group was not refused naming why: %v", err)
+		}
+		if len(m.ran) != 0 {
+			t.Errorf("something was made or started: %v", m.ran)
+		}
+		if _, err := os.Stat(filepath.Join(dir, "config")); err == nil {
+			t.Error("a resource of a declaration naming a root group was applied")
+		}
+	}
+	for _, raw := range []string{
+		`{"declaration":1,"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]}`,
+		`{"declaration":1,"resources":[{"id":"m.app","type":"container","name":"m","image":"` + pinned + `",` +
+			`"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":0}}]}`,
+	} {
+		if _, err := declaration.Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "never root") {
+			t.Errorf("a number 0 was not refused on arrival: %v", err)
+		}
+	}
+}
+
+// Row 3: a container declaring identity is told it — by the variables its image reads, or the runtime's user
+// option — and a changed identity is a container recreated.
+func TestAContainerIsToldItsModulesIdentity(t *testing.T) {
+	env := &declaration.Container{ID: "radarr.app", Name: "radarr", Image: pinned, RunAs: &declaration.ContainerIdentity{
+		Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", UID: 61001, GID: 61001}}
+	run := &declaration.Container{ID: "plex.app", Name: "plex", Image: pinned, RunAs: &declaration.ContainerIdentity{
+		Mode: declaration.IdentityByRun, UID: 61002, GID: 61002}}
+	if got := strings.Join(foregroundRunArgs(env, "x"), " "); !strings.Contains(got, "--env PUID=61001 --env PGID=61001") {
+		t.Errorf("an image told by environment was not: %s", got)
+	}
+	if got := strings.Join(foregroundRunArgs(run, "x"), " "); !strings.Contains(got, "--user 61002:61002") {
+		t.Errorf("an image told by the runtime's user option was not: %s", got)
+	}
+	plain := *env
+	plain.RunAs = nil
+	moved := *env
+	moved.RunAs = &declaration.ContainerIdentity{Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID",
+		UID: 61009, GID: 61009}
+	if containerSpec(env, inputs{}) == containerSpec(&plain, inputs{}) ||
+		containerSpec(env, inputs{}) == containerSpec(&moved, inputs{}) {
+		t.Error("an identity told, or renumbered, does not recreate the container")
+	}
+}
+
+// ADR 0325 §5 supersedes ADR 0312's group grant: what a grant put an account in is taken back on the first apply of
+// the access, and nothing is granted again.
+func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) {
+	dir := t.TempDir()
+	groups := "nox media"
+	var ran []string
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		switch name {
+		case "id":
+			return groups + "\n", nil
+		case "gpasswd":
+			groups = "nox"
+		}
+		return "", nil
+	}
+	known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
+		Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
+	d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write","grant":{"account":"nox"}}`)
+	report, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if groups != "nox" || !strings.Contains(strings.Join(ran, "\n"), "gpasswd --delete nox media") {
+		t.Fatalf("the grant's group was not taken back: %v", ran)
+	}
+	if o := outcomeOf(report, "importer.books"); o.Action != "updated" || !strings.Contains(o.Detail, "taken back") {
+		t.Errorf("the take-back is not said: %+v", o)
+	}
+	if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 {
+		t.Errorf("the record still holds the grant's groups: %+v", rec)
+	}
+}
+
+// Review L7: a group the grant put an account in that could not be left stays recorded, and is tried again.
+func TestAGrantsGroupThatCouldNotBeLeftIsTriedAgain(t *testing.T) {
+	dir := t.TempDir()
+	refuse := true
+	groups := "nox media"
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		switch name {
+		case "id":
+			return groups + "\n", nil
+		case "gpasswd":
+			if refuse {
+				return "", errors.New("gpasswd exited 1: locked")
+			}
+			groups = "nox"
+		}
+		return "", nil
+	}
+	known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
+		Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
+	d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write"}`)
+	_, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rec, _ := known.Find("importer.books"); rec.User != "nox" || len(rec.Groups) != 1 {
+		t.Fatalf("a group that could not be left was forgotten: %+v", rec)
+	}
+	refuse = false
+	if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 || groups != "nox" {
+		t.Fatalf("the group was not taken back on the next apply: %+v (%s)", rec, groups)
+	}
+}
+
+// Every system the engine runs on that makes accounts makes module identities: arch and alpine alike.
+func TestArchAndAlpineMakeModuleIdentities(t *testing.T) {
+	for _, name := range []string{"arch", "alpine"} {
+		sys, err := system.For(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := toolFor(sys); err != nil {
+			t.Errorf("%s makes no module identity: %v", name, err)
+		}
+	}
+}

internal/apply/into.go

@@ -10,7 +10,7 @@ import (
 	"slices"
 	"sort"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/into_handover_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/into_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/keep.go

@@ -0,0 +1,244 @@
+package apply
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// An access is an operator's path a module mounts and the mesh does not own (novox/hq ADR 0051): mounted when
+// present, refused when absent, nothing changed on it. ADR 0325 kept a shared folder through ACL entries; ADR 0328
+// supersedes it with libraries (library.go), and an access is again exactly what it was before: an access's `users`
+// and `taken` are accepted and not acted on.
+
+// applyAccess is one access, applied as ADR 0051 has it; what ADR 0312's grant put an account in is taken back
+// first, so no account is in a group for data but a library's (ADR 0328 §11).
+func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
+	in inputs) (Outcome, error) {
+	gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed)
+	out, err := legacyAccess(r)
+	if len(left) > 0 {
+		out.user, out.groups = previous.User, left
+	}
+	if said != "" {
+		out.Detail = joinDetail(out.Detail, said)
+		if gave && out.Action == "unchanged" {
+			// Updated, so what names the access under restart-on — the tool runner — holds the groups it has now.
+			out.Action = "updated"
+		}
+	}
+	return out, err
+}
+
+// takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes
+// back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group.
+// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next
+// apply tries again — and what was said; nothing for an access that granted nothing.
+func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted,
+	handed Handed) (bool, []string, string) {
+	if previous.User == "" || len(previous.Groups) == 0 {
+		return false, nil, ""
+	}
+	account := previous.User
+	in, err := system.GroupsOf(ctx, system.Runner(run), account)
+	if err != nil {
+		return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+
+			"back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err)
+	}
+	already := setOf(in)
+	gave := false
+	var left, said []string
+	for _, g := range previous.Groups {
+		switch {
+		case !already[g]:
+			continue
+		case len(wanted.othersAsk(account, g, previous.ID)) > 0:
+			said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g,
+				strings.Join(wanted.othersAsk(account, g, previous.ID), ", ")))
+			handed.hand(account, g)
+			continue
+		}
+		gone, why := leaveGroup(ctx, sys, account, g, run)
+		if !gone {
+			left = append(left, g)
+			why += "; tried again on the next apply"
+		}
+		gave = gave || gone
+		said = append(said, why)
+	}
+	return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ")
+}
+
+// legacyAccess is an access from a controller older than ADR 0325, applied exactly as before it (ADR 0051 §7).
+//
+// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by the container
+// runtime, as root, with whatever mode it picks — which is exactly the silent wrong-ownership 04-ISSUES/026
+// records. So the host checks first and says plainly that the operator must provide the path, rather than
+// conjuring a directory it does not own and cannot give the right owner. Nothing is written, so this never reports
+// a change: the machine did not move, the host merely confirmed a fact about it.
+func legacyAccess(r *declaration.Access) (Outcome, error) {
+	out := begin(r)
+	info, err := os.Stat(r.Path)
+	if errors.Is(err, os.ErrNotExist) {
+		return out, fmt.Errorf(
+			"%s is not there, and the mesh does not own it — the operator must provide it. It is "+
+				"shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+
+				"nothing, so a missing one is said here rather than made as root by the container "+
+				"runtime", r.Path)
+	}
+	if err != nil {
+		return out, err
+	}
+	if !info.IsDir() {
+		return out, fmt.Errorf(
+			"%s is not a directory, and an access is a shared directory the operator provides",
+			r.Path)
+	}
+	mode := declaration.AccessRead
+	if r.Mode != "" {
+		mode = r.Mode
+	}
+	out.Action = "unchanged"
+	out.Detail = "operator-owned; present, " + mode + ", nothing managed"
+	return out, nil
+}
+
+// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the
+// folder's filesystem is not the one a warrant was given for. Empty when the filesystem cannot be named, and a folder
+// on it is then never kept (it fails closed, and waits).
+func PlacedID(mount string) string { return placedFrom(mount, filesystemKey(mount)) }
+
+// placedFrom is PlacedID from a filesystem key already read.
+func placedFrom(mount, key string) string {
+	if mount == "" || key == "" {
+		return ""
+	}
+	sum := sha256.Sum256([]byte(mount + "\x00" + key))
+	return "fs-" + hex.EncodeToString(sum[:6])
+}
+
+// filesystemKey is what names the filesystem at mount across reboots, in this order:
+//
+//   - a filesystem on a block device: its UUID, found by matching the mount's device against the devices in
+//     /dev/disk/by-uuid (never blkid). xfs's statfs id follows the device number, so the UUID comes first;
+//   - else the kernel's filesystem id (statfs f_fsid) when it is not zero: ZFS's from its dataset, btrfs's from its
+//     UUID;
+//   - else mountinfo's source with the filesystem's type: an NFS export's server:/export, a FUSE filesystem's own
+//     source string (both report f_fsid 0).
+//
+// Never the mount point and type alone, and never a device's name, which a reboot may renumber. Empty when none of
+// these answers within statBound: then nothing is kept there. Read once per mount at a time.
+func filesystemKey(mount string) string {
+	if mount == "" {
+		return ""
+	}
+	v, ok := boundedOnce("fs "+mount, func() any { return readFilesystemKey(mount) })
+	if !ok {
+		return ""
+	}
+	return v.(string)
+}
+
+func readFilesystemKey(mount string) string {
+	if u := MountUUID(mount); u != "" {
+		return "uuid:" + u
+	}
+	if id := MountFsid(mount); strings.Trim(id, "0") != "" {
+		return "fsid:" + id
+	}
+	if source, fstype := mountSourceOf(mount); source != "" && source != "none" {
+		return "source:" + fstype + ":" + source
+	}
+	return ""
+}
+
+// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
+// server is down blocks a stat for ever, and one access must never hold the apply.
+var statBound = 5 * time.Second
+
+// errNoAnswer is a folder whose read did not return within statBound.
+var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")
+
+// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
+
+func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) }
+func statWithin(path string) (os.FileInfo, error)  { return within("stat", os.Stat, path) }
+
+// flight is one read in flight, shared by everyone who asks while it is: a read that hangs on a dead mount holds
+// one goroutine per path, never one per apply or look.
+type flight struct {
+	done chan struct{}
+	v    any
+}
+
+var inFlight = struct {
+	sync.Mutex
+	reads map[string]*flight
+}{reads: map[string]*flight{}}
+
+// boundedOnce is read's answer within statBound, sharing a read of key already in flight; false when it did not
+// answer in time.
+func boundedOnce(key string, read func() any) (any, bool) {
+	inFlight.Lock()
+	f, running := inFlight.reads[key]
+	if !running {
+		f = &flight{done: make(chan struct{})}
+		inFlight.reads[key] = f
+		go func() {
+			f.v = read()
+			inFlight.Lock()
+			delete(inFlight.reads, key)
+			inFlight.Unlock()
+			close(f.done)
+		}()
+	}
+	inFlight.Unlock()
+	select {
+	case <-f.done:
+		return f.v, true
+	case <-time.After(statBound):
+		return nil, false
+	}
+}
+
+func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
+	type read struct {
+		info os.FileInfo
+		err  error
+	}
+	v, ok := boundedOnce(op+" "+path, func() any {
+		info, err := stat(path)
+		return read{info, err}
+	})
+	if !ok {
+		return nil, errNoAnswer
+	}
+	r := v.(read)
+	return r.info, r.err
+}
+
+// absPath is an absolute path in words: what pathless takes out.
+var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`)
+
+// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other
+// path said as "a path".
+func Pathless(err error) string { return pathless(err) }
+
+func pathless(err error) string {
+	var pe *os.PathError
+	if errors.As(err, &pe) {
+		return pe.Op + ": " + pe.Err.Error()
+	}
+	return absPath.ReplaceAllString(err.Error(), "${1}a path")
+}

internal/apply/left_out_test.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/library.go

@@ -0,0 +1,569 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+)
+
+// A library is kept usable by its group (novox/hq ADR 0328, issue 567).
+//
+// A library is a named tree of shared data with one home (§1). On its home the engine keeps its root and every
+// sub-path its modules ask for — those folders themselves, never what is in them — in the library's group with
+// group read, write and enter and the setgid bit, so what is made beneath keeps the group (§6); a missing sub-path
+// is made with the root's owner and the library's group (§9). The owner, the owner's bits and others' bits are
+// never changed. What the root already holds joins only on the operator's word (§7): until then the library waits
+// with reason keep and nothing under it is walked. A root that is absent, or whose disk is not there, makes
+// nothing, holds every container that mounts the library, and waits with reason disk (§9). Off its home a library is
+// a mount the mounts module makes: nothing is kept there, and a mount that is not there holds the containers too.
+//
+// **When a root is the mesh's to keep without asking.** When it holds nothing yet, or is already in the library's
+// group with group write and setgid (kept so before, by hand or by the mesh). Every other root waits for the take
+// act, which walks it whole (TakeLibrary). Kept, it stays kept on the filesystem it was kept on — named by UUID,
+// filesystem id or source (PlacedID) — and another filesystem at its root is judged afresh.
+
+// libraryRecords is where the engine keeps what it knows of each library it is home to: the file beside the state
+// in the engine, memory in a test.
+type libraryRecords interface {
+	All() (map[string]store.LibraryRecord, error)
+	Update(name string, change func(r *store.LibraryRecord, had bool) bool) error
+}
+
+// Libraries is the engine's record of the libraries it is home to. The engine sets the file beside its state.
+var Libraries libraryRecords = NewLibraryMemory()
+
+type libraryMemory struct {
+	mu sync.Mutex
+	m  map[string]store.LibraryRecord
+}
+
+// NewLibraryMemory is an empty record held in memory: a test's, or a one-shot apply's.
+func NewLibraryMemory() libraryRecords { return &libraryMemory{m: map[string]store.LibraryRecord{}} }
+
+func (k *libraryMemory) All() (map[string]store.LibraryRecord, error) {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	out := map[string]store.LibraryRecord{}
+	for n, r := range k.m {
+		out[n] = r
+	}
+	return out, nil
+}
+
+func (k *libraryMemory) Update(name string, change func(r *store.LibraryRecord, had bool) bool) error {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	r, had := k.m[name]
+	r.Library = name
+	if change(&r, had) {
+		k.m[name] = r
+	} else {
+		delete(k.m, name)
+	}
+	return nil
+}
+
+// libraryRecord is the record of one library, and whether there is one. A record that cannot be read is none: the
+// library is judged afresh, which waits rather than keeps.
+func libraryRecord(name string) (store.LibraryRecord, bool) {
+	all, err := Libraries.All()
+	if err != nil {
+		return store.LibraryRecord{}, false
+	}
+	r, had := all[name]
+	return r, had
+}
+
+// libraryLocks is one lock per library, held by every change to its root and every walk under it.
+var libraryLocks sync.Map
+
+func libraryLock(name string) *sync.Mutex {
+	l, _ := libraryLocks.LoadOrStore(name, &sync.Mutex{})
+	return l.(*sync.Mutex)
+}
+
+// libraryWalks is every library a take walk is running under now, with the filesystem it walks: set once the walk
+// has been judged and started, cleared when it ends.
+var libraryWalks sync.Map
+
+// LibraryAccess is the access a library's wait and its take act name.
+func LibraryAccess(name string) string { return declaration.LibraryAccessPrefix + name }
+
+// KeepWords and DiskWords are what a library's wait says the operator gives (ADR 0328 §8, §9): never a path.
+func KeepWords(name string) string { return "let the mesh keep the library " + name }
+func DiskWords(name string) string { return "mount the disk that holds the library " + name }
+
+// The judgements of a library.
+const (
+	// libraryKeep is a library the mesh keeps: taken, empty when first seen, or found in shape.
+	libraryKeep = "keep"
+	// libraryWaitKeep is a root that holds data never taken: it waits for the operator's word.
+	libraryWaitKeep = "wait-keep"
+	// libraryWaitDisk is a root that is absent, or whose disk (or, off its home, whose mount) is not there.
+	libraryWaitDisk = "wait-disk"
+	// libraryUnknown is a root whose filesystem could not be named now: left as it is until it is.
+	libraryUnknown = "unknown"
+	// libraryMounted is a library off its home whose mount is there: nothing to keep.
+	libraryMounted = "mounted"
+)
+
+// libraryVerdict is the judgement of one library, with what it read.
+type libraryVerdict struct {
+	kind string
+	// how is why a kept library is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptInShape.
+	how string
+	// mount is the mount point that holds the root; placed and fs name its filesystem.
+	mount, placed, fs string
+	// moved is that another filesystem than the one the library was kept on is at its root.
+	moved bool
+	why   string
+	err   error
+}
+
+// assessLibrary judges a library as an apply, a look and the take act all do (ADR 0328 §7, §9, §15): its group
+// first, then its disk, then whether the mesh may keep it. Its words carry no path: they reach the operator's
+// channel. Reads only.
+func assessLibrary(ctx context.Context, l declaration.Library, rec store.LibraryRecord) libraryVerdict {
+	if why := rootGroup(l.GID); why != "" {
+		return libraryVerdict{err: fmt.Errorf("its group is %s, and a library's group is never a root group "+
+			"(novox/hq ADR 0328 §15); nothing was made or changed", why)}
+	}
+	// Nowhere the machine's own trees are, as for an access (novox/hq issue 339): a library at /etc would be
+	// the machine's configuration given a group.
+	if err := refusePath(l.Root, placingAccess, "", ""); err != nil {
+		return libraryVerdict{err: errors.New(pathless(err) + "; nothing was made or changed")}
+	}
+	if !l.Home {
+		// Off its home the mounts module mounts it: there when its root is a mount point now.
+		for _, p := range MountPoints() {
+			if p == l.Root {
+				return libraryVerdict{kind: libraryMounted, mount: p}
+			}
+		}
+		return libraryVerdict{kind: libraryWaitDisk, why: "its mount is not there"}
+	}
+	info, err := lstatWithin(l.Root)
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+		return libraryVerdict{kind: libraryWaitDisk, why: "its root is not there"}
+	case errors.Is(err, errNoAnswer):
+		return libraryVerdict{kind: libraryUnknown, why: "its root does not answer: a read of it did not return " +
+			"within its bound"}
+	case err != nil:
+		return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
+	case !info.IsDir():
+		return libraryVerdict{err: errors.New("its root is not a folder (a file, or a link), and a library's root " +
+			"is a folder the engine reaches through no link")}
+	}
+	pl := placedOn(ctx, l.Root, l.Root, rec.Mount, recordedLibraryMounts(l.Library))
+	if pl.Away {
+		return libraryVerdict{kind: libraryWaitDisk, mount: pl.Mount, why: "the disk that holds its root is not there"}
+	}
+	fs := filesystemKey(pl.Mount)
+	here := placedFrom(pl.Mount, fs)
+	// **A filesystem that cannot be named now is unknown, never a move**: the library holds what it is — nothing
+	// walked, nothing newly kept, nothing reset — and is judged again when it is named. One never kept is so never
+	// kept: it fails closed.
+	if here == "" {
+		return libraryVerdict{kind: libraryUnknown, mount: pl.Mount,
+			why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"}
+	}
+	moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here)
+	v := libraryVerdict{mount: pl.Mount, placed: here, fs: fs, moved: moved}
+	// The controller's word that the library was taken holds for the filesystem the warrant was carried out on.
+	takenHere := l.Taken && (rec.KeptOn == here || (rec.KeptOn == "" && !moved))
+	switch {
+	case rec.Kept && rec.KeptOn == here && !moved && rec.How == store.KeptByWarrant, takenHere:
+		// Taken whole on the operator's word: every sub-path is the library's.
+		v.kind, v.how = libraryKeep, store.KeptByWarrant
+		return v
+	case rec.Kept && rec.KeptOn == here && !moved:
+		// Kept without a warrant (empty, or in shape): a sub-path asked for since that holds data not in the group
+		// waits for the operator's word as the root would (§7).
+		v.kind, v.how = libraryKeep, rec.How
+		if !pathsInShape(l, rec.Paths) {
+			v.kind, v.how = libraryWaitKeep, ""
+		}
+		return v
+	}
+	facts, err := readFolder(l.Root)
+	if err != nil {
+		return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
+	}
+	switch {
+	case facts.Empty:
+		// Nothing there, so no sub-path either: every one is made.
+		v.kind, v.how = libraryKeep, store.KeptEmpty
+	case int(facts.GID) == l.GID && facts.Mode&0o2070 == 0o2070 && sampleInShape(l.Root, l.GID) &&
+		pathsInShape(l, nil):
+		// The root and a bounded sample beneath it already in the group with group write and setgid: data that
+		// is the library's already, kept without asking. One entry of the sample out of shape asks (§7).
+		v.kind, v.how = libraryKeep, store.KeptInShape
+	default:
+		// What is there holds data the mesh never took — a disk mounted over a root kept while it was empty below
+		// it included (a move): it waits for the operator's word, and nothing under it is walked.
+		v.kind = libraryWaitKeep
+	}
+	return v
+}
+
+// recordedLibraryMounts is every mount point recorded for another library: where a disk was once seen.
+func recordedLibraryMounts(except string) []string {
+	all, err := Libraries.All()
+	if err != nil {
+		return nil
+	}
+	var out []string
+	for name, r := range all {
+		if name != except && r.Mount != "" && r.Mount != "/" {
+			out = append(out, r.Mount)
+		}
+	}
+	return out
+}
+
+// applyLibrary is one library, applied (ADR 0328 §6, §7, §9): kept on its home when the mesh may keep it, waiting
+// otherwise, and failing for a root group. Nothing is changed on the root but the group, group rwx and setgid of
+// the root and its requested sub-paths, and the sub-paths made.
+func applyLibrary(ctx context.Context, l declaration.Library) (Outcome, libraryVerdict, error) {
+	out := Outcome{ID: LibraryAccess(l.Library), Type: "library", Target: l.Root}
+	lock := libraryLock(l.Library)
+	if !lock.TryLock() {
+		// **A take walk is running (up to its bound, half an hour): the apply does not wait for it.** The library is
+		// the walk's until it is through; it keeps it, and its containers are not held for it. Only a walk that
+		// has started, on the filesystem it judged: anything else holding the lock (a take act about to be
+		// refused, say) is short, and the apply waits for it and judges the library as always.
+		if on, walking := libraryWalks.Load(l.Library); walking {
+			out.Action = "unchanged"
+			out.Detail = "the take act is setting its group under its root now; this apply leaves the library to it"
+			return out, libraryVerdict{kind: libraryKeep, how: store.KeptByWarrant, placed: on.(string)}, nil
+		}
+		lock.Lock()
+	}
+	defer lock.Unlock()
+	rec, _ := libraryRecord(l.Library)
+	v := assessLibrary(ctx, l, rec)
+	if v.err != nil {
+		return out, v, v.err
+	}
+	switch v.kind {
+	case libraryMounted:
+		out.Action, out.Detail = "unchanged", "not its home: mounted, and nothing is kept here"
+		return out, v, nil
+	case libraryWaitDisk:
+		out.Action = "waiting"
+		out.Detail = v.why + ": nothing is made, and no container that mounts it is created until it is there " +
+			"(novox/hq ADR 0328 §9); waits for the operator to " + DiskWords(l.Library)
+		return out, v, nil
+	case libraryUnknown:
+		out.Action = "waiting"
+		out.Detail = v.why + ": the library is left exactly as it is, and judged again on the next apply"
+		return out, v, nil
+	case libraryWaitKeep:
+		_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, had bool) bool {
+			if !had || v.moved {
+				r.Root, r.Mount, r.Placed, r.Fs = l.Root, v.mount, v.placed, v.fs
+			}
+			if v.moved {
+				// Kept on another filesystem: what was kept there no longer holds here.
+				r.Kept, r.How, r.Cursor = false, "", nil
+			}
+			return true
+		})
+		out.Action = "waiting"
+		out.Detail = "its root holds data the mesh has not taken: nothing is changed under it until the operator's " +
+			"warrant (novox/hq ADR 0328 §7); waits for the operator to " + KeepWords(l.Library)
+		return out, v, nil
+	}
+	changed, made, err := keepFolders(l.Root, l.Paths, l.GID)
+	if err != nil {
+		return out, v, err
+	}
+	_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, _ bool) bool {
+		if !r.Kept || r.KeptOn != v.placed {
+			r.How, r.At = v.how, time.Now().UTC()
+		}
+		r.Root, r.Mount, r.Placed, r.Fs, r.GID = l.Root, v.mount, v.placed, v.fs, l.GID
+		r.Kept, r.KeptOn = true, v.placed
+		r.Paths = unionPaths(r.Paths, l.Paths)
+		return true
+	})
+	out.Action = "unchanged"
+	if changed+made > 0 {
+		out.Action = "updated"
+	}
+	out.Detail = fmt.Sprintf("kept in %s: its root and %d sub-path(s) with group rwx and setgid, %d made; owners "+
+		"and everyone else's rights unchanged (novox/hq ADR 0328 §6)", l.Group, len(l.Paths), made)
+	return out, v, nil
+}
+
+// heldRoots is every library root whose containers are held: its disk, or its mount, is not there (§9).
+func heldRoots(verdicts map[string]libraryVerdict, libs []declaration.Library) map[string]string {
+	out := map[string]string{}
+	for _, l := range libs {
+		if v, ok := verdicts[l.Library]; ok && v.kind == libraryWaitDisk {
+			out[l.Root] = l.Library
+		}
+	}
+	return out
+}
+
+// heldLibraries is the root of every library whose disk or mount is not there, with its name, judged now and read
+// only: what a full apply holds its containers for (ADR 0328 §9), for one that judges without applying libraries.
+func heldLibraries(ctx context.Context, libs []declaration.Library) map[string]string {
+	verdicts := map[string]libraryVerdict{}
+	for _, l := range libs {
+		rec, _ := libraryRecord(l.Library)
+		verdicts[l.Library] = assessLibrary(ctx, l, rec)
+	}
+	return heldRoots(verdicts, libs)
+}
+
+// mountsHeld is the library a container waits for: one whose root it mounts, or mounts below, is held.
+func mountsHeld(c *declaration.Container, held map[string]string) string {
+	for _, v := range c.Volumes {
+		src := mountSource(v)
+		if !strings.HasPrefix(src, "/") {
+			continue
+		}
+		for root, name := range held {
+			if atOrBelow(filepath.Clean(src), root) {
+				return name
+			}
+		}
+	}
+	return ""
+}
+
+// WaitingOnLibrary is every container of the declaration that waits for a library whose disk or mount is not
+// there, read now: not created by the apply, so not judged alive or dead — the library says why.
+func WaitingOnLibrary(d *declaration.Declaration) map[string]bool {
+	out := map[string]bool{}
+	if d == nil || len(d.Libraries) == 0 {
+		return out
+	}
+	held := heldLibraries(context.Background(), d.Libraries)
+	for _, r := range d.Resources {
+		if c, ok := r.(*declaration.Container); ok && mountsHeld(c, held) != "" {
+			out[c.ID] = true
+		}
+	}
+	return out
+}
+
+// The judgements of a library as a look says them.
+const (
+	VerdictKeep     = "keep"
+	VerdictMounted  = "mounted"
+	VerdictWaitKeep = "wait-keep"
+	VerdictWaitDisk = "wait-disk"
+	VerdictUnknown  = "unknown"
+	VerdictBroken   = "broken"
+)
+
+// LibraryJudgement is a look's judgement of a library, and why, in words with no path; Placed names its root's
+// filesystem for a keep wait, with no path (PlacedID).
+type LibraryJudgement struct {
+	Kind, Why, Placed string
+	// Act is the last walk under its root.
+	Act *store.KeptAct
+}
+
+// JudgeLibrary judges a library for a look, read only.
+func JudgeLibrary(ctx context.Context, l declaration.Library) LibraryJudgement {
+	rec, _ := libraryRecord(l.Library)
+	v := assessLibrary(ctx, l, rec)
+	j := LibraryJudgement{Act: rec.Act}
+	switch {
+	case v.err != nil:
+		j.Kind, j.Why = VerdictBroken, pathless(v.err)
+	case v.kind == libraryWaitKeep:
+		j.Kind, j.Placed = VerdictWaitKeep, v.placed
+	case v.kind == libraryWaitDisk:
+		j.Kind, j.Why = VerdictWaitDisk, v.why
+	case v.kind == libraryUnknown:
+		j.Kind, j.Why = VerdictUnknown, v.why
+	case v.kind == libraryMounted:
+		j.Kind = VerdictMounted
+	default:
+		j.Kind = VerdictKeep
+	}
+	return j
+}
+
+// joinRunner puts a library's tool runner's account in its group (ADR 0328 §11).
+func joinRunner(ctx context.Context, sys system.System, l declaration.Library, known *store.State,
+	run Runner) (bool, string, error) {
+	if l.Runner == "" {
+		return false, "", nil
+	}
+	return joinGroup(ctx, sys, l.Runner, l.Group, store.Membership{Library: l.Library}, known, run)
+}
+
+// IdentityChange is what a tool runner joining a module's own group changes (ADR 0334 §2): `identity:<module>`,
+// which the tool runner's unit names under restart-on, as it names `library:<name>`.
+func IdentityChange(module string) string { return "identity:" + module }
+
+// joinGroup puts a tool runner's account in a group (ADR 0328 §11, ADR 0334 §2), as ADR 0252 puts an account in a
+// group: recorded only when the mesh put it there, so one it was in before is never taken back. The account must be
+// here and never root's: uid 0, or a root group as its primary (§15). Whether it moved, and what was said.
+func joinGroup(ctx context.Context, sys system.System, account, group string, of store.Membership, known *store.State,
+	run Runner) (bool, string, error) {
+	who, found, err := lookUp(ctx, run, "passwd", account)
+	switch {
+	case err != nil:
+		return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
+	case !found:
+		return false, "", fmt.Errorf("the tool runner's account %s is not on this machine, so it joins no group", account)
+	}
+	if err := rootIdentity(who.id, who.prim); err != nil {
+		return false, "", fmt.Errorf("the tool runner's account %s joins no group: %w", account, err)
+	}
+	in, err := system.GroupsOf(ctx, system.Runner(run), account)
+	if err != nil {
+		return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
+	}
+	if setOf(in)[group] {
+		return false, "", nil
+	}
+	if err := sys.AddUserToGroup(ctx, system.Runner(run), account, group); err != nil {
+		return false, "", err
+	}
+	for _, m := range known.Members {
+		if m.Account == account && m.Group == group {
+			return true, fmt.Sprintf("put the tool runner's %s back in %s", account, group), nil
+		}
+	}
+	of.Account, of.Group = account, group
+	known.Members = append(known.Members, of)
+	return true, fmt.Sprintf("put the tool runner's %s in %s; it holds it once restarted", account, group), nil
+}
+
+// leaveRunners takes every tool runner's account the mesh put in a library's group, or a module's own group, out of
+// it once no declared library or identity puts it there and no declared user resource asks for it (ADR 0252 §3). Never fatal: what could not be
+// left stays recorded, said, for the next apply.
+func leaveRunners(ctx context.Context, sys system.System, libs []declaration.Library, ids []declaration.Identity,
+	known *store.State, run Runner, wanted Wanted) []Outcome {
+	declared := map[string]bool{}
+	for _, l := range libs {
+		if l.Runner != "" {
+			declared[l.Runner+" "+l.Group] = true
+		}
+	}
+	for _, id := range ids {
+		if id.Runner != "" {
+			declared[id.Runner+" "+id.Name] = true
+		}
+	}
+	var out []Outcome
+	var keep []store.Membership
+	for _, m := range known.Members {
+		if declared[m.Account+" "+m.Group] {
+			keep = append(keep, m)
+			continue
+		}
+		o := Outcome{ID: LibraryAccess(m.Library) + " runner", Type: "library", Target: m.Account,
+			changes: LibraryAccess(m.Library)}
+		if m.Module != "" {
+			o.ID, o.Type = "identity "+declaration.IdentityPrefix+m.Module+" runner", "identity"
+			o.changes = IdentityChange(m.Module)
+		}
+		if others := wanted.othersAsk(m.Account, m.Group, ""); len(others) > 0 {
+			o.Action, o.Detail = "forgotten", fmt.Sprintf("left %s in %s, which %s asks for", m.Account, m.Group,
+				strings.Join(others, ", "))
+			out = append(out, o)
+			continue
+		}
+		gone, why := leaveGroup(ctx, sys, m.Account, m.Group, run)
+		o.Action, o.Detail = "restored", m.Account+" "+why
+		if !gone {
+			o.Action = "waiting"
+			keep = append(keep, m)
+		}
+		out = append(out, o)
+	}
+	known.Members = keep
+	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
+	return out
+}
+
+// pathsInShape is whether every requested sub-path not kept before is absent (it is made), empty, or already in the
+// group with group write and setgid, it and a bounded sample beneath it: what may be kept without the operator's
+// word (§7). One that cannot be read is not.
+func pathsInShape(l declaration.Library, kept []string) bool {
+	done := map[string]bool{}
+	for _, p := range kept {
+		done[p] = true
+	}
+	for _, p := range l.Paths {
+		if done[p] {
+			continue
+		}
+		path := filepath.Join(l.Root, p)
+		if _, err := lstatWithin(path); errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		facts, err := readFolder(path)
+		if err != nil {
+			return false
+		}
+		if facts.Empty {
+			continue
+		}
+		if int(facts.GID) != l.GID || facts.Mode&0o2070 != 0o2070 || !sampleInShape(path, l.GID) {
+			return false
+		}
+	}
+	return true
+}
+
+// unionPaths is a with every path of b not in it.
+func unionPaths(a, b []string) []string {
+	have := map[string]bool{}
+	for _, p := range a {
+		have[p] = true
+	}
+	for _, p := range b {
+		if !have[p] {
+			a = append(a, p)
+			have[p] = true
+		}
+	}
+	sort.Strings(a)
+	return a
+}
+
+// RunnerWarning is a tool runner's account that joins no group (missing, or root's): said as a resource of its own,
+// `runner:<module>` or `runner:library:<name>`, unhealthy with words with no path — never a failure of the apply.
+type RunnerWarning struct {
+	Resource, Module, Why string
+}
+
+var runnerWarnings struct {
+	sync.Mutex
+	list []RunnerWarning
+}
+
+// setRunnerWarnings is the warnings the last apply found, in place of the ones before.
+func setRunnerWarnings(ws []RunnerWarning) {
+	runnerWarnings.Lock()
+	defer runnerWarnings.Unlock()
+	runnerWarnings.list = append([]RunnerWarning(nil), ws...)
+}
+
+// RunnerWarnings is every tool runner's account the last apply could not put in a group.
+func RunnerWarnings() []RunnerWarning {
+	runnerWarnings.Lock()
+	defer runnerWarnings.Unlock()
+	return append([]RunnerWarning(nil), runnerWarnings.list...)
+}

internal/apply/library_linux.go

@@ -0,0 +1,161 @@
+//go:build linux
+
+package apply
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// The take act on a library, and the walk that goes on with one stopped at its bound (novox/hq ADR 0328 §7).
+
+// actRecord is a walk's counts as the record keeps them.
+func actRecord(warrant, by, on string, started time.Time, w *treeWalk, err error) *store.KeptAct {
+	c := w.counts
+	rec := &store.KeptAct{Act: "take", Warrant: warrant, By: by, On: on, State: "done", Started: started,
+		Ended: time.Now().UTC(), Folders: c.Folders, Files: c.Files, Kept: c.Kept, Links: c.Links, Other: c.Other,
+		Mounts: c.Mounts, Shared: c.Shared, Linked: c.Linked, Stripped: c.Stripped}
+	if c.Shared+c.Linked+c.Stripped > 0 {
+		rec.Why = fmt.Sprintf("%d file(s) with another name given the group, %d root's left as they are, %d whose "+
+			"setuid or setgid bit the group change took", c.Linked, c.Shared, c.Stripped)
+	}
+	switch {
+	case errors.Is(err, ErrWalkBound):
+		rec.State, rec.Why = "stopped", "it met its bound of entries or time; it goes on from where it stopped"
+	case err != nil:
+		rec.State, rec.Why = "failed", err.Error()
+	case w.Failed != "":
+		rec.State = "failed"
+		rec.Why = fmt.Sprintf("%d entries could not be given the group: %s; walked again on the next look", c.Failed,
+			w.Failed)
+	}
+	return rec
+}
+
+// TakeLibrary is the take act (ADR 0328 §7): on the operator's warrant, the library's group, group read and write,
+// and on folders group enter and setgid, set on its root and everything under it — no link followed, no other
+// filesystem entered, the owner and the owner's and others' bits unchanged, a file with another name elsewhere left
+// as it is. It judges the ask against the declaration last applied and the root as it is, and starts the walk: what
+// it answers at once, and a channel closed when the walk is through or stopped at its bound. Refused, nothing is
+// changed.
+func TakeLibrary(ctx context.Context, d *declaration.Declaration, access, path, warrant, by string) (<-chan struct{},
+	string, error) {
+	if d == nil {
+		return nil, "", errors.New("this engine has applied no declaration, and takes no library it was not told of")
+	}
+	var lib *declaration.Library
+	for i := range d.Libraries {
+		if LibraryAccess(d.Libraries[i].Library) == access {
+			lib = &d.Libraries[i]
+		}
+	}
+	switch {
+	case lib == nil:
+		return nil, "", fmt.Errorf("the access %s names no library declared on this machine", access)
+	case !lib.Home:
+		return nil, "", fmt.Errorf("this machine is not the home of the library %s, and a library is taken on its "+
+			"home", lib.Library)
+	case lib.Root != path:
+		return nil, "", fmt.Errorf("the library %s is declared at another root than the one asked", lib.Library)
+	}
+	lock := libraryLock(lib.Library)
+	if !lock.TryLock() {
+		return nil, "", errors.New("a walk of the library is running; asked again once it is through, the act goes on")
+	}
+	rec, _ := libraryRecord(lib.Library)
+	v := assessLibrary(ctx, *lib, rec)
+	var refused error
+	switch {
+	case v.err != nil:
+		refused = v.err
+	case v.kind == libraryWaitDisk:
+		refused = errors.New(v.why)
+	case v.kind == libraryUnknown:
+		refused = errors.New(v.why + ", so a warrant could name no filesystem it was given for")
+	}
+	if refused != nil {
+		lock.Unlock()
+		return nil, "", refused
+	}
+	if _, _, err := keepFolders(lib.Root, lib.Paths, lib.GID); err != nil {
+		lock.Unlock()
+		return nil, "", err
+	}
+	started := time.Now().UTC()
+	_ = Libraries.Update(lib.Library, func(r *store.LibraryRecord, _ bool) bool {
+		r.Root, r.Mount, r.Placed, r.Fs, r.GID = lib.Root, v.mount, v.placed, v.fs, lib.GID
+		r.Kept, r.KeptOn, r.How, r.At = true, v.placed, store.KeptByWarrant, started
+		r.Warrant, r.By, r.Cursor = warrant, by, nil
+		r.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, On: v.placed, State: "running", Started: started}
+		return true
+	})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		defer lock.Unlock()
+		walkLibrary(lib.Library, lib.Root, lib.GID, warrant, by, v.placed, nil)
+	}()
+	return done, fmt.Sprintf("the take act started under the warrant %s: %s's group, group read and write, and "+
+		"setgid on its folders are set on everything under its root, following no link and entering no other "+
+		"filesystem", warrant, lib.Library), nil
+}
+
+// walkLibrary walks a library's root from resume with its group, and records what it did: a walk stopped at its
+// bound keeps its cursor for the next. The caller holds the library's lock.
+func walkLibrary(name, root string, gid int, warrant, by, on string, resume []string) {
+	libraryWalks.Store(name, on)
+	defer libraryWalks.Delete(name)
+	started := time.Now().UTC()
+	w, err := walkRoot(root, ActLimits, machineStats, resume, groupVisit(gid))
+	rec := actRecord(warrant, by, on, started, w, err)
+	_ = Libraries.Update(name, func(r *store.LibraryRecord, _ bool) bool {
+		r.Act = rec
+		r.Cursor = nil
+		if errors.Is(err, ErrWalkBound) {
+			r.Cursor = w.Cursor
+		}
+		return true
+	})
+}
+
+// ReconcileLibraries goes on with every take act that stopped at its bound or left entries it could not change,
+// from where it stopped — only while the library is still kept on the filesystem it was taken on (judged as an apply
+// judges it). What it did, in words, for the engine to say.
+func ReconcileLibraries(ctx context.Context, d *declaration.Declaration) []string {
+	if d == nil {
+		return nil
+	}
+	var said []string
+	for _, l := range d.Libraries {
+		if ctx.Err() != nil {
+			break
+		}
+		if !l.Home {
+			continue
+		}
+		rec, _ := libraryRecord(l.Library)
+		if rec.Act == nil || (rec.Act.State != "stopped" && rec.Act.State != "failed") || rec.How != store.KeptByWarrant {
+			continue
+		}
+		lock := libraryLock(l.Library)
+		if !lock.TryLock() {
+			continue
+		}
+		v := assessLibrary(ctx, l, rec)
+		if v.err != nil || v.kind != libraryKeep || v.placed != rec.KeptOn || v.moved {
+			lock.Unlock()
+			continue
+		}
+		walkLibrary(l.Library, l.Root, l.GID, rec.Warrant, rec.By, rec.KeptOn, rec.Cursor)
+		lock.Unlock()
+		after, _ := libraryRecord(l.Library)
+		said = append(said, fmt.Sprintf("the library %s: the take act went on, %s (%d folders and %d files changed)",
+			l.Library, after.Act.State, after.Act.Folders, after.Act.Files))
+	}
+	return said
+}

internal/apply/logging_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/maintenance_window_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/mesh_dirs.go

@@ -30,7 +30,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // LinkInPathError is a mesh directory refused because a link in its path is one another account could replace.
@@ -94,13 +94,14 @@ func groupNamed(gid uint32) string {
 func JudgeDirectories(d *declaration.Declaration) string {
 	var lines []string
 	var placed, refused, guarded, homed int
+	declared := declaredHomes(d.Resources)
 	for _, r := range d.Resources {
 		dir, ok := r.(*declaration.Directory)
 		if !ok {
 			continue
 		}
 		path := filepath.Clean(dir.Path)
-		if err := refusePath(path, placingDirectory, moduleOfID(dir.ID), dir.Owner); err != nil {
+		if err := refusePathWith(path, placingDirectory, moduleOfID(dir.ID), dir.Owner, declared); err != nil {
 			guarded++
 			lines = append(lines, path+": refused already by the placement guard: "+err.Error())
 			continue

internal/apply/mesh_dirs_apply_linux.go

@@ -14,7 +14,7 @@ import (
 
 	"golang.org/x/sys/unix"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // meshDirsByDescriptor says this system can open a directory component by component without following a link.
@@ -77,9 +77,10 @@ func applyMeshDirectory(r *declaration.Directory, out Outcome, mode os.FileMode,
 	if err := unix.Fstat(fd, &st); err != nil {
 		return out, &os.PathError{Op: "fstat", Path: r.Path, Err: err}
 	}
-	beforeMode, beforeUID, beforeGID := os.FileMode(st.Mode&0o777), int(st.Uid), int(st.Gid)
-	// Set explicitly even when it existed: a mode given at creation is not a mode maintained.
-	if err := unix.Fchmod(fd, uint32(mode.Perm())); err != nil {
+	beforeMode, beforeUID, beforeGID := statDirMode(st.Mode), int(st.Uid), int(st.Gid)
+	// Set explicitly even when it existed: a mode given at creation is not a mode maintained — its setgid bit
+	// with it (novox/hq ADR 0334 §1).
+	if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
 		return out, &os.PathError{Op: "chmod", Path: r.Path, Err: err}
 	}
 	ownedAlready := !chown || (beforeUID == uid && beforeGID == gid)
@@ -87,6 +88,10 @@ func applyMeshDirectory(r *declaration.Directory, out Outcome, mode os.FileMode,
 		if err := unix.Fchown(fd, uid, gid); err != nil {
 			return out, fmt.Errorf("cannot give %s to %q: %w", r.Path, r.Owner, &os.PathError{Op: "chown", Path: r.Path, Err: err})
 		}
+		// A chown may take the setgid bit: the mode is set again after it.
+		if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
+			return out, &os.PathError{Op: "chmod", Path: r.Path, Err: err}
+		}
 	}
 
 	// Read back, from the descriptor that was changed.
@@ -96,8 +101,8 @@ func applyMeshDirectory(r *declaration.Directory, out Outcome, mode os.FileMode,
 	if st.Mode&unix.S_IFMT != unix.S_IFDIR {
 		return out, fmt.Errorf("%s is not a directory after applying", r.Path)
 	}
-	if got := os.FileMode(st.Mode & 0o777); got != mode.Perm() {
-		return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, got, mode.Perm())
+	if got := statDirMode(st.Mode); got != dirBits(mode) {
+		return out, fmt.Errorf("%s is mode %s after setting %s", r.Path, modeText(got), modeText(mode))
 	}
 
 	out.Action = "unchanged"
@@ -106,9 +111,9 @@ func applyMeshDirectory(r *declaration.Directory, out Outcome, mode os.FileMode,
 		out.Action = "created"
 	case !ownedAlready:
 		out.Action = "updated"
-	case beforeMode != mode.Perm():
+	case beforeMode != dirBits(mode):
 		out.Action = "updated"
-		out.Detail = fmt.Sprintf("mode %o to %o", beforeMode, mode.Perm())
+		out.Detail = fmt.Sprintf("mode %s to %s", modeText(beforeMode), modeText(mode))
 	}
 	return out, nil
 }

internal/apply/mesh_dirs_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/mesh_names_test.go

@@ -3,7 +3,7 @@ package apply
 import (
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // **A container's mesh names are part of what it is** (novox/hq 04-ISSUES/135).
@@ -23,7 +23,7 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) {
 		Name: was.Name, Image: was.Image,
 		Hosts: []string{"novox.internal:10.10.0.1", "umami.novox.be:10.10.0.1"},
 	}
-	if containerSpecReading(was, nil, nil, nil) == containerSpecReading(moved, nil, nil, nil) {
+	if containerSpecReading(was, nil, nil) == containerSpecReading(moved, nil, nil) {
 		t.Fatal("a container whose mesh names moved compares equal, so it is never recreated")
 	}
 
@@ -33,14 +33,14 @@ func TestAContainersMeshNamesAreComparedLikeTheRestOfIt(t *testing.T) {
 		Name: moved.Name, Image: moved.Image,
 		Hosts: []string{moved.Hosts[1], moved.Hosts[0]},
 	}
-	if containerSpecReading(moved, nil, nil, nil) != containerSpecReading(reordered, nil, nil, nil) {
+	if containerSpecReading(moved, nil, nil) != containerSpecReading(reordered, nil, nil) {
 		t.Fatal("the same names in another order read as a different container")
 	}
 
 	// A container the mesh gives no names is unaffected, so nothing is recreated for a field it
 	// does not set.
 	plain := &declaration.Container{Name: "plex", Image: was.Image}
-	if containerSpecReading(plain, nil, nil, nil) == containerSpecReading(was, nil, nil, nil) {
+	if containerSpecReading(plain, nil, nil) == containerSpecReading(was, nil, nil) {
 		return // different for other reasons, which is fine
 	}
 }

internal/apply/missing.go

@@ -1,122 +0,0 @@
-package apply
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/novox/mesh-host/internal/declaration"
-)
-
-// A missing accessed path waits for the operator (novox/hq ADR 0324, issue 567).
-//
-// ADR 0051 refused an access whose path is not there, and the refusal failed the whole apply: one module's
-// folder not made yet held every other module on the machine. The runtime must still not make it — a bind
-// whose source is missing is made by it as root, with whatever mode it picks (04-ISSUES/026) — so the engine
-// mounts nothing for the access: every container that mounts the path or below it is not created, and is said
-// waiting with the access, while the rest of the declaration applies. The access is stated waiting on every look
-// with a wait of kind access, and the operator's warrant for the create act makes the path, each level as its
-// nearest existing ancestor is. Only when that ancestor is a directory: one that is a file or a link is refused
-// as before, and one owned by a group that is never granted is stated unhealthy, with no wait.
-
-// MissingPath is an accessed path that is not there, and its nearest existing ancestor.
-type MissingPath struct {
-	Ancestor string
-	// Levels are the names to make below Ancestor, outermost first.
-	Levels []string
-	// Info is the ancestor as lstat reads it: a link is not a directory.
-	Info os.FileInfo
-}
-
-// NearestAncestor reads, for a path that is not there, its nearest existing ancestor by lstat. An error for a
-// path that is there, one not stated exactly, or an ancestor that cannot be read.
-func NearestAncestor(path string) (MissingPath, error) {
-	if err := shareablePath(path); err != nil {
-		return MissingPath{}, err
-	}
-	if _, err := os.Lstat(path); err == nil || !errors.Is(err, os.ErrNotExist) {
-		return MissingPath{}, fmt.Errorf("%s is there, or cannot be read: %v", path, err)
-	}
-	var levels []string
-	for p := path; p != "/"; p = filepath.Dir(p) {
-		levels = append([]string{filepath.Base(p)}, levels...)
-		parent := filepath.Dir(p)
-		info, err := os.Lstat(parent)
-		if errors.Is(err, os.ErrNotExist) {
-			continue
-		}
-		if err != nil {
-			return MissingPath{}, err
-		}
-		return MissingPath{Ancestor: parent, Levels: levels, Info: info}, nil
-	}
-	return MissingPath{}, errors.New("no part of the path is there")
-}
-
-// missingAccess is whether r's path is missing below an ancestor that is a directory: the access waits.
-func missingAccess(path string) (MissingPath, bool) {
-	m, err := NearestAncestor(filepath.Clean(path))
-	if err != nil || !m.Info.IsDir() {
-		return MissingPath{}, false
-	}
-	return m, true
-}
-
-// missingAccesses is every declared access whose path is missing below a directory, by path, with its id: what
-// no container mounts until it is made. A path the guard refuses is not read (issue 339).
-func missingAccesses(resources []declaration.Resource, refused map[string]error) map[string]string {
-	out := map[string]string{}
-	for _, r := range resources {
-		a, ok := r.(*declaration.Access)
-		if !ok || refused[filepath.Clean(a.Path)] != nil {
-			continue
-		}
-		if _, missing := missingAccess(a.Path); missing {
-			out[filepath.Clean(a.Path)] = a.ID
-		}
-	}
-	return out
-}
-
-// mountsMissing is the access a container waits for: one whose path it mounts, or mounts below, is missing.
-func mountsMissing(c *declaration.Container, missing map[string]string) string {
-	for _, v := range c.Volumes {
-		src := mountSource(v)
-		if !strings.HasPrefix(src, "/") {
-			continue
-		}
-		for path, id := range missing {
-			if atOrBelow(filepath.Clean(src), path) {
-				return id
-			}
-		}
-	}
-	return ""
-}
-
-// WaitingOnAccess is every container of the declaration that waits for a missing accessed path, read now: not
-// created by the apply, so not judged alive or dead and not fired on a schedule — the access says why.
-func WaitingOnAccess(d *declaration.Declaration) map[string]bool {
-	out := map[string]bool{}
-	if d == nil {
-		return out
-	}
-	missing := missingAccesses(d.Resources, refusedPlaces(d.Resources))
-	if len(missing) == 0 {
-		return out
-	}
-	for _, r := range d.Resources {
-		if c, ok := r.(*declaration.Container); ok && mountsMissing(c, missing) != "" {
-			out[c.ID] = true
-		}
-	}
-	return out
-}
-
-// MissingWords is the outcome of an access whose path is missing below a directory.
-func MissingWords(path string) string {
-	return fmt.Sprintf("%s is not there yet, below a folder that is: nothing mounts it and the containers that do are "+
-		"not created until it is made — on the operator's warrant, by the create act (novox/hq ADR 0324)", path)
-}

internal/apply/mode.go

@@ -3,7 +3,7 @@ package apply
 import (
 	"fmt"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/module_results.go

@@ -0,0 +1,99 @@
+package apply
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+)
+
+// failuresOf is every failure of an apply as the error carries it: the first among them, then the rest, so a module
+// whose resource failed second is still named (issue 1132, and issue 500, which counted the rest and named none).
+func failuresOf(failures []*Error) []Failure {
+	out := make([]Failure, 0, len(failures))
+	for _, f := range failures {
+		out = append(out, Failure{Resource: f.Resource, Err: f.Err})
+	}
+	return out
+}
+
+// ModuleResults groups an apply's outcome per module the declaration names (novox/hq ADR 0429 §4, issue 1132): the
+// module's resources all applied, or it failed and Detail says which resource failed with its error, which resources
+// were skipped behind a step that did not complete, or that its resources were not attempted because a step of no
+// module stopped the apply. A module is named by what the declaration says of each resource (ModuleOfID), so one
+// whose id would guess another is still its own.
+//
+// A failure that is no resource's (an error that is not an Error, or a resource of no module) belongs to no module and
+// changes none: the node says it in its report. The report is the outcome of an apply that did not fail; an apply that
+// failed carries its own account in the Error, and that is read in its place.
+func ModuleResults(d *declaration.Declaration, report Report, err error) map[string]ModuleResult {
+	if d == nil {
+		return nil
+	}
+	out := map[string]ModuleResult{}
+	attempted := map[string]bool{}
+	var failed *Error
+	errors.As(err, &failed)
+	outcomes := report.Outcomes
+	if failed != nil {
+		outcomes = append(append([]Outcome(nil), outcomes...), failed.Done.Outcomes...)
+	}
+	skipped := map[string]string{}
+	for _, o := range outcomes {
+		module, ok := d.ModuleOfID(o.ID)
+		if !ok {
+			continue
+		}
+		if o.Action == "skipped" {
+			skipped[module] = o.Detail
+			continue
+		}
+		attempted[o.ID] = true
+	}
+	var reasons = map[string][]string{}
+	for _, r := range d.Resources {
+		if module, ok := d.ModuleOfID(r.Identity()); ok {
+			out[module] = ModuleResult{}
+			if failed != nil && failed.Stopped && !attempted[r.Identity()] {
+				reasons[module] = append(reasons[module], fmt.Sprintf("%s was not attempted: %s stopped the apply", r.Identity(), failed.Resource))
+			}
+		}
+	}
+	if failed != nil {
+		for _, f := range failed.Failures {
+			if module, ok := d.ModuleOfID(f.Resource); ok {
+				if _, named := out[module]; named {
+					reasons[module] = append(reasons[module], fmt.Sprintf("%s failed: %v", f.Resource, f.Err))
+				}
+			}
+		}
+		if len(failed.Failures) == 0 {
+			if module, ok := d.ModuleOfID(failed.Resource); ok {
+				if _, named := out[module]; named {
+					reasons[module] = append(reasons[module], fmt.Sprintf("%s failed: %v", failed.Resource, failed.Err))
+				}
+			}
+		}
+		for _, module := range failed.Modules {
+			if _, named := out[module]; named {
+				why := skipped[module]
+				if why == "" {
+					why = "a step it declares did not complete"
+				}
+				reasons[module] = append(reasons[module], "gated: "+why)
+			}
+		}
+	}
+	for module, why := range skipped {
+		if _, named := out[module]; named && len(reasons[module]) == 0 {
+			reasons[module] = append(reasons[module], "gated: "+why)
+		}
+	}
+	for module, why := range reasons {
+		sort.Strings(why)
+		out[module] = ModuleResult{Failed: true, Detail: strings.Join(why, "; ")}
+	}
+	return out
+}

internal/apply/module_results_test.go

@@ -0,0 +1,76 @@
+package apply
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+)
+
+func twoModules(t *testing.T) *declaration.Declaration {
+	t.Helper()
+	d, err := declaration.Parse([]byte(`{"declaration":1,"builds":{"a":"b-1","b":"b-2"},"resources":[
+		{"id":"a.dir","type":"directory","path":"/srv/a","module":"a"},
+		{"id":"b.dir","type":"directory","path":"/srv/b","module":"b"},
+		{"id":"b.conf","type":"file","path":"/srv/b/c","content":"x","module":"b"},
+		{"id":"c.dir","type":"directory","path":"/srv/c","module":"c"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d
+}
+
+// 1132/AC3: every resource of module A applied and one of module B failed: A is a success, B failed with the failing
+// resource and its error named; a module gated by another's failure is failed too.
+func TestAnApplyIsGroupedPerModule(t *testing.T) {
+	d := twoModules(t)
+	done := Report{Outcomes: []Outcome{
+		{ID: "a.dir", Action: "created"},
+		{ID: "c.dir", Action: "skipped", Detail: "a step this module declares did not complete"},
+	}}
+	err := &Error{Resource: "b.dir", Err: errors.New("no space left"), Done: done, Others: 1,
+		Failures: []Failure{{Resource: "b.dir", Err: errors.New("no space left")}, {Resource: "b.conf", Err: errors.New("denied")}},
+		Modules:  []string{"c"}}
+	got := ModuleResults(d, Report{}, err)
+	if len(got) != 3 {
+		t.Fatalf("modules %v, want a, b and c", got)
+	}
+	if got["a"].Failed || got["a"].Detail != "" {
+		t.Errorf("a: %+v, want a clean success", got["a"])
+	}
+	b := got["b"]
+	if !b.Failed || !strings.Contains(b.Detail, "b.dir") || !strings.Contains(b.Detail, "no space left") ||
+		!strings.Contains(b.Detail, "b.conf") || !strings.Contains(b.Detail, "denied") {
+		t.Errorf("b: %+v, want both failing resources and their errors", b)
+	}
+	c := got["c"]
+	if !c.Failed || !strings.Contains(c.Detail, "did not complete") {
+		t.Errorf("c: %+v, want it failed as gated, with why", c)
+	}
+}
+
+// 1132/AC3: an apply that did not fail is a success for every module the declaration names.
+func TestACleanApplyIsASuccessForEveryModule(t *testing.T) {
+	got := ModuleResults(twoModules(t), Report{}, nil)
+	for _, m := range []string{"a", "b", "c"} {
+		if r, ok := got[m]; !ok || r.Failed {
+			t.Errorf("%s: %+v (%v), want a success", m, r, ok)
+		}
+	}
+}
+
+// 1132/AC3: a step that belongs to no module stops the apply there; a module with a resource not attempted is not a
+// success.
+func TestAModuleNotAttemptedIsNotASuccess(t *testing.T) {
+	d := twoModules(t)
+	err := &Error{Resource: "boot", Err: errors.New("boom"), Stopped: true, Failures: []Failure{{Resource: "boot", Err: errors.New("boom")}},
+		Done: Report{Outcomes: []Outcome{{ID: "a.dir", Action: "created"}}}}
+	got := ModuleResults(d, Report{}, err)
+	if got["a"].Failed {
+		t.Errorf("a was done: %+v", got["a"])
+	}
+	if !got["b"].Failed || !strings.Contains(got["b"].Detail, "boot") {
+		t.Errorf("b was not attempted: %+v", got["b"])
+	}
+}

internal/apply/move_out.go

@@ -0,0 +1,371 @@
+package apply
+
+// A directory or archive that says unknown "move-out" keeps only what it knows (issue 1024, novox/hq ADR 0422).
+//
+// Everything inside it that nothing declared, no archive unpacked and `knows` does not name is moved, whole, into
+// `<home>/.local/state/mesh-moved/<module>/<UTC time>/`, with a list beside it that names each move before it is
+// made (ADR 0417 §7). Nothing is deleted: the owner of the path reacts to a file it did not write by keeping it
+// where its writer can find it. The pass runs after the resources of an apply, so a reconcile (every five minutes)
+// runs it too.
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// movedOutWait is how old a path is before it is moved: a younger one may be mid-write by whatever made it, and
+// waits for the next look (ADR 0417 §4).
+const movedOutWait = time.Minute
+
+// MovedOut is one path a move-out directory moved.
+type MovedOut struct {
+	Resource string `json:"resource"`
+	From     string `json:"from"`
+	To       string `json:"to"`
+}
+
+// LeftInPlace is a socket, pipe or device a move-out directory did not move (ADR 0422 §4).
+type LeftInPlace struct {
+	Resource string `json:"resource"`
+	Path     string `json:"path"`
+}
+
+// renamePath and deviceOf are the machine's calls, variables so a test can fail a move or name another file system.
+// renamePath renames from to to without following a link in either parent below home (move_out_linux.go).
+var renamePath = renameUnder
+
+type movedEntry struct {
+	From string `json:"from"`
+	To   string `json:"to"`
+}
+
+// moveOutOwner is the user and group the kept folders and the list are given: the declared owner, or the home's
+// account.
+func moveOutOwner(home, owner string) (uid, gid int, err error) {
+	if owner != "" {
+		return idsOf(owner)
+	}
+	acc, ok := accountsOfHomes()[home]
+	if !ok {
+		return 0, 0, fmt.Errorf("%s is not the home of an account", home)
+	}
+	if u, g, err := idsOf(acc.Name); err == nil {
+		return u, g, nil
+	}
+	return acc.UID, acc.UID, nil
+}
+
+// knownUnder is what a directory knows of its own contents: the declared targets, the files an archive unpacked,
+// `knows` entries, and every folder above one of them.
+type knownUnder struct {
+	exact    map[string]bool
+	prefixes []string
+	// own are nested directories and archives that say what they do with unknown paths themselves.
+	own map[string]bool
+}
+
+func (k *knownUnder) add(root, path string) {
+	path = filepath.Clean(path)
+	for path != root && strings.HasPrefix(path, root+string(os.PathSeparator)) {
+		k.exact[path] = true
+		path = filepath.Dir(path)
+	}
+}
+
+func (k *knownUnder) knows(path string) bool {
+	if k.exact[path] {
+		return true
+	}
+	for _, p := range k.prefixes {
+		if path == p || strings.HasPrefix(path, p+string(os.PathSeparator)) {
+			return true
+		}
+	}
+	return false
+}
+
+func knownOf(d *declaration.Declaration, known store.State, id, root string, knows []string) *knownUnder {
+	k := &knownUnder{exact: map[string]bool{}, own: map[string]bool{}}
+	for _, r := range d.Resources {
+		target := filepath.Clean(r.Target())
+		if target == "" || target == "." {
+			continue
+		}
+		if r.Identity() != id && strings.HasPrefix(target, root+string(os.PathSeparator)) {
+			k.add(root, target)
+			switch o := r.(type) {
+			case *declaration.Directory:
+				if o.Unknown != "" {
+					k.own[target] = true
+				}
+			case *declaration.Archive:
+				if o.Unknown != "" {
+					k.own[target] = true
+				}
+			}
+		}
+		if a, ok := r.(*declaration.Archive); ok {
+			if rec, found := known.Find(a.ID); found && rec.Unpacked != nil {
+				base := filepath.Clean(a.Path)
+				for _, f := range append(append([]string{}, rec.Unpacked.Files...), rec.Unpacked.Dirs...) {
+					p := filepath.Join(base, filepath.FromSlash(f))
+					if p == root || strings.HasPrefix(p, root+string(os.PathSeparator)) {
+						k.add(root, p)
+					}
+				}
+			}
+		}
+	}
+	for _, e := range knows {
+		p := filepath.Join(root, filepath.FromSlash(strings.TrimSuffix(e, "/")))
+		if strings.HasSuffix(e, "/") {
+			k.prefixes = append(k.prefixes, p)
+		} else {
+			k.exact[p] = true
+		}
+		k.add(root, p)
+	}
+	return k
+}
+
+// moveOut moves what the directory at dir does not know, for the resource id. It answers the paths moved.
+func moveOut(d *declaration.Declaration, known store.State, id, module, dir, owner string, knows []string,
+	log func(string)) ([]MovedOut, []LeftInPlace, error) {
+	dir = filepath.Clean(dir)
+	home := homeAbove(dir)
+	if home == "" {
+		return nil, nil, fmt.Errorf("%s: move-out is only for a directory under an account's home, and %s is under none "+
+			"(ADR 0422 §5); nothing was moved", id, dir)
+	}
+	k := knownOf(d, known, id, dir, knows)
+	skip := filepath.Join(keptRoot(home), module)
+	var candidates []string
+	var left []string
+	err := filepath.WalkDir(dir, func(path string, e fs.DirEntry, err error) error {
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil
+			}
+			return err
+		}
+		if path == dir {
+			return nil
+		}
+		if path == skip || path == keptRoot(home) {
+			return fs.SkipDir
+		}
+		if k.own[path] {
+			if e.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		if k.knows(path) {
+			if e.IsDir() && !k.exact[path] {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		// Unknown. Moved whole, so never descended into.
+		info, err := e.Info()
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil
+			}
+			return err
+		}
+		switch {
+		case info.Mode()&(os.ModeSocket|os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice) != 0:
+			left = append(left, path)
+		case time.Since(info.ModTime()) < movedOutWait:
+		default:
+			candidates = append(candidates, path)
+		}
+		if e.IsDir() {
+			return fs.SkipDir
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("%s: cannot look through %s: %w", id, dir, err)
+	}
+	for _, p := range left {
+		log(fmt.Sprintf("  left %s in %s: a socket, pipe or device is not moved (ADR 0422)", p, dir))
+	}
+	var stayed []LeftInPlace
+	for _, p := range left {
+		stayed = append(stayed, LeftInPlace{Resource: id, Path: p})
+	}
+	if len(candidates) == 0 {
+		return nil, stayed, nil
+	}
+	sort.Strings(candidates)
+
+	uid, gid, err := moveOutOwner(home, owner)
+	if err != nil {
+		return nil, nil, fmt.Errorf("%s: %w", id, err)
+	}
+
+	// The same file system first, so a move is a rename and nothing is copied (ADR 0417 §5).
+	ancestor := skip
+	for {
+		if _, err := os.Lstat(ancestor); err == nil {
+			break
+		}
+		ancestor = filepath.Dir(ancestor)
+	}
+	devDir, err1 := deviceOf(dir)
+	devKept, err2 := deviceOf(ancestor)
+	if err1 != nil || err2 != nil {
+		return nil, nil, fmt.Errorf("%s: cannot read the file system of %s or %s: %v %v", id, dir, ancestor, err1, err2)
+	}
+	if devDir != devKept {
+		return nil, nil, fmt.Errorf("%s: %s and %s are on different file systems (devices %d and %d), so a move would "+
+			"copy; nothing was moved (ADR 0417 §5)", id, dir, ancestor, devDir, devKept)
+	}
+
+	// The kept folder and the list, named by the second; a clash in the same second gets -2, -3.
+	stamp := time.Now().UTC().Format("20060102T150405Z")
+	kept, listPath := "", ""
+	for n := 1; ; n++ {
+		name := stamp
+		if n > 1 {
+			name = fmt.Sprintf("%s-%d", stamp, n)
+		}
+		kept, listPath = filepath.Join(skip, name), filepath.Join(skip, name+".json")
+		if !pathExists(kept) && !pathExists(listPath) {
+			break
+		}
+	}
+	var made []string
+	cleanup := func() {
+		_ = os.Remove(listPath)
+		// In the reverse of the order they were made, so a folder goes after what is in it.
+		for i := len(made) - 1; i >= 0; i-- {
+			_ = os.Remove(made[i])
+		}
+	}
+	chown := func(paths []string) error {
+		for _, p := range paths {
+			if err := chownIfNeeded(home, p, uid, gid); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	madeNow, err := mkdirAllUnder(home, kept, 0o700)
+	made = append(made, reversed(madeNow)...)
+	if err == nil {
+		err = chown(madeNow)
+	}
+	if err != nil {
+		cleanup()
+		return nil, nil, fmt.Errorf("%s: cannot make the kept folder %s: %w", id, kept, err)
+	}
+
+	var list []movedEntry
+	var moved []MovedOut
+	var failures []string
+	for _, from := range candidates {
+		rel, _ := filepath.Rel(dir, from)
+		to := filepath.Join(kept, rel)
+		// Named before it is moved: a move that stops half way leaves its entry (ADR 0417 §7).
+		list = append(list, movedEntry{From: from, To: to})
+		raw, _ := json.MarshalIndent(list, "", "  ")
+		if err := writeUnder(home, listPath, append(raw, '\n'), 0o600); err != nil {
+			failures = append(failures, fmt.Sprintf("cannot write the list %s: %v", listPath, err))
+			break
+		}
+		if err := chownIfNeeded(home, listPath, uid, gid); err != nil {
+			failures = append(failures, err.Error())
+			break
+		}
+		parents, err := mkdirAllUnder(home, filepath.Dir(to), 0o700)
+		made = append(made, reversed(parents)...)
+		if err == nil {
+			err = chown(parents)
+		}
+		if err == nil {
+			err = renamePath(home, from, to)
+		}
+		if err != nil {
+			failures = append(failures, fmt.Sprintf("cannot move %s to %s: %v", from, to, err))
+			continue
+		}
+		log(fmt.Sprintf("  moved %s to %s (ADR 0422)", from, to))
+		moved = append(moved, MovedOut{Resource: id, From: from, To: to})
+	}
+	if len(moved) == 0 {
+		cleanup()
+	}
+	if len(failures) > 0 {
+		return moved, stayed, fmt.Errorf("%s: %s", id, strings.Join(failures, "; "))
+	}
+	return moved, stayed, nil
+}
+
+// reversed is paths shallowest first: mkdirAllUnder answers deepest first.
+func reversed(in []string) []string {
+	out := make([]string, len(in))
+	for i, p := range in {
+		out[len(in)-1-i] = p
+	}
+	return out
+}
+
+func keptRoot(home string) string { return filepath.Join(home, ".local", "state", "mesh-moved") }
+
+// chownIfNeeded gives a path to uid and gid through the home-safe call, unless it is theirs already.
+func chownIfNeeded(home, path string, uid, gid int) error {
+	if u, g, ok := ownerIDs(path); ok && u == uid && g == gid {
+		return nil
+	}
+	return chownUnder(home, path, uid, gid)
+}
+
+// moveOutPass runs move-out for every directory and archive of d that says so and applied in this run.
+func moveOutPass(d *declaration.Declaration, known store.State, applied map[string]bool, log func(string)) (
+	moved []MovedOut, left []LeftInPlace, failures []*Error) {
+	for _, r := range d.Resources {
+		var unknown, owner string
+		var knows []string
+		switch o := r.(type) {
+		case *declaration.Directory:
+			unknown, owner, knows = o.Unknown, o.Owner, o.Knows
+		case *declaration.Archive:
+			unknown, owner, knows = o.Unknown, o.Owner, o.Knows
+		default:
+			continue
+		}
+		if unknown != declaration.UnknownMoveOut || !applied[r.Identity()] {
+			continue
+		}
+		module, _ := d.ModuleOfID(r.Identity())
+		if module == "" {
+			module = r.Identity()
+		}
+		m, l, err := moveOut(d, known, r.Identity(), module, r.Target(), owner, knows, log)
+		moved = append(moved, m...)
+		left = append(left, l...)
+		if err != nil {
+			log(fmt.Sprintf("  failed %s (%s): %v", r.Identity(), r.Target(), err))
+			failures = append(failures, &Error{Resource: r.Identity(), Err: err})
+		}
+	}
+	return moved, left, failures
+}
+
+func pathExists(path string) bool {
+	_, err := os.Lstat(path)
+	return err == nil
+}

internal/apply/move_out_linux.go

@@ -0,0 +1,61 @@
+//go:build linux
+
+package apply
+
+import (
+	"fmt"
+	"path/filepath"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+// deviceOf is the device a path lives on; a variable so a test can name another file system. Below a home it is
+// read through a descriptor opened without following a link (issue 1024).
+var deviceOf = func(path string) (uint64, error) {
+	var st unix.Stat_t
+	if home := homeAbove(path); home != "" {
+		fd, err := openDirUnder(home, path)
+		if err != nil {
+			return 0, err
+		}
+		defer unix.Close(fd)
+		if err := unix.Fstat(fd, &st); err != nil {
+			return 0, err
+		}
+		return uint64(st.Dev), nil
+	}
+	if err := unix.Stat(path, &st); err != nil {
+		return 0, err
+	}
+	return uint64(st.Dev), nil
+}
+
+// renameUnder moves from to to, both below home, through descriptors of their parents opened without following a
+// link, and never over something that is there: an account that owns the folders cannot point either at another
+// place between the look and the move (issue 1024, novox/hq ADR 0266).
+func renameUnder(home, from, to string) error {
+	fromDir, err := openDirUnder(home, filepath.Dir(from))
+	if err != nil {
+		return err
+	}
+	defer unix.Close(fromDir)
+	toDir, err := openDirUnder(home, filepath.Dir(to))
+	if err != nil {
+		return err
+	}
+	defer unix.Close(toDir)
+	if err := unix.Renameat2(fromDir, filepath.Base(from), toDir, filepath.Base(to), unix.RENAME_NOREPLACE); err != nil {
+		return fmt.Errorf("rename %s to %s: %w", from, to, err)
+	}
+	return nil
+}
+
+// ownerIDs is the user and group of a path, without following a link.
+func ownerIDs(path string) (uid, gid int, ok bool) {
+	var st syscall.Stat_t
+	if err := syscall.Lstat(path, &st); err != nil {
+		return 0, 0, false
+	}
+	return int(st.Uid), int(st.Gid), true
+}

internal/apply/move_out_other.go

@@ -0,0 +1,19 @@
+//go:build !linux
+
+package apply
+
+import "os"
+
+var deviceOf = func(string) (uint64, error) { return 0, nil }
+
+func ownerIDs(string) (int, int, bool) { return 0, 0, false }
+
+func renameUnder(home, from, to string) error {
+	if err := refuseLinksUnderHome(from); err != nil {
+		return err
+	}
+	if err := refuseLinksUnderHome(to); err != nil {
+		return err
+	}
+	return os.Rename(from, to)
+}

internal/apply/move_out_test.go

@@ -0,0 +1,355 @@
+package apply
+
+// Issue 1024, novox/hq ADR 0422: a directory that says unknown "move-out" keeps only what it knows, and moves every
+// other path into a kept folder under the account's home, listed before it is moved.
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	osuser "os/user"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-host/internal/store"
+	"golang.org/x/sys/unix"
+)
+
+// me is the account running the test, which owns the home the test names.
+func me() string {
+	u, err := osuser.Current()
+	if err != nil {
+		panic(err)
+	}
+	return u.Username
+}
+
+// aMoveOutHome is a home the test names, of the account running the test, so no chown needs privilege.
+func aMoveOutHome(t *testing.T) string {
+	t.Helper()
+	home := filepath.Join(t.TempDir(), "home", "agent")
+	if err := os.MkdirAll(home, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	wasHomes, wasAccounts := homes, accountsOfHomes
+	homes = func() []string { return []string{home} }
+	accountsOfHomes = func() map[string]homeAccount {
+		return map[string]homeAccount{home: {Name: me(), UID: os.Getuid()}}
+	}
+	t.Cleanup(func() { homes, accountsOfHomes = wasHomes, wasAccounts })
+	return home
+}
+
+func put(t *testing.T, path, content string, age time.Duration) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	old := time.Now().Add(-age)
+	if err := os.Chtimes(path, old, old); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func keptLists(t *testing.T, home string) []string {
+	t.Helper()
+	out, _ := filepath.Glob(filepath.Join(keptRoot(home), "*", "*.json"))
+	return out
+}
+
+func readList(t *testing.T, path string) []movedEntry {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var list []movedEntry
+	if err := json.Unmarshal(raw, &list); err != nil {
+		t.Fatal(err)
+	}
+	return list
+}
+
+func TestAMoveOutDirectoryMovesOnlyWhatIsNotKnown(t *testing.T) {
+	home := aMoveOutHome(t)
+	dir := filepath.Join(home, ".ssh")
+	outside := filepath.Join(t.TempDir(), "target")
+	put(t, outside, "elsewhere", time.Hour)
+	put(t, filepath.Join(dir, "config"), "declared", time.Hour)
+	put(t, filepath.Join(dir, "known_hosts"), "knows", time.Hour)
+	put(t, filepath.Join(dir, "x", "inside"), "in x", time.Hour)
+	put(t, filepath.Join(dir, "stray"), "stray content", time.Hour)
+	put(t, filepath.Join(dir, "oldfolder", "deep", "f"), "deep", time.Hour)
+	put(t, filepath.Join(dir, "young"), "just made", 5*time.Second)
+	if err := os.Symlink(outside, filepath.Join(dir, "alink")); err != nil {
+		t.Fatal(err)
+	}
+	old := time.Now().Add(-time.Hour)
+	_ = os.Chtimes(filepath.Join(dir, "oldfolder"), old, old)
+	_ = os.Chtimes(filepath.Join(dir, "x"), old, old)
+	// A named pipe stands for the socket: a socket's path is too long under a test directory to bind.
+	if err := syscall.Mkfifo(filepath.Join(dir, "agent.sock"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	_ = os.Chtimes(filepath.Join(dir, "agent.sock"), old, old)
+	// The directory itself and its declared file are old enough too.
+	// The symbolic link is older than a minute as well.
+	ts := []unix.Timespec{unix.NsecToTimespec(old.UnixNano()), unix.NsecToTimespec(old.UnixNano())}
+	if err := unix.UtimesNanoAt(unix.AT_FDCWD, filepath.Join(dir, "alink"), ts, unix.AT_SYMLINK_NOFOLLOW); err != nil {
+		t.Fatal(err)
+	}
+
+	var logged []string
+	report, _, err := Apply(context.Background(), archHost(t), declare(t,
+		`{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out","knows":["known_hosts","x/"]},`+
+			`{"id":"mod.config","type":"file","path":"`+dir+`/config","content":"declared"}`),
+		store.State{}, store.OriginDeclared, noServices, func(s string) { logged = append(logged, s) }, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = report
+	for _, kept := range []string{"config", "known_hosts", "x/inside", "young", "agent.sock"} {
+		if !exists(filepath.Join(dir, kept)) {
+			t.Errorf("%s should stay", kept)
+		}
+	}
+	for _, gone := range []string{"stray", "oldfolder", "alink"} {
+		if exists(filepath.Join(dir, gone)) {
+			t.Errorf("%s should have moved", gone)
+		}
+	}
+	lists := keptLists(t, home)
+	if len(lists) != 1 {
+		t.Fatalf("one list expected, got %v", lists)
+	}
+	if !strings.Contains(lists[0], filepath.Join("mesh-moved", "mod")) {
+		t.Errorf("the module's name is not in the kept path: %s", lists[0])
+	}
+	list := readList(t, lists[0])
+	if len(list) != 3 {
+		t.Fatalf("the list names three paths: %+v", list)
+	}
+	kept := strings.TrimSuffix(lists[0], ".json")
+	for _, e := range list {
+		if !strings.HasPrefix(e.To, kept+string(os.PathSeparator)) || !exists(e.To) {
+			t.Errorf("not at %s: %+v", kept, e)
+		}
+	}
+	if b, _ := os.ReadFile(filepath.Join(kept, "stray")); string(b) != "stray content" {
+		t.Errorf("content changed: %q", b)
+	}
+	if b, _ := os.ReadFile(filepath.Join(kept, "oldfolder", "deep", "f")); string(b) != "deep" {
+		t.Errorf("the folder was not moved whole: %q", b)
+	}
+	if info, err := os.Lstat(filepath.Join(kept, "alink")); err != nil || info.Mode()&os.ModeSymlink == 0 {
+		t.Errorf("the link is not still a link: %v", err)
+	}
+	if !strings.Contains(strings.Join(logged, "\n"), "(ADR 0422)") {
+		t.Errorf("the moves were not logged: %v", logged)
+	}
+	if len(report.Left) != 1 || filepath.Base(report.Left[0].Path) != "agent.sock" {
+		t.Errorf("the report does not list the pipe it left: %+v", report.Left)
+	}
+	if len(report.MovedOut) != 3 {
+		t.Errorf("the report does not count the moves: %+v", report.MovedOut)
+	}
+}
+
+func TestAMoveOutListIsWrittenBeforeTheMove(t *testing.T) {
+	home := aMoveOutHome(t)
+	dir := filepath.Join(home, "d")
+	put(t, filepath.Join(dir, "a"), "a", time.Hour)
+	put(t, filepath.Join(dir, "b"), "b", time.Hour)
+	was := renamePath
+	renamePath = func(h, from, to string) error {
+		if filepath.Base(from) == "b" {
+			return errors.New("injected")
+		}
+		return was(h, from, to)
+	}
+	t.Cleanup(func() { renamePath = was })
+	_, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out"}`, nil, store.State{})
+	if err == nil {
+		t.Fatal("a failed move must fail the resource")
+	}
+	if !exists(filepath.Join(dir, "b")) || exists(filepath.Join(dir, "a")) {
+		t.Error("b stays, a moved")
+	}
+	lists := keptLists(t, home)
+	if len(lists) != 1 {
+		t.Fatalf("lists: %v", lists)
+	}
+	names := map[string]bool{}
+	for _, e := range readList(t, lists[0]) {
+		names[filepath.Base(e.From)] = true
+	}
+	if !names["a"] || !names["b"] {
+		t.Errorf("the entry for the failed move was not listed first: %v", names)
+	}
+}
+
+func TestAMoveOutThatMovedNothingMakesNothing(t *testing.T) {
+	home := aMoveOutHome(t)
+	dir := filepath.Join(home, "d")
+	put(t, filepath.Join(dir, "known"), "k", time.Hour)
+	put(t, filepath.Join(dir, "young"), "y", time.Second)
+	if _, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out","knows":["known"]}`, nil, store.State{}); err != nil {
+		t.Fatal(err)
+	}
+	if exists(keptRoot(home)) {
+		t.Error("a kept folder was made with nothing to move")
+	}
+	// And when every move fails, what was made goes.
+	put(t, filepath.Join(dir, "stray"), "s", time.Hour)
+	was := renamePath
+	renamePath = func(string, string, string) error { return errors.New("injected") }
+	t.Cleanup(func() { renamePath = was })
+	if _, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out","knows":["known"]}`, nil, store.State{}); err == nil {
+		t.Fatal("expected a failure")
+	}
+	if len(keptLists(t, home)) != 0 {
+		t.Error("a list stayed though nothing moved")
+	}
+	if matches, _ := filepath.Glob(filepath.Join(keptRoot(home), "*", "*")); len(matches) != 0 {
+		t.Errorf("a kept folder stayed though nothing moved: %v", matches)
+	}
+}
+
+func TestAMoveOutOfANestedStrayThatFailsLeavesNoKeptFolder(t *testing.T) {
+	home := aMoveOutHome(t)
+	dir := filepath.Join(home, "d")
+	put(t, filepath.Join(dir, "x", "f"), "declared", time.Hour)
+	put(t, filepath.Join(dir, "x", "stray"), "s", time.Hour)
+	was := renamePath
+	renamePath = func(string, string, string) error { return errors.New("injected") }
+	t.Cleanup(func() { renamePath = was })
+	_, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out"},`+
+		`{"id":"mod.f","type":"file","path":"`+dir+`/x/f","content":"declared","owner":"`+me()+`"}`, nil, store.State{})
+	if err == nil {
+		t.Fatal("expected a failure")
+	}
+	if exists(keptRoot(home)) {
+		matches, _ := filepath.Glob(filepath.Join(keptRoot(home), "*", "*"))
+		t.Errorf("something was left behind though nothing moved: %v", matches)
+	}
+}
+
+// A link swapped into a parent folder, on either side, between the look and the move, is not followed.
+func TestAMoveOutDoesNotFollowALinkSwappedIntoAParent(t *testing.T) {
+	for _, side := range []string{"from", "to"} {
+		t.Run(side, func(t *testing.T) {
+			home := aMoveOutHome(t)
+			dir := filepath.Join(home, "d")
+			outside := filepath.Join(t.TempDir(), "etc")
+			put(t, filepath.Join(outside, "stray"), "root's", time.Hour)
+			put(t, filepath.Join(dir, "x", "f"), "declared", time.Hour)
+			put(t, filepath.Join(dir, "x", "stray"), "s", time.Hour)
+			was := renamePath
+			renamePath = func(h, from, to string) error {
+				victim := filepath.Dir(from)
+				if side == "to" {
+					victim = filepath.Dir(to)
+					_ = os.Remove(filepath.Join(victim, "nothing"))
+				}
+				if err := os.RemoveAll(victim); err != nil {
+					t.Fatal(err)
+				}
+				if side == "from" {
+					put(t, filepath.Join(outside, "other"), "o", time.Hour)
+				}
+				if err := os.Symlink(outside, victim); err != nil {
+					t.Fatal(err)
+				}
+				return was(h, from, to)
+			}
+			t.Cleanup(func() { renamePath = was })
+			_, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out"},`+
+				`{"id":"mod.f","type":"file","path":"`+dir+`/x/f","content":"declared","owner":"`+me()+`"}`, nil, store.State{})
+			if err == nil {
+				t.Fatal("a move through a swapped link must fail")
+			}
+			if b, e := os.ReadFile(filepath.Join(outside, "stray")); e != nil || string(b) != "root's" {
+				t.Errorf("the file outside the home was touched: %q %v", b, e)
+			}
+			entries, _ := os.ReadDir(outside)
+			if side == "to" && len(entries) != 1 || side == "from" && len(entries) != 2 {
+				t.Errorf("something was moved into or out of the folder outside the home: %v", entries)
+			}
+		})
+	}
+}
+
+func TestOtherUnknownValuesAndNoValueMoveNothing(t *testing.T) {
+	home := aMoveOutHome(t)
+	for _, v := range []string{`,"unknown":"program"`, `,"unknown":"module"`, ``} {
+		dir := filepath.Join(home, "d")
+		put(t, filepath.Join(dir, "stray"), "s", time.Hour)
+		if _, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`"`+v+`}`, nil, store.State{}); err != nil {
+			t.Fatal(err)
+		}
+		if !exists(filepath.Join(dir, "stray")) || exists(keptRoot(home)) {
+			t.Errorf("%q moved something", v)
+		}
+	}
+}
+
+func TestAMoveOutArchiveKeepsWhatItUnpackedAndMovesAStray(t *testing.T) {
+	home := aMoveOutHome(t)
+	dir := filepath.Join(home, "theme")
+	put(t, filepath.Join(dir, "stray"), "s", time.Hour)
+	res, _ := archiveDecl(t, "mod.theme", dir, map[string]string{"one.txt": "1", "sub/two.txt": "2"}, "")
+	res = strings.Replace(res, `"path":`, `"owner":"`+me()+`","unknown":"move-out","path":`, 1)
+	if _, _, err := Apply(context.Background(), archHost(t), declare(t, res), store.State{},
+		store.OriginDeclared, noServices, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	for _, f := range []string{"one.txt", "sub/two.txt"} {
+		if !exists(filepath.Join(dir, f)) {
+			t.Errorf("%s was moved", f)
+		}
+	}
+	if exists(filepath.Join(dir, "stray")) || len(keptLists(t, home)) != 1 {
+		t.Error("the stray file did not move")
+	}
+}
+
+func TestMoveOutNeedsAHome(t *testing.T) {
+	aMoveOutHome(t)
+	dir := filepath.Join(t.TempDir(), "elsewhere")
+	put(t, filepath.Join(dir, "stray"), "s", time.Hour)
+	_, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out"}`, nil, store.State{})
+	if err == nil || !strings.Contains(err.Error(), "home") {
+		t.Fatalf("expected a refusal naming the home: %v", err)
+	}
+	if !exists(filepath.Join(dir, "stray")) {
+		t.Error("something moved")
+	}
+}
+
+func TestMoveOutRefusesAnotherFileSystem(t *testing.T) {
+	home := aMoveOutHome(t)
+	dir := filepath.Join(home, "d")
+	put(t, filepath.Join(dir, "stray"), "s", time.Hour)
+	was := deviceOf
+	deviceOf = func(p string) (uint64, error) {
+		if p == dir {
+			return 1, nil
+		}
+		return 2, nil
+	}
+	t.Cleanup(func() { deviceOf = was })
+	_, _, err := applyOnce(t, `{"id":"mod.dir","type":"directory","path":"`+dir+`","owner":"`+me()+`","unknown":"move-out"}`, nil, store.State{})
+	if err == nil || !strings.Contains(err.Error(), "file system") {
+		t.Fatalf("expected a refusal naming the file systems: %v", err)
+	}
+	if !exists(filepath.Join(dir, "stray")) || exists(keptRoot(home)) {
+		t.Error("something moved or was made")
+	}
+}

internal/apply/opening.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/firewall"
 	"github.com/novox/mesh-host/internal/store"
 )

internal/apply/opening_test.go

@@ -10,7 +10,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/ownership_test.go

@@ -0,0 +1,129 @@
+package apply
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// Defends novox/hq ADR 0349 §5 (issue 667): a file that says who owns it at runtime is written as the
+// form it names always was — owned as a whole file whose drift is written back, seed as create-once,
+// managed as only its keys or its marked block. Each is applied beside its old form, through the same
+// drift, and the two must end the same: the same actions, the same bytes on disk.
+
+// ownershipCase is one ownership word and the old form it maps onto.
+type ownershipCase struct {
+	name string
+	// said and old are the resource's fields after id, type and path: with the word, and in the old form.
+	said, old string
+	// before is what is at the path before the first apply; "" leaves the path absent.
+	before string
+	// drift is what the machine writes there between the two applies.
+	drift string
+	// first and second are the actions the old form is known to report.
+	first, second string
+}
+
+func TestEachOwnershipBehavesAsItsOldForm(t *testing.T) {
+	for _, c := range []ownershipCase{
+		{
+			name:  "owned is a whole file whose drift is written back",
+			said:  `"content":"correct\n","mode":"0644","owner-at-runtime":"owned"`,
+			old:   `"content":"correct\n","mode":"0644"`,
+			drift: "someone edited this\n",
+			first: "created", second: "corrected",
+		},
+		{
+			name:  "seed is written once and then the program's",
+			said:  `"content":"user default on\n","mode":"0600","owner-at-runtime":"seed"`,
+			old:   `"content":"user default on\n","mode":"0600","create-once":true`,
+			drift: "user default on\nuser app-one on\n",
+			first: "created", second: "kept",
+		},
+		{
+			name:   "managed json sets only its keys",
+			said:   `"into":"json","content":"{\"insecure-registries\":[\"10.42.0.1:5000\"]}","owner-at-runtime":"managed"`,
+			old:    `"into":"json","content":"{\"insecure-registries\":[\"10.42.0.1:5000\"]}"`,
+			before: machinesOwn,
+			drift:  `{"data-root":"/srv/other","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}`,
+			first:  "updated", second: "corrected",
+		},
+		{
+			name:   "managed block holds only its region",
+			said:   `"into":"block","content":"10.42.0.1 ace\n","owner-at-runtime":"managed"`,
+			old:    `"into":"block","content":"10.42.0.1 ace\n"`,
+			before: workstationHosts,
+			drift:  workstationHosts + "\n" + marked("f", "6.6.6.6 evil\n") + "192.0.2.9 the-machines-own\n",
+			first:  "updated", second: "corrected",
+		},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			saidActions, saidBytes := applyThroughDrift(t, c, c.said)
+			oldActions, oldBytes := applyThroughDrift(t, c, c.old)
+			if saidActions != oldActions {
+				t.Errorf("with the word the actions were %s; the old form's were %s", saidActions, oldActions)
+			}
+			if want := c.first + " " + c.second; oldActions != want {
+				t.Errorf("the old form's actions were %s; known to be %s", oldActions, want)
+			}
+			if saidBytes != oldBytes {
+				t.Errorf("with the word the file ends\n%q\nthe old form's ends\n%q", saidBytes, oldBytes)
+			}
+		})
+	}
+}
+
+// applyThroughDrift applies one file, lets the machine write the case's drift into it, applies again,
+// and says the two actions and what the file holds at the end.
+func applyThroughDrift(t *testing.T, c ownershipCase, fields string) (string, string) {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "the-file")
+	if c.before != "" {
+		if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"f","type":"file","path":%q,%s}]}`, path, fields))
+	first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, []byte(c.drift), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return first.Outcomes[0].Action + " " + second.Outcomes[0].Action, readText(t, path)
+}
+
+// A seed said by the word alone, with no create-once beside it, is still kept once present: the word is
+// enough, and the file the program grew is not written back.
+func TestASeedByTheWordAloneKeepsWhatGrew(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "acl.conf")
+	d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
+	  {"id":"acl","type":"file","path":%q,"content":"user default on\n","owner-at-runtime":"seed"}
+	]}`, path))
+	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, []byte("grown\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := report.Outcomes[0].Action; got != "kept" {
+		t.Errorf("a seed was %q", got)
+	}
+	if got := readText(t, path); got != "grown\n" {
+		t.Errorf("what grew in the seed was written over: %q", got)
+	}
+}

internal/apply/placed.go

@@ -0,0 +1,261 @@
+package apply
+
+import (
+	"bufio"
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Whether a library's disk is there (novox/hq ADR 0328 §9; first ADR 0325 §9).
+//
+// **The rule the engine keeps.** A folder is placed on the filesystem it is first seen on: the mount point that
+// holds it — the longest mount point in /proc/self/mountinfo that is the folder or above it — is recorded beside the
+// kept folder. On every later apply the folder (or, when it is absent, its nearest existing parent) is judged
+// against that record: held by another mount point than the one recorded while the recorded one is not mounted is
+// the disk not there. A recorded mount point that is mounted while another holds the folder is a **move**: what is
+// at the path now is not what was kept (a disk mounted over a folder first seen empty below it, a mount made
+// deeper), so the folder is judged afresh — kept again only if it holds nothing, else it waits for the operator's
+// word as any folder holding data does (ADR 0328 §7).
+//
+// **On first sight, with no record,** the engine knows a disk is missing only from where disks are meant to be:
+// every mount target in /etc/fstab, every `Where=` of a systemd .mount unit, every mount point a ZFS dataset names
+// (`zfs list`, when the machine has zfs), every mount point this engine has seen mounted since it started, and every
+// mount point recorded for another kept folder. A target that is the folder or above it, lies below the mount point
+// that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that holds what
+// is there is the placed one. A mount nothing names in those places (a pool not imported, whose datasets `zfs list`
+// does not show; a hand `mount` never seen) is not known to be missing on first sight: then a folder first seen
+// empty below it is kept as empty, and the move above takes it back the moment the disk is there.
+
+// MountPoints is every mount point this machine has mounted now, from /proc/self/mountinfo. A variable so a test
+// can stand in for mounts it cannot make.
+var MountPoints = func() []string {
+	return readMountPoints()
+}
+
+// mountHistory is every mount point this engine has seen mounted since it started (§9's "mountinfo history").
+var mountHistory = struct {
+	sync.Mutex
+	seen map[string]bool
+}{seen: map[string]bool{}}
+
+// sawMounted adds points to the history, and is the history: every mount point seen, never "/".
+func sawMounted(points []string) []string {
+	mountHistory.Lock()
+	defer mountHistory.Unlock()
+	for _, p := range points {
+		if p != "/" {
+			mountHistory.seen[p] = true
+		}
+	}
+	out := make([]string, 0, len(mountHistory.seen))
+	for p := range mountHistory.seen {
+		out = append(out, p)
+	}
+	return out
+}
+
+func readMountPoints() []string {
+	raw, err := os.ReadFile("/proc/self/mountinfo")
+	if err != nil {
+		return []string{"/"}
+	}
+	return mountinfoPoints(string(raw))
+}
+
+// MountTargets is every place a disk is meant to be mounted: /etc/fstab's targets and ZFS datasets' mount points,
+// read at most once a minute — every look judges every access by them. A variable so a test can stand in.
+var MountTargets = func(ctx context.Context) []string {
+	targets.Lock()
+	defer targets.Unlock()
+	if targets.read.IsZero() || time.Since(targets.read) > time.Minute {
+		targets.list, targets.read = readMountTargets(ctx), time.Now()
+	}
+	return append([]string(nil), targets.list...)
+}
+
+var targets struct {
+	sync.Mutex
+	list []string
+	read time.Time
+}
+
+func readMountTargets(ctx context.Context) []string {
+	var out []string
+	if raw, err := os.ReadFile("/etc/fstab"); err == nil {
+		out = append(out, fstabTargets(string(raw))...)
+	}
+	out = append(out, mountUnitTargets(systemdUnitDirs)...)
+	if _, err := exec.LookPath("zfs"); err == nil {
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		if raw, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "mountpoint").Output(); err == nil {
+			for _, line := range strings.Split(string(raw), "\n") {
+				if p := strings.TrimSpace(line); strings.HasPrefix(p, "/") {
+					out = append(out, filepath.Clean(p))
+				}
+			}
+		}
+	}
+	return out
+}
+
+// systemdUnitDirs are where systemd keeps the units a machine mounts by.
+var systemdUnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/run/systemd/generator",
+	"/usr/lib/systemd/system", "/lib/systemd/system"}
+
+// mountUnitTargets reads the `Where=` of every .mount unit in dirs.
+func mountUnitTargets(dirs []string) []string {
+	var out []string
+	for _, dir := range dirs {
+		names, err := filepath.Glob(filepath.Join(dir, "*.mount"))
+		if err != nil {
+			continue
+		}
+		for _, name := range names {
+			raw, err := os.ReadFile(name)
+			if err != nil {
+				continue
+			}
+			for _, line := range strings.Split(string(raw), "\n") {
+				if v, ok := strings.CutPrefix(strings.TrimSpace(line), "Where="); ok && strings.HasPrefix(v, "/") {
+					out = append(out, filepath.Clean(strings.TrimSpace(v)))
+				}
+			}
+		}
+	}
+	return out
+}
+
+// mountinfoPoints reads the mount point of each line of mountinfo, its fifth field, with the kernel's octal escapes
+// undone.
+func mountinfoPoints(raw string) []string {
+	var out []string
+	for _, line := range strings.Split(raw, "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 5 {
+			continue
+		}
+		out = append(out, filepath.Clean(unescapeMount(fields[4])))
+	}
+	return out
+}
+
+// mountSourceOf is the source and filesystem type of the mount at point, from mountinfo: for a ZFS dataset, its
+// name. Empty when the machine says nothing of it.
+func mountSourceOf(point string) (source, fstype string) { return MountSource(point) }
+
+// MountSource is mountSourceOf as the machine answers it. A variable so a test can stand in for another filesystem
+// mounted at the same point.
+var MountSource = func(point string) (source, fstype string) {
+	raw, err := os.ReadFile("/proc/self/mountinfo")
+	if err != nil {
+		return "", ""
+	}
+	for _, line := range strings.Split(string(raw), "\n") {
+		before, after, ok := strings.Cut(line, " - ")
+		fields, rest := strings.Fields(before), strings.Fields(after)
+		if !ok || len(fields) < 5 || len(rest) < 2 {
+			continue
+		}
+		if filepath.Clean(unescapeMount(fields[4])) == point {
+			source, fstype = unescapeMount(rest[1]), rest[0]
+		}
+	}
+	return source, fstype
+}
+
+// fstabTargets reads the second field of each fstab line that mounts somewhere: not swap, not a comment.
+func fstabTargets(raw string) []string {
+	var out []string
+	sc := bufio.NewScanner(strings.NewReader(raw))
+	for sc.Scan() {
+		fields := strings.Fields(sc.Text())
+		if len(fields) < 2 || strings.HasPrefix(fields[0], "#") || !strings.HasPrefix(fields[1], "/") {
+			continue
+		}
+		out = append(out, filepath.Clean(unescapeMount(fields[1])))
+	}
+	return out
+}
+
+// unescapeMount undoes the kernel's and fstab's octal escapes: \040 is a space.
+func unescapeMount(s string) string {
+	if !strings.Contains(s, `\`) {
+		return s
+	}
+	var b strings.Builder
+	for i := 0; i < len(s); i++ {
+		if s[i] == '\\' && i+4 <= len(s) {
+			if v, ok := octal3(s[i+1 : i+4]); ok {
+				b.WriteByte(v)
+				i += 3
+				continue
+			}
+		}
+		b.WriteByte(s[i])
+	}
+	return b.String()
+}
+
+func octal3(s string) (byte, bool) {
+	if len(s) != 3 {
+		return 0, false
+	}
+	v := 0
+	for _, c := range s {
+		if c < '0' || c > '7' {
+			return 0, false
+		}
+		v = v*8 + int(c-'0')
+	}
+	return byte(v), v < 256
+}
+
+// mountHolding is the longest of points that is path or above it: the mount point that holds it.
+func mountHolding(path string, points []string) string {
+	best := "/"
+	for _, p := range points {
+		if atOrBelow(path, p) && len(p) > len(best) {
+			best = p
+		}
+	}
+	return best
+}
+
+// placement is the judgement of where a folder's data lives.
+type placement struct {
+	// Mount is the mount point that holds what is there: the folder, or its nearest existing parent.
+	Mount string
+	// Away is that the disk the folder was placed on is not there.
+	Away bool
+	// Moved is that the recorded mount point is mounted and another holds the folder: what is there is not what
+	// was kept.
+	Moved bool
+}
+
+// placedOn judges path, of which existing is what is there (the folder itself, or its nearest existing parent),
+// against what was recorded and where disks are meant to be.
+func placedOn(ctx context.Context, path, existing, recorded string, others []string) placement {
+	points := MountPoints()
+	mounted := map[string]bool{}
+	for _, p := range points {
+		mounted[p] = true
+	}
+	have := mountHolding(existing, points)
+	seen := sawMounted(points)
+	if recorded != "" {
+		return placement{Mount: have, Away: have != recorded && !mounted[recorded],
+			Moved: have != recorded && mounted[recorded]}
+	}
+	for _, t := range append(append(MountTargets(ctx), others...), seen...) {
+		if atOrBelow(path, t) && t != have && atOrBelow(t, have) && !mounted[t] {
+			return placement{Mount: have, Away: true}
+		}
+	}
+	return placement{Mount: have}
+}

internal/apply/placement_guard.go

@@ -56,7 +56,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 )
 
 // protectedRoots are paths no directory, access or mount source may be, nor hold.
@@ -384,6 +384,9 @@ func ownedByAccount(owner string, a homeAccount) bool {
 	if owner == a.Name {
 		return true
 	}
+	if a.UID < 0 {
+		return false // an account the declaration makes has no uid yet: by name alone, never by a number
+	}
 	user, _, _ := strings.Cut(owner, ":")
 	if uid, err := strconv.Atoi(user); err == nil {
 		return uid == a.UID
@@ -413,12 +416,91 @@ const (
 // refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module,
 // owner a directory's declared owner.
 func refusePath(path string, kind placing, module, owner string) error {
+	return refusePathWith(path, kind, module, owner, nil)
+}
+
+// declaredHomes is the home of each account the declaration itself makes (a user resource with a home), by the
+// account's name; the account is not in the user database yet when the declaration is judged, and the engine makes
+// it where it is declared, before the directory that is owned by it is reached (novox/hq issue 1542). Its uid is
+// not known, so a directory is owned by it by name alone. Only for the directory placement guard; no other
+// reader of accountsOfHomes sees these.
+func declaredHomes(resources []declaration.Resource) map[string]homeAccount {
+	var accounts []passwdEntry
+	var out map[string]homeAccount
+	for _, r := range resources {
+		u, ok := r.(*declaration.User)
+		if !ok || u.Name == "" {
+			continue
+		}
+		home := filepath.Clean(u.Home)
+		if u.Home == "" || !filepath.IsAbs(home) || home == "/" {
+			continue
+		}
+		if accounts == nil {
+			accounts = passwdEntries()
+		}
+		// Only ever adds a home. An account the machine has keeps the home it has (applyUser never moves it), so
+		// a declared home of one is not its home, unless it is the very home it has: the account an earlier apply
+		// made, as declared, which must be judged the same at every later apply. And a home at or below another
+		// account's, or at or above a protected root, is not a new one.
+		taken := false
+		for _, e := range accounts {
+			if e.name == u.Name && e.home != home {
+				taken = true
+			}
+			if e.name != u.Name && e.home != "" && atOrBelow(home, e.home) {
+				taken = true
+			}
+		}
+		for _, root := range protectedRoots {
+			taken = taken || home == root || below(root, home)
+		}
+		if taken {
+			continue
+		}
+		if out == nil {
+			out = map[string]homeAccount{}
+		}
+		out[home] = homeAccount{Name: u.Name, UID: -1}
+	}
+	return out
+}
+
+// passwdEntry is an account of the user database and its home, whatever its uid.
+type passwdEntry struct{ name, home string }
+
+// passwdEntries is every account of the user database with its home (empty and root homes as "").
+func passwdEntries() []passwdEntry {
+	out := []passwdEntry{}
+	f, err := os.Open(passwdFile)
+	if err != nil {
+		return out
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		fields := strings.Split(sc.Text(), ":")
+		if len(fields) < 6 {
+			continue
+		}
+		home := filepath.Clean(fields[5])
+		if home == "." || home == "/" || home == "/nonexistent" {
+			home = ""
+		}
+		out = append(out, passwdEntry{fields[0], home})
+	}
+	return out
+}
+
+// refusePathWith is refusePath with the homes of the accounts the declaration makes (declaredHomes) counted
+// among the machine's.
+func refusePathWith(path string, kind placing, module, owner string, declared map[string]homeAccount) error {
 	clean := filepath.Clean(path)
 	if !filepath.IsAbs(clean) {
 		return nil // the declaration refuses a relative path already; a named volume is not a path
 	}
 	for _, p := range []string{clean, resolved(clean)} {
-		if err := refuseOne(p, kind, module, owner); err != nil {
+		if err := refuseOne(p, kind, module, owner, declared); err != nil {
 			if p != clean {
 				err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why)
 				err.Path = clean
@@ -429,7 +511,7 @@ func refusePath(path string, kind placing, module, owner string) error {
 	return nil
 }
 
-func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError {
+func refuseOne(path string, kind placing, module, owner string, declared map[string]homeAccount) *PlacementRefusedError {
 	for _, root := range protectedRoots {
 		if path == root || below(root, path) {
 			return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " +
@@ -482,6 +564,18 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
 		return nil
 	}
 	homes := accountsOfHomes()
+	if ssh && len(declared) > 0 {
+		merged := map[string]homeAccount{}
+		for home, a := range homes {
+			merged[home] = a
+		}
+		for home, a := range declared {
+			if _, held := merged[home]; !held {
+				merged[home] = a
+			}
+		}
+		homes = merged
+	}
 	var deepest string
 	for home := range homes {
 		if below(path, home) && len(home) > len(deepest) {
@@ -518,11 +612,12 @@ func moduleOfID(id string) string {
 // a file into it, because nothing judged the file's path.
 func refusedPlaces(resources []declaration.Resource) map[string]error {
 	out := map[string]error{}
+	declared := declaredHomes(resources)
 	for _, r := range resources {
 		var err error
 		switch res := r.(type) {
 		case *declaration.Directory:
-			err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
+			err = refusePathWith(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner, declared)
 			// And outside every home, the walk's judgement before anything is applied (novox/hq issue 358): a
 			// link another account could make in its path refuses it, and with it what is below it, whatever
 			// order they are declared in. The apply walks again at the moment of use. A directory handed over

internal/apply/plan.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/firewall"
 	"github.com/novox/mesh-host/internal/store"
 )
@@ -112,7 +112,7 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
 			step.Verb, step.Why = "restore", "no longer declared; the original the mesh wrote over goes back "+
 				"from "+orphan.Kept+", unless the file was changed since the mesh last wrote it"
 		case orphan.Type == string(declaration.TypeAccess) && orphan.User != "" && len(orphan.Groups) > 0:
-			// In removeAccess's words (novox/hq ADR 0312 §5).
+			// In takeBackGranted's words (novox/hq ADR 0328 §11: no account is in a group for data but a library's).
 			step.Verb, step.Why = "restore", "no longer declared; the path is left as it is, and "+orphan.User+
 				" leaves "+strings.Join(orphan.Groups, ", ")+", which the mesh put it in, unless another declared "+
 				"resource still asks for it"

internal/apply/plan_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/process.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 	"github.com/novox/mesh-host/internal/witness"
 )

internal/apply/process_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 

internal/apply/reads_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )
 
@@ -212,7 +212,7 @@ func TestAContainerLabelledBeforeTheHostReadItsFilesIsAcceptedNotRecreated(t *te
 		t.Fatal(err)
 	}
 	d := declare("5432")
-	legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil, nil)
+	legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil)
 	known := store.State{}
 	known.Record(store.Applied{ID: "forge.env", Type: "file", Origin: store.OriginCarried, Target: env,
 		Wrote: digestOf("DATABASE_PORT=5432\n")})

internal/apply/references.go

@@ -0,0 +1,95 @@
+package apply
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"unicode/utf8"
+
+	"encoding/base64"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+)
+
+// FetchObject reads the object of the declaration-files bucket named by name, no larger than limit bytes (novox/hq ADR
+// 0486 §1, §2). Nil is a node-engine with no link to the bus: a file by reference is then only the file already there.
+type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, error)
+
+// ResolveFileReferences puts the content of every file given by digest and size into the declaration, so what is
+// applied afterwards is the declaration as it was before references existed (novox/hq ADR 0486 §1, issue 1695).
+//
+// **Nothing of the declaration is applied before every reference is resolved**: a missing object, one whose size or
+// sha256 differs from what the signed declaration names, or a fetch that fails, refuses the declaration whole. The
+// content is checked with File.Verify before it is kept, and is only ever written by the apply that follows.
+//
+// A file already on the machine whose content verifies is used as it is and fetches nothing: the reconcile of an
+// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
+// once however many files name its digest.
+func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
+	fetched := map[string][]byte{}
+	for i, r := range d.Resources {
+		f, ok := r.(*declaration.File)
+		if !ok || !f.IsReference() {
+			continue
+		}
+		body, err := referencedContent(ctx, f, fetch, fetched)
+		if err != nil {
+			return err
+		}
+		inline := *f
+		inline.Digest, inline.Size = "", 0
+		if utf8.Valid(body) {
+			inline.Content = string(body)
+		} else {
+			inline.Bytes = base64.StdEncoding.EncodeToString(body)
+		}
+		d.Resources[i] = &inline
+	}
+	return nil
+}
+
+func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
+	fetched map[string][]byte) ([]byte, error) {
+	if body, ok := fetched[f.Digest]; ok {
+		return body, nil
+	}
+	if body, ok := alreadyThere(f); ok {
+		fetched[f.Digest] = body
+		return body, nil
+	}
+	name, err := declaration.ObjectName(f.Digest)
+	if err != nil {
+		return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
+	}
+	if fetch == nil {
+		return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
+			"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
+	}
+	body, err := fetch(ctx, name, f.Size)
+	if err != nil {
+		return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w",
+			f.ID, f.Path, name, err)
+	}
+	if err := f.Verify(body); err != nil {
+		return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
+	}
+	fetched[f.Digest] = body
+	return body, nil
+}
+
+// alreadyThere is the content of the file on the machine when it is what the reference names.
+func alreadyThere(f *declaration.File) ([]byte, bool) {
+	if !filepath.IsAbs(f.Path) {
+		return nil, false
+	}
+	info, err := os.Lstat(f.Path)
+	if err != nil || !info.Mode().IsRegular() || info.Size() != f.Size {
+		return nil, false
+	}
+	body, err := os.ReadFile(f.Path)
+	if err != nil || f.Verify(body) != nil {
+		return nil, false
+	}
+	return body, true
+}

internal/apply/references_test.go

@@ -0,0 +1,132 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+func referenceTo(t *testing.T, path string, body []byte) *declaration.File {
+	t.Helper()
+	return &declaration.File{ID: "m.big", Type: declaration.TypeFile, Path: path,
+		Digest: declaration.DigestOf(body), Size: int64(len(body))}
+}
+
+func objects(m map[string][]byte, asked *[]string) FetchObject {
+	return func(_ context.Context, name string, limit int64) ([]byte, error) {
+		*asked = append(*asked, name)
+		b, ok := m[name]
+		if !ok {
+			return nil, errors.New("object not found")
+		}
+		return b, nil
+	}
+}
+
+func objectOf(body []byte) string {
+	name, _ := declaration.ObjectName(declaration.DigestOf(body))
+	return name
+}
+
+// 1695/AC1: both forms apply: a file by reference becomes the inline file, an inline one is left as it is.
+func TestAFileByReferenceBecomesTheInlineFile(t *testing.T) {
+	body := []byte(strings.Repeat("a line\n", 20000))
+	inline := &declaration.File{ID: "m.small", Type: declaration.TypeFile, Path: "/etc/small", Content: "x"}
+	d := &declaration.Declaration{Resources: []declaration.Resource{inline, referenceTo(t, "/nowhere/big", body)}}
+	var asked []string
+	if err := ResolveFileReferences(context.Background(), d, objects(map[string][]byte{objectOf(body): body}, &asked)); err != nil {
+		t.Fatal(err)
+	}
+	got := d.Resources[1].(*declaration.File)
+	if got.IsReference() || got.Content != string(body) || got.Path != "/nowhere/big" || d.Resources[0] != declaration.Resource(inline) {
+		t.Errorf("1695/AC1: resolved to %+v", got)
+	}
+	if len(asked) != 1 || asked[0] != objectOf(body) {
+		t.Errorf("1695/AC1: asked %v", asked)
+	}
+}
+
+// 1695/AC2: an object that differs by one byte, or in size, fails the whole resolve and gives no content.
+func TestAnObjectThatDiffersFailsTheApply(t *testing.T) {
+	body := []byte(strings.Repeat("x", 70000))
+	bad := append([]byte(nil), body...)
+	bad[3] ^= 1
+	for name, served := range map[string][]byte{"one byte": bad, "shorter": body[1:]} {
+		d := &declaration.Declaration{Resources: []declaration.Resource{referenceTo(t, "/nowhere/big", body)}}
+		var asked []string
+		err := ResolveFileReferences(context.Background(), d, objects(map[string][]byte{objectOf(body): served}, &asked))
+		if err == nil || !strings.Contains(err.Error(), "nothing was applied") {
+			t.Errorf("1695/AC2: %s: %v", name, err)
+		}
+		if !d.Resources[0].(*declaration.File).IsReference() {
+			t.Errorf("1695/AC2: %s: the reference was replaced by unverified content", name)
+		}
+	}
+}
+
+// 1695/AC3: a missing object, or a node with no bus, fails the apply naming the file.
+func TestAMissingObjectFailsTheApply(t *testing.T) {
+	body := []byte(strings.Repeat("y", 70000))
+	d := &declaration.Declaration{Resources: []declaration.Resource{referenceTo(t, "/nowhere/big", body)}}
+	var asked []string
+	if err := ResolveFileReferences(context.Background(), d, objects(nil, &asked)); err == nil ||
+		!strings.Contains(err.Error(), "m.big") || !strings.Contains(err.Error(), "object bucket") {
+		t.Errorf("1695/AC3: missing object: %v", err)
+	}
+	if err := ResolveFileReferences(context.Background(), d, nil); err == nil || !strings.Contains(err.Error(), "no link to the bus") {
+		t.Errorf("1695/AC3: no bus: %v", err)
+	}
+}
+
+// 1695/AC4: a file already on the machine that verifies is used and nothing is fetched; one that does not is fetched.
+func TestAVerifiedFileOnTheMachineFetchesNothing(t *testing.T) {
+	body := []byte(strings.Repeat("z", 70000))
+	path := filepath.Join(t.TempDir(), "big")
+	if err := os.WriteFile(path, body, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	var asked []string
+	d := &declaration.Declaration{Resources: []declaration.Resource{referenceTo(t, path, body)}}
+	if err := ResolveFileReferences(context.Background(), d, objects(nil, &asked)); err != nil || len(asked) != 0 {
+		t.Errorf("1695/AC4: err %v, asked %v", err, asked)
+	}
+	if err := os.WriteFile(path, body[:100], 0o644); err != nil {
+		t.Fatal(err)
+	}
+	d = &declaration.Declaration{Resources: []declaration.Resource{referenceTo(t, path, body)}}
+	if err := ResolveFileReferences(context.Background(), d, objects(map[string][]byte{objectOf(body): body}, &asked)); err != nil || len(asked) != 1 {
+		t.Errorf("1695/AC4: a differing file: err %v, asked %v", err, asked)
+	}
+}
+
+// 1695/AC5: binary content comes back as bytes, and two files naming one digest fetch it once.
+func TestBinaryContentAndOneFetchPerDigest(t *testing.T) {
+	body := append([]byte{0xff, 0xfe, 0x00}, []byte(strings.Repeat("b", 70000))...)
+	d := &declaration.Declaration{Resources: []declaration.Resource{referenceTo(t, "/n/a", body), referenceTo(t, "/n/b", body)}}
+	var asked []string
+	if err := ResolveFileReferences(context.Background(), d, objects(map[string][]byte{objectOf(body): body}, &asked)); err != nil {
+		t.Fatal(err)
+	}
+	a, _ := d.Resources[0].(*declaration.File).Body()
+	if d.Resources[0].(*declaration.File).Bytes == "" || string(a) != string(body) || len(asked) != 1 {
+		t.Errorf("1695/AC5: bytes %q asked %v", d.Resources[0].(*declaration.File).Bytes[:4], asked)
+	}
+}
+
+// 1695/AC9: an unresolved reference reaching the apply is refused, never written as an empty file.
+func TestAnUnresolvedReferenceIsNeverWrittenEmpty(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "big")
+	f := referenceTo(t, path, []byte(strings.Repeat("q", 70000)))
+	if _, err := applyFile(f, store.Applied{}, nil, nil); err == nil || !strings.Contains(err.Error(), "not written empty") {
+		t.Errorf("1695/AC9: %v", err)
+	}
+	if _, err := os.Stat(path); err == nil {
+		t.Error("1695/AC9: a file was written")
+	}
+}

internal/apply/replay358_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/novox/mesh-host/internal/declaration"
+	"git.novox.be/novox/mesh-sdk/go/declaration"
 	"github.com/novox/mesh-host/internal/store"
 )