Keep the mesh's trust anchors at the terminal, refuse containerd's tree, and read a found directory as a wait

The review of #170: step-ca's root, roots and path and the identity
provider's issuer are what every consumer trusts, and any caller of the
settings verb could replace them; they are now terminal keys like places and
accesses (hq issue 339). /var/lib/containerd joins the runtimes' data. And a
directory the node-engine uses as found failed its module's gate and rolled
its builds back; found before the send, it is now a wait for a person the
gate passes with, as a relogin is (ADR 0254), and only one the send itself
found holds the module.

cmd/mesh-controller/found_wait_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the
+// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an
+// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds.
+func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
+		Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
+		Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
+			"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
+}
+
+func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
+	now := time.Now()
+	sent := now.Add(-time.Minute)
+	f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}}
+	h, why := moduleHealthWord("notes", "laptop", sent, f)
+	if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") {
+		t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why)
+	}
+	// Found by this very send: the send brought it, and it is not passed.
+	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}}
+	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
+		t.Fatalf("a directory this send found reads %v %q; want not yet", h, why)
+	}
+	// A container down beside the old wait is a fault, as before.
+	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{
+		foundDirectory("notes", sent.Add(-time.Hour)),
+		{Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}}
+	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
+		t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why)
+	}
+}
+
+// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
+// its pass kept and the wait said; a directory this very send found holds it and puts it back.
+func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
+	for _, c := range []struct {
+		name   string
+		found  time.Duration // when the directory was found, against now
+		passes bool
+	}{
+		{"found a day before the send", -24 * time.Hour, true},
+		{"found by this send", time.Hour, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			b := aBacklog(t)
+			ctx := t.Context()
+			inv := b.open.inventory
+			releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
+				return map[string]bool{"anchor": true, "laptop": true}, nil
+			}
+			backlogFacts := gatherGateFacts
+			gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+				f, err := backlogFacts(ctx, open, component)
+				f.health = map[string]inventory.NodeHealth{}
+				for _, n := range []string{"anchor", "laptop"} {
+					f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
+						{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
+						foundDirectory("app", time.Now().Add(c.found)),
+						{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
+				}
+				return f, err
+			}
+			wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+			t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+			gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
+			deadline := time.Now().Add(5 * time.Second)
+			for time.Now().Before(deadline) {
+				advancePlans(ctx, b.open)
+				if p := b.release(t); p.State != inventory.PlanRolling {
+					break
+				}
+				time.Sleep(20 * time.Millisecond)
+			}
+			p := b.release(t)
+			v, found, err := inv.GateOf(ctx, "build-app-c2")
+			if c.passes {
+				if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
+					t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
+				}
+				if !strings.Contains(v.Why+p.Note, "hand it over") {
+					t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
+				}
+				return
+			}
+			if p.State == inventory.PlanDone {
+				t.Fatalf("a directory this send found let the walk through: %s", p.Note)
+			}
+		})
+	}
+}

cmd/mesh-controller/module_health.go

@@ -499,6 +499,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 	if waits && !f.groupsAdded[module] {
 		waits = false
 	}
+	var found []string
 	for _, r := range h.Resources {
 		if r.Module != module {
 			continue
@@ -506,6 +507,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 		if waits && r.State == link.StateUnhealthy {
 			continue
 		}
+		// **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine
+		// left its owner and mode, and only someone at the machine can hand it over. It is no fault of this
+		// build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict
+		// carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy.
+		if usedAsFound(r) && r.Since.Before(since) {
+			found = append(found, r.Resource)
+			continue
+		}
 		switch r.State {
 		case link.StateHealthy:
 		case link.StateStarting:
@@ -521,12 +530,25 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 				reasonAfter(r.Reason))
 		}
 	}
-	if waits {
-		return healthPerson, wait
+	if waits || len(found) > 0 {
+		var said []string
+		if waits {
+			said = append(said, wait)
+		}
+		if len(found) > 0 {
+			said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+
+				"(`mesh-host hand-over <directory>` at the machine)", machine, module, strings.Join(found, ", ")))
+		}
+		return healthPerson, strings.Join(said, "; ")
 	}
 	return healthGood, ""
 }
 
+// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339).
+func usedAsFound(r inventory.ResourceHealth) bool {
+	return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound)
+}
+
 func reasonAfter(s string) string {
 	if s == "" {
 		return ""

cmd/mesh-controller/modules.go

@@ -1062,12 +1062,12 @@ func declaresTools(m catalogue.Manifest) bool {
 	return false
 }
 
-// terminalSettings are the keys no verb may change (novox/hq issue 339). `places` says where the node-engine
-// creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says which of
-// the machine's paths are mounted into a module's container. Set through a verb, either lets any caller of the
-// mesh's console — an agent among them — have root hand it a directory, or mount one of the machine's into a
-// container it reaches. They are the operator's, typed at the controller's terminal.
-var terminalSettings = []string{catalogue.PlacesSetting, catalogue.AccessesSetting}
+// The keys no verb may change are catalogue.TerminalKeys (novox/hq issue 339). `places` says where the
+// node-engine creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says
+// which of the machine's paths are mounted into a module's container; a provider's trust anchors say what every
+// consumer trusts. Set through a verb, any of them lets any caller of the mesh's verbs — an agent among them —
+// have root hand it a directory, mount one of the machine's into a container it reaches, or have the mesh trust
+// an authority of its own. They are the operator's, typed at the controller's terminal.
 
 // throughAVerb says whether this process runs a seat verb's command line: the serving controller names the
 // verb in the environment of every command it runs for one (runVerb), and a person at the terminal runs none.
@@ -1085,16 +1085,16 @@ func refuseTerminalSettingsThroughAVerb(before, after map[string]any, module, wh
 	if !through {
 		return nil
 	}
-	for _, key := range terminalSettings {
+	for _, key := range catalogue.TerminalKeys(module) {
 		was, _ := json.Marshal(before[key])
 		now, _ := json.Marshal(after[key])
 		if string(was) == string(now) {
 			continue
 		}
 		return fmt.Errorf("%s of %s on %s is set at the controller's terminal only, never through a verb (this "+
-			"line came through %q): it says where root creates and owns a module's directories, or which of "+
-			"the machine's paths are mounted into its container, and whoever may call a verb includes agents "+
-			"(novox/hq issue 339). Nothing was changed", key, module, where, verb)
+			"line came through %q): it says where root creates and owns a module's directories, which of "+
+			"the machine's paths are mounted into its container, or what the mesh's consumers trust, and whoever "+
+			"may call a verb includes agents (novox/hq issue 339). Nothing was changed", key, module, where, verb)
 	}
 	return nil
 }

cmd/mesh-controller/terminal_settings_test.go

@@ -150,3 +150,47 @@ func TestPlacesAndAccessesAreRefusedThroughEveryVerb(t *testing.T) {
 		t.Fatalf("a refused call changed the layer: %s, was %s", got, kept)
 	}
 }
+
+// The mesh's trust anchors are set at the terminal alone (novox/hq issue 339): through the settings verb, a caller
+// could replace the internal authority's root every consumer trusts, or the issuer every login is checked against.
+func TestATrustAnchorIsRefusedThroughAVerb(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
+		Provides: catalogue.FromAnywhere("acme-ca"),
+		Serves:   map[string]map[string]any{"acme-ca": {"root": "", "path": "/acme/acme/directory"}},
+		Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/step.conf", "mode": "0644",
+			"content": "x = ${setting:x}\n"}}})
+	register(t, open, catalogue.Manifest{Module: "keycloak", Version: "1",
+		Provides: catalogue.FromAnywhere("oidc-client"),
+		Serves:   map[string]map[string]any{"oidc-client": {"issuer": "${setting:issuer}"}},
+		Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/kc.conf", "mode": "0644",
+			"content": "issuer = ${setting:issuer}\nx = ${setting:x}\n"}}})
+	for _, m := range []string{"step-ca", "keycloak"} {
+		if _, err := assign(ctx, open, "anchor", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+	refused := func(what string, err error) {
+		t.Helper()
+		if err == nil || !strings.Contains(err.Error(), "controller's terminal") {
+			t.Fatalf("%s: %v", what, err)
+		}
+	}
+	if err := atTheTerminal(t, "settings", "set", "keycloak", `{"issuer":"https://id.example/realms/mesh","x":0}`,
+		"--node", "anchor"); err != nil {
+		t.Fatalf("the issuer at the terminal: %v", err)
+	}
+	refused("the issuer through the verb", throughVerb(t, "settings", map[string]any{"module": "keycloak",
+		"node": "anchor", "values": `{"issuer":"https://evil.example/realms/mesh","x":0}`}))
+	refused("the authority's root path through the verb", throughVerb(t, "settings", map[string]any{"module": "step-ca",
+		"node": "anchor", "values": `{"path":"/evil","x":0}`}))
+	refused("the authority's root path, mesh-wide, through the verb", throughVerb(t, "settings",
+		map[string]any{"module": "step-ca", "values": `{"path":"/evil"}`}))
+	refused("clearing the issuer through the verb", throughVerb(t, "settings", map[string]any{"module": "keycloak",
+		"node": "anchor", "clear": "true"}))
+	if err := throughVerb(t, "settings", map[string]any{"module": "keycloak", "node": "anchor",
+		"values": `{"issuer":"https://id.example/realms/mesh","x":1}`}); err != nil {
+		t.Fatalf("another key through the verb, the issuer kept: %v", err)
+	}
+}

internal/catalogue/placement.go

@@ -62,7 +62,7 @@ var ownerShape = regexp.MustCompile(`^[0-9]+:[0-9]+$`)
 var (
 	systemTrees = []string{"/etc", "/usr", "/boot", "/root", "/run", "/var/run", "/var/lock", "/proc", "/sys",
 		"/dev", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/var/lib/mesh", "/var/lib/mesh-host", "/var/spool",
-		"/var/lib/docker", "/var/lib/containers", "/opt"}
+		"/var/lib/docker", "/var/lib/containers", "/var/lib/containerd", "/opt"}
 	systemRoots = []string{"/", "/var", "/var/lib", "/var/cache", "/var/log", "/var/tmp", "/home",
 		"/mnt", "/media", "/srv", "/tmp", "/storage", "/data", "/services"}
 )
@@ -438,3 +438,24 @@ func namesOfAccessIDs(accesses map[string]string) []string {
 	sort.Strings(names)
 	return names
 }
+
+// trustAnchors are the settings a provider serves its consumers as what they trust, by module (novox/hq issue
+// 339): set through a verb, any caller could point every consumer at an authority or an issuer of its own.
+//
+//   - step-ca, the mesh's internal ACME authority: `root`, the root a consumer is handed to trust (the one
+//     setting that may hold lines, settingsHoldOneLine); `roots` and `path`, where a consumer fetches the roots
+//     and the ACME directory from, which a setting may override as it may any served fact.
+//   - keycloak, the identity provider: `issuer`, the issuer every OIDC consumer checks a login's token against.
+//
+// Named here, not in the manifests, because no manifest field says "this is trusted" yet; the catalogue was read
+// for every served fact and every ${setting:…} on 2026-10-09, and these are the ones a consumer trusts.
+var trustAnchors = map[string][]string{
+	rootModule: {rootSetting, "roots", "path"},
+	"keycloak": {"issuer"},
+}
+
+// TerminalKeys are the settings keys of a module that are set at the controller's terminal alone, never through
+// a verb (novox/hq issue 339): places and accesses for every module, and a provider's trust anchors.
+func TerminalKeys(module string) []string {
+	return append([]string{PlacesSetting, AccessesSetting}, trustAnchors[module]...)
+}

internal/catalogue/terminal_settings_test.go

@@ -109,7 +109,7 @@ func TestTheRuntimesDataAndAnyHomesSSHAreTheMachinesOwn(t *testing.T) {
 	m := Manifest{Module: "notes", Resources: []map[string]any{{"id": "data", "type": "directory"}},
 		Accesses: []Access{{ID: "media"}}}
 	for _, path := range []string{"/var/spool", "/var/spool/cron", "/var/lib/docker", "/var/lib/docker/volumes",
-		"/var/lib/containers/storage", "/opt", "/opt/app", "/home/alice/.ssh", "/home/alice/.ssh/keys",
+		"/var/lib/containers/storage", "/var/lib/containerd", "/var/lib/containerd/io.containerd.snapshotter.v1", "/opt", "/opt/app", "/home/alice/.ssh", "/home/alice/.ssh/keys",
 		"/srv/backup/.ssh", "/root/.ssh"} {
 		layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}}
 		if _, err := Places(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") {
@@ -127,3 +127,18 @@ func TestTheRuntimesDataAndAnyHomesSSHAreTheMachinesOwn(t *testing.T) {
 		}
 	}
 }
+
+// A trust anchor the mesh hands its consumers is the terminal's too (novox/hq issue 339): the authority's root,
+// where its roots and directory are, and the identity provider's issuer.
+func TestTrustAnchorsAreTerminalKeys(t *testing.T) {
+	for module, keys := range map[string][]string{
+		"step-ca":  {"places", "accesses", "root", "roots", "path"},
+		"keycloak": {"places", "accesses", "issuer"},
+		"mailu":    {"places", "accesses"},
+	} {
+		got := TerminalKeys(module)
+		if strings.Join(got, ",") != strings.Join(keys, ",") {
+			t.Errorf("%s: %v; want %v", module, got, keys)
+		}
+	}
+}

internal/link/protocol.go

@@ -446,6 +446,14 @@ const KindUnit = "unit"
 // starting ReasonRelogin when only a new login is missing.
 const KindAccount = "account"
 
+// KindDirectory is a directory of a module that the node-engine uses as found (novox/hq issue 339): there before
+// the mesh, with another owner or mode than declared, and left so until a person hands it over at the machine
+// (`mesh-host hand-over`). Stated unhealthy with a reason that starts ReasonUsedAsFound.
+const KindDirectory = "directory"
+
+// ReasonUsedAsFound starts the reason of a directory used as found.
+const ReasonUsedAsFound = "used as found:"
+
 // ReasonRelogin starts the reason of an account whose running session began before it was put in a group
 // (ADR 0252): the build did what it should, and a person has one step left (novox/hq ADR 0254).
 const ReasonRelogin = "relogin needed"