Keep the mesh's trust anchors at the terminal, refuse containerd's tree, and read a found directory as a wait

The review of #170: step-ca's root, roots and path and the identity
provider's issuer are what every consumer trusts, and any caller of the
settings verb could replace them; they are now terminal keys like places and
accesses (hq issue 339). /var/lib/containerd joins the runtimes' data. And a
directory the node-engine uses as found failed its module's gate and rolled
its builds back; found before the send, it is now a wait for a person the
gate passes with, as a relogin is (ADR 0254), and only one the send itself
found holds the module.

cmd/mesh-controller/found_wait_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the
+// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an
+// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds.
+func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
+		Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
+		Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
+			"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
+}
+
+func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
+	now := time.Now()
+	sent := now.Add(-time.Minute)
+	f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}}
+	h, why := moduleHealthWord("notes", "laptop", sent, f)
+	if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") {
+		t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why)
+	}
+	// Found by this very send: the send brought it, and it is not passed.
+	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}}
+	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
+		t.Fatalf("a directory this send found reads %v %q; want not yet", h, why)
+	}
+	// A container down beside the old wait is a fault, as before.
+	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{
+		foundDirectory("notes", sent.Add(-time.Hour)),
+		{Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}}
+	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
+		t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why)
+	}
+}
+
+// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
+// its pass kept and the wait said; a directory this very send found holds it and puts it back.
+func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
+	for _, c := range []struct {
+		name   string
+		found  time.Duration // when the directory was found, against now
+		passes bool
+	}{
+		{"found a day before the send", -24 * time.Hour, true},
+		{"found by this send", time.Hour, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			b := aBacklog(t)
+			ctx := t.Context()
+			inv := b.open.inventory
+			releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
+				return map[string]bool{"anchor": true, "laptop": true}, nil
+			}
+			backlogFacts := gatherGateFacts
+			gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+				f, err := backlogFacts(ctx, open, component)
+				f.health = map[string]inventory.NodeHealth{}
+				for _, n := range []string{"anchor", "laptop"} {
+					f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
+						{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
+						foundDirectory("app", time.Now().Add(c.found)),
+						{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
+				}
+				return f, err
+			}
+			wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+			t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+			gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
+			deadline := time.Now().Add(5 * time.Second)
+			for time.Now().Before(deadline) {
+				advancePlans(ctx, b.open)
+				if p := b.release(t); p.State != inventory.PlanRolling {
+					break
+				}
+				time.Sleep(20 * time.Millisecond)
+			}
+			p := b.release(t)
+			v, found, err := inv.GateOf(ctx, "build-app-c2")
+			if c.passes {
+				if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
+					t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
+				}
+				if !strings.Contains(v.Why+p.Note, "hand it over") {
+					t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
+				}
+				return
+			}
+			if p.State == inventory.PlanDone {
+				t.Fatalf("a directory this send found let the walk through: %s", p.Note)
+			}
+		})
+	}
+}

cmd/mesh-controller/module_health.go

@@ -499,6 +499,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 	if waits && !f.groupsAdded[module] {
 		waits = false
 	}
+	var found []string
 	for _, r := range h.Resources {
 		if r.Module != module {
 			continue
@@ -506,6 +507,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 		if waits && r.State == link.StateUnhealthy {
 			continue
 		}
+		// **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine
+		// left its owner and mode, and only someone at the machine can hand it over. It is no fault of this
+		// build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict
+		// carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy.
+		if usedAsFound(r) && r.Since.Before(since) {
+			found = append(found, r.Resource)
+			continue
+		}
 		switch r.State {
 		case link.StateHealthy:
 		case link.StateStarting:
@@ -521,12 +530,25 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 				reasonAfter(r.Reason))
 		}
 	}
-	if waits {
-		return healthPerson, wait
+	if waits || len(found) > 0 {
+		var said []string
+		if waits {
+			said = append(said, wait)
+		}
+		if len(found) > 0 {
+			said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for a person to hand it over "+
+				"(`mesh-host hand-over <directory>` at the machine)", machine, module, strings.Join(found, ", ")))
+		}
+		return healthPerson, strings.Join(said, "; ")
 	}
 	return healthGood, ""
 }
 
+// usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339).
+func usedAsFound(r inventory.ResourceHealth) bool {
+	return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound)
+}
+
 func reasonAfter(s string) string {
 	if s == "" {
 		return ""

cmd/mesh-controller/modules.go

@@ -1062,12 +1062,12 @@ func declaresTools(m catalogue.Manifest) bool {
 	return false
 }
 
-// terminalSettings are the keys no verb may change (novox/hq issue 339). `places` says where the node-engine
-// creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says which of
-// the machine's paths are mounted into a module's container. Set through a verb, either lets any caller of the
-// mesh's console — an agent among them — have root hand it a directory, or mount one of the machine's into a
-// container it reaches. They are the operator's, typed at the controller's terminal.
-var terminalSettings = []string{catalogue.PlacesSetting, catalogue.AccessesSetting}
+// The keys no verb may change are catalogue.TerminalKeys (novox/hq issue 339). `places` says where the
+// node-engine creates and, as root, owns a module's directories, with an owner the setting names; `accesses` says
+// which of the machine's paths are mounted into a module's container; a provider's trust anchors say what every
+// consumer trusts. Set through a verb, any of them lets any caller of the mesh's verbs — an agent among them —
+// have root hand it a directory, mount one of the machine's into a container it reaches, or have the mesh trust
+// an authority of its own. They are the operator's, typed at the controller's terminal.
 
 // throughAVerb says whether this process runs a seat verb's command line: the serving controller names the
 // verb in the environment of every command it runs for one (runVerb), and a person at the terminal runs none.
@@ -1085,16 +1085,16 @@ func refuseTerminalSettingsThroughAVerb(before, after map[string]any, module, wh
 	if !through {
 		return nil
 	}
-	for _, key := range terminalSettings {
+	for _, key := range catalogue.TerminalKeys(module) {
 		was, _ := json.Marshal(before[key])
 		now, _ := json.Marshal(after[key])
 		if string(was) == string(now) {
 			continue
 		}
 		return fmt.Errorf("%s of %s on %s is set at the controller's terminal only, never through a verb (this "+
-			"line came through %q): it says where root creates and owns a module's directories, or which of "+
-			"the machine's paths are mounted into its container, and whoever may call a verb includes agents "+
-			"(novox/hq issue 339). Nothing was changed", key, module, where, verb)
+			"line came through %q): it says where root creates and owns a module's directories, which of "+
+			"the machine's paths are mounted into its container, or what the mesh's consumers trust, and whoever "+
+			"may call a verb includes agents (novox/hq issue 339). Nothing was changed", key, module, where, verb)
 	}
 	return nil
 }

cmd/mesh-controller/terminal_settings_test.go

@@ -150,3 +150,47 @@ func TestPlacesAndAccessesAreRefusedThroughEveryVerb(t *testing.T) {
 		t.Fatalf("a refused call changed the layer: %s, was %s", got, kept)
 	}
 }
+
+// The mesh's trust anchors are set at the terminal alone (novox/hq issue 339): through the settings verb, a caller
+// could replace the internal authority's root every consumer trusts, or the issuer every login is checked against.
+func TestATrustAnchorIsRefusedThroughAVerb(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
+		Provides: catalogue.FromAnywhere("acme-ca"),
+		Serves:   map[string]map[string]any{"acme-ca": {"root": "", "path": "/acme/acme/directory"}},
+		Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/step.conf", "mode": "0644",
+			"content": "x = ${setting:x}\n"}}})
+	register(t, open, catalogue.Manifest{Module: "keycloak", Version: "1",
+		Provides: catalogue.FromAnywhere("oidc-client"),
+		Serves:   map[string]map[string]any{"oidc-client": {"issuer": "${setting:issuer}"}},
+		Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/kc.conf", "mode": "0644",
+			"content": "issuer = ${setting:issuer}\nx = ${setting:x}\n"}}})
+	for _, m := range []string{"step-ca", "keycloak"} {
+		if _, err := assign(ctx, open, "anchor", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+	refused := func(what string, err error) {
+		t.Helper()
+		if err == nil || !strings.Contains(err.Error(), "controller's terminal") {
+			t.Fatalf("%s: %v", what, err)
+		}
+	}
+	if err := atTheTerminal(t, "settings", "set", "keycloak", `{"issuer":"https://id.example/realms/mesh","x":0}`,
+		"--node", "anchor"); err != nil {
+		t.Fatalf("the issuer at the terminal: %v", err)
+	}
+	refused("the issuer through the verb", throughVerb(t, "settings", map[string]any{"module": "keycloak",
+		"node": "anchor", "values": `{"issuer":"https://evil.example/realms/mesh","x":0}`}))
+	refused("the authority's root path through the verb", throughVerb(t, "settings", map[string]any{"module": "step-ca",
+		"node": "anchor", "values": `{"path":"/evil","x":0}`}))
+	refused("the authority's root path, mesh-wide, through the verb", throughVerb(t, "settings",
+		map[string]any{"module": "step-ca", "values": `{"path":"/evil"}`}))
+	refused("clearing the issuer through the verb", throughVerb(t, "settings", map[string]any{"module": "keycloak",
+		"node": "anchor", "clear": "true"}))
+	if err := throughVerb(t, "settings", map[string]any{"module": "keycloak", "node": "anchor",
+		"values": `{"issuer":"https://id.example/realms/mesh","x":1}`}); err != nil {
+		t.Fatalf("another key through the verb, the issuer kept: %v", err)
+	}
+}