Keep the mesh's trust anchors at the terminal, refuse containerd's tree, and read a found directory as a wait

The review of #170: step-ca's root, roots and path and the identity
provider's issuer are what every consumer trusts, and any caller of the
settings verb could replace them; they are now terminal keys like places and
accesses (hq issue 339). /var/lib/containerd joins the runtimes' data. And a
directory the node-engine uses as found failed its module's gate and rolled
its builds back; found before the send, it is now a wait for a person the
gate passes with, as a relogin is (ADR 0254), and only one the send itself
found holds the module.

cmd/mesh-controller/found_wait_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the
+// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an
+// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds.
+func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
+		Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
+		Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
+			"not given it — `mesh-host hand-over` at the machine hands it to the mesh"}
+}
+
+func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
+	now := time.Now()
+	sent := now.Add(-time.Minute)
+	f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}}
+	h, why := moduleHealthWord("notes", "laptop", sent, f)
+	if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") {
+		t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why)
+	}
+	// Found by this very send: the send brought it, and it is not passed.
+	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}}
+	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
+		t.Fatalf("a directory this send found reads %v %q; want not yet", h, why)
+	}
+	// A container down beside the old wait is a fault, as before.
+	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{
+		foundDirectory("notes", sent.Add(-time.Hour)),
+		{Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}}
+	if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
+		t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why)
+	}
+}
+
+// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
+// its pass kept and the wait said; a directory this very send found holds it and puts it back.
+func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
+	for _, c := range []struct {
+		name   string
+		found  time.Duration // when the directory was found, against now
+		passes bool
+	}{
+		{"found a day before the send", -24 * time.Hour, true},
+		{"found by this send", time.Hour, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			b := aBacklog(t)
+			ctx := t.Context()
+			inv := b.open.inventory
+			releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
+				return map[string]bool{"anchor": true, "laptop": true}, nil
+			}
+			backlogFacts := gatherGateFacts
+			gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+				f, err := backlogFacts(ctx, open, component)
+				f.health = map[string]inventory.NodeHealth{}
+				for _, n := range []string{"anchor", "laptop"} {
+					f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
+						{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
+						foundDirectory("app", time.Now().Add(c.found)),
+						{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
+				}
+				return f, err
+			}
+			wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+			t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+			gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
+			deadline := time.Now().Add(5 * time.Second)
+			for time.Now().Before(deadline) {
+				advancePlans(ctx, b.open)
+				if p := b.release(t); p.State != inventory.PlanRolling {
+					break
+				}
+				time.Sleep(20 * time.Millisecond)
+			}
+			p := b.release(t)
+			v, found, err := inv.GateOf(ctx, "build-app-c2")
+			if c.passes {
+				if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
+					t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
+				}
+				if !strings.Contains(v.Why+p.Note, "hand it over") {
+					t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
+				}
+				return
+			}
+			if p.State == inventory.PlanDone {
+				t.Fatalf("a directory this send found let the walk through: %s", p.Note)
+			}
+		})
+	}
+}