Merge pull request 'Do not raise agent-can-become-root while the first setuid search is within its bound (hq ADR 0266, note E)' (#175) from fix/a-search-not-finished-after-a-restart-is-not-urgent into main

cmd/mesh-controller/agent_account.go

@@ -117,6 +117,69 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim
 		h.SaidAt.Local().Format("2006-01-02 15:04"))
 }
 
+// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
+// search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the
+// engine's own value), counted from when this controller first saw it waiting, never from the engine's start.
+const searchQuietFor = link.RootSearchBound
+
+// The kinds of an agent account's verdict, for the quiet a search earns.
+const (
+	verdictOther    = iota // anything else: a stale statement, an older engine, no verdict, another unknown
+	verdictPending         // waiting for the search, and otherwise healthy
+	verdictComplete        // judged: healthy, or a way to root found
+)
+
+// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it
+// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when
+// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at
+// every restart of the engine.
+func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int {
+	if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract {
+		return verdictOther
+	}
+	pending, any := false, false
+	for _, r := range h.Resources {
+		if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever {
+			continue
+		}
+		any = true
+		switch {
+		case r.State == link.StateHealthy:
+		case r.State == link.StateUnhealthy:
+			return verdictComplete
+		case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending):
+			pending = true
+		default:
+			return verdictOther
+		}
+	}
+	switch {
+	case !any:
+		return verdictOther
+	case pending:
+		return verdictPending
+	}
+	return verdictComplete
+}
+
+// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid
+// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a
+// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began.
+func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth,
+	had bool, now time.Time) (bool, error) {
+	switch rootVerdictKind(agent, h, had, now) {
+	case verdictComplete:
+		return false, inv.RootSearchJudged(ctx, node)
+	case verdictPending:
+		since, err := inv.RootSearchPending(ctx, node, now)
+		if err != nil {
+			return false, err
+		}
+		return now.Sub(since) <= searchQuietFor, nil
+	}
+	return false, nil
+}
+
 // probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's
 // newest statement, unable to become root without a person (ADR 0266).
 func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
@@ -134,10 +197,22 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
 		if err != nil {
 			return nil, err
 		}
-		confined, why := judgedConfined(n.AgentAccount, h, had, time.Now())
+		now := time.Now()
+		quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now)
+		if err != nil {
+			return nil, err
+		}
+		confined, why := judgedConfined(n.AgentAccount, h, had, now)
 		if confined {
 			continue
 		}
+		// Not judged yet only because the first search since the node-engine started is still running: not the
+		// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
+		// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
+		// runs out its bound, or the statement goes stale.
+		if quiet {
+			continue
+		}
 		out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root",
 			Machine: n.Name, Severity: conditions.Urgent,
 			Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and "+

cmd/mesh-controller/agent_account_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/novox/mesh-host/rootsearch"
 	"strings"
 	"testing"
 	"time"
@@ -197,3 +198,76 @@ func TestTheNodeVerbOnlyShows(t *testing.T) {
 		}
 	}
 }
+
+// After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account
+// is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from
+// when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an
+// engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still
+// says not judged; a search that failed, or a way to root found, is urgent at once.
+func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
+	if searchQuietFor != rootsearch.Bound {
+		t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound)
+	}
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	if _, err := inv.NodeByName(ctx, "anchor"); err != nil {
+		if _, err := inv.AddNode(ctx, "anchor"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil {
+		t.Fatal(err)
+	}
+	d := &doctor{open: open}
+	say := func(state, reason string) []conditions.Observation {
+		t.Helper()
+		// The engine's since is always now: it was just restarted.
+		v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
+			Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
+			Account: "agent", Since: time.Now()}
+		if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
+			SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
+			t.Fatal(err)
+		}
+		found, err := probeAgentAccounts(ctx, d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return onlyMachine(found, "anchor")
+	}
+	running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
+	healthy := func() {
+		t.Helper()
+		if found := say(link.StateHealthy, ""); len(found) != 0 {
+			t.Fatalf("a healthy verdict raised: %+v", found)
+		}
+	}
+	if found := say(link.StateUnknown, running); len(found) != 0 {
+		t.Fatalf("a search first seen now was raised: %+v", found)
+	}
+	if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") {
+		t.Fatalf("an account whose search runs was read as confined: %q", why)
+	}
+	// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
+	if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
+		t.Fatal(err)
+	}
+	healthy() // a complete verdict forgets when the waiting began …
+	if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
+		t.Fatal(err) // … and this restart loop began past the bound
+	}
+	if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
+		t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
+	}
+	healthy()
+	incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
+		"19:23: timeout); it is tried again later"
+	if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
+		t.Fatalf("a search that did not finish was not urgent: %+v", found)
+	}
+	if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
+		found[0].Severity != conditions.Urgent {
+		t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
+	}
+}

internal/inventory/migrations/0085-the-controller-keeps-when-a-root-search-began-pending.sql

@@ -0,0 +1,9 @@
+-- The controller keeps when it first saw an agent account's root verdict waiting for the node-engine's search
+-- for setuid programs (novox/hq ADR 0266), cleared by the next complete verdict.
+--
+-- The self-check does not raise `agent-can-become-root` while that search is within its bound, so a restart is
+-- not an urgent condition each time. The node-engine's own "since" starts again at every restart: an agent
+-- that restarted the engine in a loop kept the condition quiet for ever. This time is the controller's, kept
+-- across the engine's restarts and its own, so the quiet ends once the bound has passed since the search first
+-- read as pending, however often the engine started again.
+alter table node add column agent_root_pending_since timestamptz;

internal/inventory/nodes.go

@@ -1292,3 +1292,24 @@ func (i *Inventory) Sequence(ctx context.Context, id string) (int64, error) {
 	}
 	return *n, nil
 }
+
+// RootSearchPending keeps when the controller first saw this node's agent account waiting for the node-engine's
+// setuid search (novox/hq ADR 0266) and answers it: the first time it is seen since the last complete verdict,
+// at, kept; seen again, what was kept. Kept across the engine's restarts and the controller's own.
+func (i *Inventory) RootSearchPending(ctx context.Context, node string, at time.Time) (time.Time, error) {
+	var since time.Time
+	err := i.store.Pool().QueryRow(ctx,
+		`update node set agent_root_pending_since = coalesce(agent_root_pending_since, $2)
+		  where name = $1 returning agent_root_pending_since`, node, at).Scan(&since)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, node)
+	}
+	return since, err
+}
+
+// RootSearchJudged forgets when a search began pending: a complete verdict came.
+func (i *Inventory) RootSearchJudged(ctx context.Context, node string) error {
+	_, err := i.store.Pool().Exec(ctx,
+		`update node set agent_root_pending_since = null where name = $1 and agent_root_pending_since is not null`, node)
+	return err
+}

internal/link/protocol.go

@@ -8,6 +8,7 @@ package link
 
 import (
 	"encoding/base64"
+	"github.com/novox/mesh-host/rootsearch"
 	"strconv"
 	"strings"
 	"time"
@@ -430,6 +431,15 @@ const RootContract = 3
 // own words (mesh-host internal/accounts ReasonRoot).
 const ReasonRoot = "can become root without a person"
 
+// ReasonRootPending starts the reason of an account verdict the node-engine cannot give yet because its search
+// for setuid programs, started when the engine started, has not finished: not judged yet, said as such, never a
+// pass. The node-engine's own words (mesh-host rootsearch.ReasonPending), read from it rather than copied.
+const ReasonRootPending = rootsearch.ReasonPending
+
+// RootSearchBound is the longest the node-engine lets one search for setuid programs run (mesh-host
+// rootsearch.Bound): the one value both read.
+const RootSearchBound = rootsearch.Bound
+
 // RootNever is the value of a user's `root`, and of a verdict's Root, that the account must never become
 // root without a person (ADR 0266).
 const RootNever = "never"

vendor/github.com/novox/mesh-host/rootsearch/rootsearch.go

@@ -0,0 +1,19 @@
+// Package rootsearch holds what the node-engine's search for setuid programs and the controller that reads its
+// verdicts must agree on (novox/hq ADR 0266): how long one search may run, and the words a verdict starts with
+// while it has no answer. Public, so the controller imports these rather than keeps copies that could drift.
+package rootsearch
+
+import "time"
+
+// Bound is how long one search for setuid programs may run. It governs the whole search, the walk and the
+// package manager's answers together; the controller does not raise an agent account as not judged while the
+// first search after a start is within it.
+const Bound = 15 * time.Minute
+
+// The reasons of a root verdict the search could not answer yet.
+const (
+	// ReasonPending starts the reason while the first search since the engine started runs.
+	ReasonPending = "not judged yet (search running)"
+	// ReasonIncomplete starts the reason when no search has finished: one failed or ran out its bound.
+	ReasonIncomplete = "not judged (search incomplete)"
+)

vendor/modules.txt

@@ -78,6 +78,7 @@ github.com/nats-io/nuid
 # github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d
 ## explicit; go 1.26.0
 github.com/novox/mesh-host/internal/declaration
+github.com/novox/mesh-host/rootsearch
 github.com/novox/mesh-host/validate
 # go.uber.org/automaxprocs v1.6.0
 ## explicit; go 1.20