Count the quiet after a restart from when the controller first saw the search pending
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery failed: its walk failed: a gate on a first machine (what it carried put back), a build, a machine

The node-engine's own since starts again at every restart, so an agent that
restarted the engine in a loop kept agent-can-become-root quiet for ever (the
review of 2026-10-09). The controller now keeps when it first saw the verdict
waiting for the setuid search (migration 0085), forgets it at the next
complete verdict, and raises once the engine's own bound has passed since;
the bound and the pending reason are read from mesh-host's rootsearch.
This commit is contained in:
jochen
2026-10-09 12:37:18 +02:00
parent d15eee61bb
commit 2e1a9abbf7
7 changed files with 143 additions and 38 deletions
+53 -17
View File
@@ -117,35 +117,67 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim
h.SaidAt.Local().Format("2006-01-02 15:04")) h.SaidAt.Local().Format("2006-01-02 15:04"))
} }
// searchQuietFor is how long a verdict may say its setuid search is still running before that is itself the // searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
// urgent condition: the node-engine's bound on one search, and one statement more (a node-engine states its // search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the
// health at least every five minutes) for the verdict that follows it to be heard. // engine's own value), counted from when this controller first saw it waiting, never from the engine's start.
const searchQuietFor = link.RootSearchBound + 5*time.Minute const searchQuietFor = link.RootSearchBound
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's first // The kinds of an agent account's verdict, for the quiet a search earns.
// search for setuid programs, still within its bound (novox/hq ADR 0266): a fresh statement from an engine that const (
// judges root, holding a verdict on the account, every verdict on it healthy or not judged yet because that verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown
// search runs — and none of those for longer than searchQuietFor. A way to root found, a search that failed or verdictPending // waiting for the search, and otherwise healthy
// did not finish, any other unknown, a stale statement: false, and DA raises it. verdictComplete // judged: healthy, or a way to root found
func searchStillRunning(agent string, h inventory.NodeHealth, had bool, now time.Time) bool { )
// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it
// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when
// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at
// every restart of the engine.
func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int {
if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract { if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract {
return false return verdictOther
} }
pending := false pending, any := false, false
for _, r := range h.Resources { for _, r := range h.Resources {
if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever { if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever {
continue continue
} }
any = true
switch { switch {
case r.State == link.StateHealthy: case r.State == link.StateHealthy:
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending) && case r.State == link.StateUnhealthy:
!r.Since.IsZero() && now.Sub(r.Since) <= searchQuietFor: return verdictComplete
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending):
pending = true pending = true
default: default:
return false return verdictOther
} }
} }
return pending switch {
case !any:
return verdictOther
case pending:
return verdictPending
}
return verdictComplete
}
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid
// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a
// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began.
func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth,
had bool, now time.Time) (bool, error) {
switch rootVerdictKind(agent, h, had, now) {
case verdictComplete:
return false, inv.RootSearchJudged(ctx, node)
case verdictPending:
since, err := inv.RootSearchPending(ctx, node, now)
if err != nil {
return false, err
}
return now.Sub(since) <= searchQuietFor, nil
}
return false, nil
} }
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's // probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's
@@ -166,6 +198,10 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
return nil, err return nil, err
} }
now := time.Now() now := time.Now()
quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now)
if err != nil {
return nil, err
}
confined, why := judgedConfined(n.AgentAccount, h, had, now) confined, why := judgedConfined(n.AgentAccount, h, had, now)
if confined { if confined {
continue continue
@@ -174,7 +210,7 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads // urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails, // agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
// runs out its bound, or the statement goes stale. // runs out its bound, or the statement goes stale.
if searchStillRunning(n.AgentAccount, h, had, now) { if quiet {
continue continue
} }
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root", out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root",
+34 -15
View File
@@ -1,6 +1,7 @@
package main package main
import ( import (
"github.com/novox/mesh-host/rootsearch"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -199,10 +200,14 @@ func TestTheNodeVerbOnlyShows(t *testing.T) {
} }
// After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account // After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account
// is not judged until it ends. DA does not raise that as urgent while the search is within its bound — the // is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from
// account is still not confined, and `node show` still says not judged — and raises it once the search failed, // when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an
// ran out its bound, or found a way to root. // engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still
// says not judged; a search that failed, or a way to root found, is urgent at once.
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
if searchQuietFor != rootsearch.Bound {
t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound)
}
open := aMesh(t) open := aMesh(t)
ctx := t.Context() ctx := t.Context()
inv := open.inventory inv := open.inventory
@@ -215,11 +220,12 @@ func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
d := &doctor{open: open} d := &doctor{open: open}
say := func(state, reason string, since time.Time) []conditions.Observation { say := func(state, reason string) []conditions.Observation {
t.Helper() t.Helper()
// The engine's since is always now: it was just restarted.
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
Account: "agent", Since: since} Account: "agent", Since: time.Now()}
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -231,23 +237,36 @@ func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
return onlyMachine(found, "anchor") return onlyMachine(found, "anchor")
} }
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
if found := say(link.StateUnknown, running, time.Now().Add(-2*time.Minute)); len(found) != 0 { healthy := func() {
t.Fatalf("a search two minutes into its bound was raised: %+v", found) t.Helper()
if found := say(link.StateHealthy, ""); len(found) != 0 {
t.Fatalf("a healthy verdict raised: %+v", found)
}
}
if found := say(link.StateUnknown, running); len(found) != 0 {
t.Fatalf("a search first seen now was raised: %+v", found)
} }
if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") { if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") {
t.Fatalf("an account whose search runs was read as confined: %q", why) t.Fatalf("an account whose search runs was read as confined: %q", why)
} }
if found := say(link.StateUnknown, running, time.Now().Add(-searchQuietFor-time.Minute)); len(found) != 1 || // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
found[0].Severity != conditions.Urgent { if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
t.Fatalf("a search past its bound was not urgent: %+v", found) t.Fatal(err)
} }
incomplete := "not judged (search incomplete): the search for setuid programs did not finish (last tried at 19:23: " + healthy() // a complete verdict forgets when the waiting began …
"timeout); it is tried again later" if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
if found := say(link.StateUnknown, incomplete, time.Now().Add(-time.Minute)); len(found) != 1 || t.Fatal(err) // … and this restart loop began past the bound
found[0].Severity != conditions.Urgent { }
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
}
healthy()
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
"19:23: timeout); it is tried again later"
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
t.Fatalf("a search that did not finish was not urgent: %+v", found) t.Fatalf("a search that did not finish was not urgent: %+v", found)
} }
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running, time.Now()); len(found) != 1 || if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
found[0].Severity != conditions.Urgent { found[0].Severity != conditions.Urgent {
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
} }
@@ -0,0 +1,9 @@
-- The controller keeps when it first saw an agent account's root verdict waiting for the node-engine's search
-- for setuid programs (novox/hq ADR 0266), cleared by the next complete verdict.
--
-- The self-check does not raise `agent-can-become-root` while that search is within its bound, so a restart is
-- not an urgent condition each time. The node-engine's own "since" starts again at every restart: an agent
-- that restarted the engine in a loop kept the condition quiet for ever. This time is the controller's, kept
-- across the engine's restarts and its own, so the quiet ends once the bound has passed since the search first
-- read as pending, however often the engine started again.
alter table node add column agent_root_pending_since timestamptz;
+21
View File
@@ -1292,3 +1292,24 @@ func (i *Inventory) Sequence(ctx context.Context, id string) (int64, error) {
} }
return *n, nil return *n, nil
} }
// RootSearchPending keeps when the controller first saw this node's agent account waiting for the node-engine's
// setuid search (novox/hq ADR 0266) and answers it: the first time it is seen since the last complete verdict,
// at, kept; seen again, what was kept. Kept across the engine's restarts and the controller's own.
func (i *Inventory) RootSearchPending(ctx context.Context, node string, at time.Time) (time.Time, error) {
var since time.Time
err := i.store.Pool().QueryRow(ctx,
`update node set agent_root_pending_since = coalesce(agent_root_pending_since, $2)
where name = $1 returning agent_root_pending_since`, node, at).Scan(&since)
if errors.Is(err, pgx.ErrNoRows) {
return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, node)
}
return since, err
}
// RootSearchJudged forgets when a search began pending: a complete verdict came.
func (i *Inventory) RootSearchJudged(ctx context.Context, node string) error {
_, err := i.store.Pool().Exec(ctx,
`update node set agent_root_pending_since = null where name = $1 and agent_root_pending_since is not null`, node)
return err
}
+6 -6
View File
@@ -8,6 +8,7 @@ package link
import ( import (
"encoding/base64" "encoding/base64"
"github.com/novox/mesh-host/rootsearch"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@@ -431,14 +432,13 @@ const RootContract = 3
const ReasonRoot = "can become root without a person" const ReasonRoot = "can become root without a person"
// ReasonRootPending starts the reason of an account verdict the node-engine cannot give yet because its search // ReasonRootPending starts the reason of an account verdict the node-engine cannot give yet because its search
// for setuid programs, started when the engine started, has not finished (mesh-host internal/accounts // for setuid programs, started when the engine started, has not finished: not judged yet, said as such, never a
// ReasonPending): not judged yet, said as such, never a pass. The search is bounded (RootSearchBound); one that // pass. The node-engine's own words (mesh-host rootsearch.ReasonPending), read from it rather than copied.
// fails or runs out its bound is said in other words, as not judged (search incomplete). const ReasonRootPending = rootsearch.ReasonPending
const ReasonRootPending = "not judged yet (search running)"
// RootSearchBound is the longest the node-engine lets one search for setuid programs run (mesh-host // RootSearchBound is the longest the node-engine lets one search for setuid programs run (mesh-host
// internal/accounts SearchBound). // rootsearch.Bound): the one value both read.
const RootSearchBound = 15 * time.Minute const RootSearchBound = rootsearch.Bound
// RootNever is the value of a user's `root`, and of a verdict's Root, that the account must never become // RootNever is the value of a user's `root`, and of a verdict's Root, that the account must never become
// root without a person (ADR 0266). // root without a person (ADR 0266).
+19
View File
@@ -0,0 +1,19 @@
// Package rootsearch holds what the node-engine's search for setuid programs and the controller that reads its
// verdicts must agree on (novox/hq ADR 0266): how long one search may run, and the words a verdict starts with
// while it has no answer. Public, so the controller imports these rather than keeps copies that could drift.
package rootsearch
import "time"
// Bound is how long one search for setuid programs may run. It governs the whole search, the walk and the
// package manager's answers together; the controller does not raise an agent account as not judged while the
// first search after a start is within it.
const Bound = 15 * time.Minute
// The reasons of a root verdict the search could not answer yet.
const (
// ReasonPending starts the reason while the first search since the engine started runs.
ReasonPending = "not judged yet (search running)"
// ReasonIncomplete starts the reason when no search has finished: one failed or ran out its bound.
ReasonIncomplete = "not judged (search incomplete)"
)
+1
View File
@@ -78,6 +78,7 @@ github.com/nats-io/nuid
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d # github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d
## explicit; go 1.26.0 ## explicit; go 1.26.0
github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/internal/declaration
github.com/novox/mesh-host/rootsearch
github.com/novox/mesh-host/validate github.com/novox/mesh-host/validate
# go.uber.org/automaxprocs v1.6.0 # go.uber.org/automaxprocs v1.6.0
## explicit; go 1.20 ## explicit; go 1.20