Count the quiet after a restart from when the controller first saw the search pending
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery failed: its walk failed: a gate on a first machine (what it carried put back), a build, a machine
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery failed: its walk failed: a gate on a first machine (what it carried put back), a build, a machine
The node-engine's own since starts again at every restart, so an agent that restarted the engine in a loop kept agent-can-become-root quiet for ever (the review of 2026-10-09). The controller now keeps when it first saw the verdict waiting for the setuid search (migration 0085), forgets it at the next complete verdict, and raises once the engine's own bound has passed since; the bound and the pending reason are read from mesh-host's rootsearch.
This commit is contained in:
@@ -117,35 +117,67 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim
|
||||
h.SaidAt.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
|
||||
// searchQuietFor is how long a verdict may say its setuid search is still running before that is itself the
|
||||
// urgent condition: the node-engine's bound on one search, and one statement more (a node-engine states its
|
||||
// health at least every five minutes) for the verdict that follows it to be heard.
|
||||
const searchQuietFor = link.RootSearchBound + 5*time.Minute
|
||||
// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid
|
||||
// search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the
|
||||
// engine's own value), counted from when this controller first saw it waiting, never from the engine's start.
|
||||
const searchQuietFor = link.RootSearchBound
|
||||
|
||||
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's first
|
||||
// search for setuid programs, still within its bound (novox/hq ADR 0266): a fresh statement from an engine that
|
||||
// judges root, holding a verdict on the account, every verdict on it healthy or not judged yet because that
|
||||
// search runs — and none of those for longer than searchQuietFor. A way to root found, a search that failed or
|
||||
// did not finish, any other unknown, a stale statement: false, and DA raises it.
|
||||
func searchStillRunning(agent string, h inventory.NodeHealth, had bool, now time.Time) bool {
|
||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||
const (
|
||||
verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown
|
||||
verdictPending // waiting for the search, and otherwise healthy
|
||||
verdictComplete // judged: healthy, or a way to root found
|
||||
)
|
||||
|
||||
// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it
|
||||
// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when
|
||||
// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at
|
||||
// every restart of the engine.
|
||||
func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int {
|
||||
if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract {
|
||||
return false
|
||||
return verdictOther
|
||||
}
|
||||
pending := false
|
||||
pending, any := false, false
|
||||
for _, r := range h.Resources {
|
||||
if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever {
|
||||
continue
|
||||
}
|
||||
any = true
|
||||
switch {
|
||||
case r.State == link.StateHealthy:
|
||||
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending) &&
|
||||
!r.Since.IsZero() && now.Sub(r.Since) <= searchQuietFor:
|
||||
case r.State == link.StateUnhealthy:
|
||||
return verdictComplete
|
||||
case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending):
|
||||
pending = true
|
||||
default:
|
||||
return false
|
||||
return verdictOther
|
||||
}
|
||||
}
|
||||
return pending
|
||||
switch {
|
||||
case !any:
|
||||
return verdictOther
|
||||
case pending:
|
||||
return verdictPending
|
||||
}
|
||||
return verdictComplete
|
||||
}
|
||||
|
||||
// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid
|
||||
// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a
|
||||
// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began.
|
||||
func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth,
|
||||
had bool, now time.Time) (bool, error) {
|
||||
switch rootVerdictKind(agent, h, had, now) {
|
||||
case verdictComplete:
|
||||
return false, inv.RootSearchJudged(ctx, node)
|
||||
case verdictPending:
|
||||
since, err := inv.RootSearchPending(ctx, node, now)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return now.Sub(since) <= searchQuietFor, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's
|
||||
@@ -166,6 +198,10 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
confined, why := judgedConfined(n.AgentAccount, h, had, now)
|
||||
if confined {
|
||||
continue
|
||||
@@ -174,7 +210,7 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio
|
||||
// urgent condition after every restart. The agent is still not confined — ADR 0259's router reads
|
||||
// agentConfined, not this — and `node show` still says not judged. Loud again once the search fails,
|
||||
// runs out its bound, or the statement goes stale.
|
||||
if searchStillRunning(n.AgentAccount, h, had, now) {
|
||||
if quiet {
|
||||
continue
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/novox/mesh-host/rootsearch"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -199,10 +200,14 @@ func TestTheNodeVerbOnlyShows(t *testing.T) {
|
||||
}
|
||||
|
||||
// After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account
|
||||
// is not judged until it ends. DA does not raise that as urgent while the search is within its bound — the
|
||||
// account is still not confined, and `node show` still says not judged — and raises it once the search failed,
|
||||
// ran out its bound, or found a way to root.
|
||||
// is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from
|
||||
// when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an
|
||||
// engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still
|
||||
// says not judged; a search that failed, or a way to root found, is urgent at once.
|
||||
func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
if searchQuietFor != rootsearch.Bound {
|
||||
t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound)
|
||||
}
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -215,11 +220,12 @@ func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &doctor{open: open}
|
||||
say := func(state, reason string, since time.Time) []conditions.Observation {
|
||||
say := func(state, reason string) []conditions.Observation {
|
||||
t.Helper()
|
||||
// The engine's since is always now: it was just restarted.
|
||||
v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account",
|
||||
Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever,
|
||||
Account: "agent", Since: since}
|
||||
Account: "agent", Since: time.Now()}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract,
|
||||
SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -231,23 +237,36 @@ func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) {
|
||||
return onlyMachine(found, "anchor")
|
||||
}
|
||||
running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet"
|
||||
if found := say(link.StateUnknown, running, time.Now().Add(-2*time.Minute)); len(found) != 0 {
|
||||
t.Fatalf("a search two minutes into its bound was raised: %+v", found)
|
||||
healthy := func() {
|
||||
t.Helper()
|
||||
if found := say(link.StateHealthy, ""); len(found) != 0 {
|
||||
t.Fatalf("a healthy verdict raised: %+v", found)
|
||||
}
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 0 {
|
||||
t.Fatalf("a search first seen now was raised: %+v", found)
|
||||
}
|
||||
if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") {
|
||||
t.Fatalf("an account whose search runs was read as confined: %q", why)
|
||||
}
|
||||
if found := say(link.StateUnknown, running, time.Now().Add(-searchQuietFor-time.Minute)); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search past its bound was not urgent: %+v", found)
|
||||
// The engine restarted again and again, each statement's own since fresh: the controller's clock runs on.
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
incomplete := "not judged (search incomplete): the search for setuid programs did not finish (last tried at 19:23: " +
|
||||
"timeout); it is tried again later"
|
||||
if found := say(link.StateUnknown, incomplete, time.Now().Add(-time.Minute)); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
healthy() // a complete verdict forgets when the waiting began …
|
||||
if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil {
|
||||
t.Fatal(err) // … and this restart loop began past the bound
|
||||
}
|
||||
if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found)
|
||||
}
|
||||
healthy()
|
||||
incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " +
|
||||
"19:23: timeout); it is tried again later"
|
||||
if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a search that did not finish was not urgent: %+v", found)
|
||||
}
|
||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running, time.Now()); len(found) != 1 ||
|
||||
if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 ||
|
||||
found[0].Severity != conditions.Urgent {
|
||||
t.Fatalf("a way to root found while the search runs was not urgent: %+v", found)
|
||||
}
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
-- The controller keeps when it first saw an agent account's root verdict waiting for the node-engine's search
|
||||
-- for setuid programs (novox/hq ADR 0266), cleared by the next complete verdict.
|
||||
--
|
||||
-- The self-check does not raise `agent-can-become-root` while that search is within its bound, so a restart is
|
||||
-- not an urgent condition each time. The node-engine's own "since" starts again at every restart: an agent
|
||||
-- that restarted the engine in a loop kept the condition quiet for ever. This time is the controller's, kept
|
||||
-- across the engine's restarts and its own, so the quiet ends once the bound has passed since the search first
|
||||
-- read as pending, however often the engine started again.
|
||||
alter table node add column agent_root_pending_since timestamptz;
|
||||
@@ -1292,3 +1292,24 @@ func (i *Inventory) Sequence(ctx context.Context, id string) (int64, error) {
|
||||
}
|
||||
return *n, nil
|
||||
}
|
||||
|
||||
// RootSearchPending keeps when the controller first saw this node's agent account waiting for the node-engine's
|
||||
// setuid search (novox/hq ADR 0266) and answers it: the first time it is seen since the last complete verdict,
|
||||
// at, kept; seen again, what was kept. Kept across the engine's restarts and the controller's own.
|
||||
func (i *Inventory) RootSearchPending(ctx context.Context, node string, at time.Time) (time.Time, error) {
|
||||
var since time.Time
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`update node set agent_root_pending_since = coalesce(agent_root_pending_since, $2)
|
||||
where name = $1 returning agent_root_pending_since`, node, at).Scan(&since)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, node)
|
||||
}
|
||||
return since, err
|
||||
}
|
||||
|
||||
// RootSearchJudged forgets when a search began pending: a complete verdict came.
|
||||
func (i *Inventory) RootSearchJudged(ctx context.Context, node string) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update node set agent_root_pending_since = null where name = $1 and agent_root_pending_since is not null`, node)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ package link
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"github.com/novox/mesh-host/rootsearch"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -431,14 +432,13 @@ const RootContract = 3
|
||||
const ReasonRoot = "can become root without a person"
|
||||
|
||||
// ReasonRootPending starts the reason of an account verdict the node-engine cannot give yet because its search
|
||||
// for setuid programs, started when the engine started, has not finished (mesh-host internal/accounts
|
||||
// ReasonPending): not judged yet, said as such, never a pass. The search is bounded (RootSearchBound); one that
|
||||
// fails or runs out its bound is said in other words, as not judged (search incomplete).
|
||||
const ReasonRootPending = "not judged yet (search running)"
|
||||
// for setuid programs, started when the engine started, has not finished: not judged yet, said as such, never a
|
||||
// pass. The node-engine's own words (mesh-host rootsearch.ReasonPending), read from it rather than copied.
|
||||
const ReasonRootPending = rootsearch.ReasonPending
|
||||
|
||||
// RootSearchBound is the longest the node-engine lets one search for setuid programs run (mesh-host
|
||||
// internal/accounts SearchBound).
|
||||
const RootSearchBound = 15 * time.Minute
|
||||
// rootsearch.Bound): the one value both read.
|
||||
const RootSearchBound = rootsearch.Bound
|
||||
|
||||
// RootNever is the value of a user's `root`, and of a verdict's Root, that the account must never become
|
||||
// root without a person (ADR 0266).
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
// Package rootsearch holds what the node-engine's search for setuid programs and the controller that reads its
|
||||
// verdicts must agree on (novox/hq ADR 0266): how long one search may run, and the words a verdict starts with
|
||||
// while it has no answer. Public, so the controller imports these rather than keeps copies that could drift.
|
||||
package rootsearch
|
||||
|
||||
import "time"
|
||||
|
||||
// Bound is how long one search for setuid programs may run. It governs the whole search, the walk and the
|
||||
// package manager's answers together; the controller does not raise an agent account as not judged while the
|
||||
// first search after a start is within it.
|
||||
const Bound = 15 * time.Minute
|
||||
|
||||
// The reasons of a root verdict the search could not answer yet.
|
||||
const (
|
||||
// ReasonPending starts the reason while the first search since the engine started runs.
|
||||
ReasonPending = "not judged yet (search running)"
|
||||
// ReasonIncomplete starts the reason when no search has finished: one failed or ran out its bound.
|
||||
ReasonIncomplete = "not judged (search incomplete)"
|
||||
)
|
||||
Vendored
+1
@@ -78,6 +78,7 @@ github.com/nats-io/nuid
|
||||
# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d
|
||||
## explicit; go 1.26.0
|
||||
github.com/novox/mesh-host/internal/declaration
|
||||
github.com/novox/mesh-host/rootsearch
|
||||
github.com/novox/mesh-host/validate
|
||||
# go.uber.org/automaxprocs v1.6.0
|
||||
## explicit; go 1.20
|
||||
|
||||
Reference in New Issue
Block a user