diff --git a/cmd/mesh-controller/agent_account.go b/cmd/mesh-controller/agent_account.go index e288c1a1..9436997e 100644 --- a/cmd/mesh-controller/agent_account.go +++ b/cmd/mesh-controller/agent_account.go @@ -117,6 +117,69 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim h.SaidAt.Local().Format("2006-01-02 15:04")) } +// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid +// search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the +// engine's own value), counted from when this controller first saw it waiting, never from the engine's start. +const searchQuietFor = link.RootSearchBound + +// The kinds of an agent account's verdict, for the quiet a search earns. +const ( + verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown + verdictPending // waiting for the search, and otherwise healthy + verdictComplete // judged: healthy, or a way to root found +) + +// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it +// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when +// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at +// every restart of the engine. +func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int { + if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract { + return verdictOther + } + pending, any := false, false + for _, r := range h.Resources { + if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever { + continue + } + any = true + switch { + case r.State == link.StateHealthy: + case r.State == link.StateUnhealthy: + return verdictComplete + case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending): + pending = true + default: + return verdictOther + } + } + switch { + case !any: + return verdictOther + case pending: + return verdictPending + } + return verdictComplete +} + +// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid +// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a +// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began. +func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth, + had bool, now time.Time) (bool, error) { + switch rootVerdictKind(agent, h, had, now) { + case verdictComplete: + return false, inv.RootSearchJudged(ctx, node) + case verdictPending: + since, err := inv.RootSearchPending(ctx, node, now) + if err != nil { + return false, err + } + return now.Sub(since) <= searchQuietFor, nil + } + return false, nil +} + // probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's // newest statement, unable to become root without a person (ADR 0266). func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observation, error) { @@ -134,10 +197,22 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio if err != nil { return nil, err } - confined, why := judgedConfined(n.AgentAccount, h, had, time.Now()) + now := time.Now() + quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now) + if err != nil { + return nil, err + } + confined, why := judgedConfined(n.AgentAccount, h, had, now) if confined { continue } + // Not judged yet only because the first search since the node-engine started is still running: not the + // urgent condition after every restart. The agent is still not confined — ADR 0259's router reads + // agentConfined, not this — and `node show` still says not judged. Loud again once the search fails, + // runs out its bound, or the statement goes stale. + if quiet { + continue + } out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root", Machine: n.Name, Severity: conditions.Urgent, Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and "+ diff --git a/cmd/mesh-controller/agent_account_test.go b/cmd/mesh-controller/agent_account_test.go index 3d384620..68143eca 100644 --- a/cmd/mesh-controller/agent_account_test.go +++ b/cmd/mesh-controller/agent_account_test.go @@ -1,6 +1,7 @@ package main import ( + "github.com/novox/mesh-host/rootsearch" "strings" "testing" "time" @@ -197,3 +198,76 @@ func TestTheNodeVerbOnlyShows(t *testing.T) { } } } + +// After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account +// is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from +// when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an +// engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still +// says not judged; a search that failed, or a way to root found, is urgent at once. +func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { + if searchQuietFor != rootsearch.Bound { + t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound) + } + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.NodeByName(ctx, "anchor"); err != nil { + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { + t.Fatal(err) + } + d := &doctor{open: open} + say := func(state, reason string) []conditions.Observation { + t.Helper() + // The engine's since is always now: it was just restarted. + v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", + Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, + Account: "agent", Since: time.Now()} + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, + SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { + t.Fatal(err) + } + found, err := probeAgentAccounts(ctx, d) + if err != nil { + t.Fatal(err) + } + return onlyMachine(found, "anchor") + } + running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" + healthy := func() { + t.Helper() + if found := say(link.StateHealthy, ""); len(found) != 0 { + t.Fatalf("a healthy verdict raised: %+v", found) + } + } + if found := say(link.StateUnknown, running); len(found) != 0 { + t.Fatalf("a search first seen now was raised: %+v", found) + } + if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") { + t.Fatalf("an account whose search runs was read as confined: %q", why) + } + // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on. + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) + } + healthy() // a complete verdict forgets when the waiting began … + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) // … and this restart loop began past the bound + } + if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent { + t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found) + } + healthy() + incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " + + "19:23: timeout); it is tried again later" + if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent { + t.Fatalf("a search that did not finish was not urgent: %+v", found) + } + if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 || + found[0].Severity != conditions.Urgent { + t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) + } +} diff --git a/internal/inventory/migrations/0085-the-controller-keeps-when-a-root-search-began-pending.sql b/internal/inventory/migrations/0085-the-controller-keeps-when-a-root-search-began-pending.sql new file mode 100644 index 00000000..66c10a2a --- /dev/null +++ b/internal/inventory/migrations/0085-the-controller-keeps-when-a-root-search-began-pending.sql @@ -0,0 +1,9 @@ +-- The controller keeps when it first saw an agent account's root verdict waiting for the node-engine's search +-- for setuid programs (novox/hq ADR 0266), cleared by the next complete verdict. +-- +-- The self-check does not raise `agent-can-become-root` while that search is within its bound, so a restart is +-- not an urgent condition each time. The node-engine's own "since" starts again at every restart: an agent +-- that restarted the engine in a loop kept the condition quiet for ever. This time is the controller's, kept +-- across the engine's restarts and its own, so the quiet ends once the bound has passed since the search first +-- read as pending, however often the engine started again. +alter table node add column agent_root_pending_since timestamptz; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index f71f2697..03f970b6 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -1292,3 +1292,24 @@ func (i *Inventory) Sequence(ctx context.Context, id string) (int64, error) { } return *n, nil } + +// RootSearchPending keeps when the controller first saw this node's agent account waiting for the node-engine's +// setuid search (novox/hq ADR 0266) and answers it: the first time it is seen since the last complete verdict, +// at, kept; seen again, what was kept. Kept across the engine's restarts and the controller's own. +func (i *Inventory) RootSearchPending(ctx context.Context, node string, at time.Time) (time.Time, error) { + var since time.Time + err := i.store.Pool().QueryRow(ctx, + `update node set agent_root_pending_since = coalesce(agent_root_pending_since, $2) + where name = $1 returning agent_root_pending_since`, node, at).Scan(&since) + if errors.Is(err, pgx.ErrNoRows) { + return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, node) + } + return since, err +} + +// RootSearchJudged forgets when a search began pending: a complete verdict came. +func (i *Inventory) RootSearchJudged(ctx context.Context, node string) error { + _, err := i.store.Pool().Exec(ctx, + `update node set agent_root_pending_since = null where name = $1 and agent_root_pending_since is not null`, node) + return err +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 36448195..9c73db5f 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -8,6 +8,7 @@ package link import ( "encoding/base64" + "github.com/novox/mesh-host/rootsearch" "strconv" "strings" "time" @@ -430,6 +431,15 @@ const RootContract = 3 // own words (mesh-host internal/accounts ReasonRoot). const ReasonRoot = "can become root without a person" +// ReasonRootPending starts the reason of an account verdict the node-engine cannot give yet because its search +// for setuid programs, started when the engine started, has not finished: not judged yet, said as such, never a +// pass. The node-engine's own words (mesh-host rootsearch.ReasonPending), read from it rather than copied. +const ReasonRootPending = rootsearch.ReasonPending + +// RootSearchBound is the longest the node-engine lets one search for setuid programs run (mesh-host +// rootsearch.Bound): the one value both read. +const RootSearchBound = rootsearch.Bound + // RootNever is the value of a user's `root`, and of a verdict's Root, that the account must never become // root without a person (ADR 0266). const RootNever = "never" diff --git a/vendor/github.com/novox/mesh-host/rootsearch/rootsearch.go b/vendor/github.com/novox/mesh-host/rootsearch/rootsearch.go new file mode 100644 index 00000000..a1c70c0b --- /dev/null +++ b/vendor/github.com/novox/mesh-host/rootsearch/rootsearch.go @@ -0,0 +1,19 @@ +// Package rootsearch holds what the node-engine's search for setuid programs and the controller that reads its +// verdicts must agree on (novox/hq ADR 0266): how long one search may run, and the words a verdict starts with +// while it has no answer. Public, so the controller imports these rather than keeps copies that could drift. +package rootsearch + +import "time" + +// Bound is how long one search for setuid programs may run. It governs the whole search, the walk and the +// package manager's answers together; the controller does not raise an agent account as not judged while the +// first search after a start is within it. +const Bound = 15 * time.Minute + +// The reasons of a root verdict the search could not answer yet. +const ( + // ReasonPending starts the reason while the first search since the engine started runs. + ReasonPending = "not judged yet (search running)" + // ReasonIncomplete starts the reason when no search has finished: one failed or ran out its bound. + ReasonIncomplete = "not judged (search incomplete)" +) diff --git a/vendor/modules.txt b/vendor/modules.txt index fb673637..b8440549 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,6 +78,7 @@ github.com/nats-io/nuid # github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration +github.com/novox/mesh-host/rootsearch github.com/novox/mesh-host/validate # go.uber.org/automaxprocs v1.6.0 ## explicit; go 1.20