diff --git a/cmd/mesh-controller/agent_account.go b/cmd/mesh-controller/agent_account.go index ef797b29..9436997e 100644 --- a/cmd/mesh-controller/agent_account.go +++ b/cmd/mesh-controller/agent_account.go @@ -117,35 +117,67 @@ func judgedConfined(agent string, h inventory.NodeHealth, had bool, now time.Tim h.SaidAt.Local().Format("2006-01-02 15:04")) } -// searchQuietFor is how long a verdict may say its setuid search is still running before that is itself the -// urgent condition: the node-engine's bound on one search, and one statement more (a node-engine states its -// health at least every five minutes) for the verdict that follows it to be heard. -const searchQuietFor = link.RootSearchBound + 5*time.Minute +// searchQuietFor is how long the controller lets an agent account's verdict wait for the node-engine's setuid +// search before that is itself the urgent condition: the engine's bound on one search (link.RootSearchBound, the +// engine's own value), counted from when this controller first saw it waiting, never from the engine's start. +const searchQuietFor = link.RootSearchBound -// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's first -// search for setuid programs, still within its bound (novox/hq ADR 0266): a fresh statement from an engine that -// judges root, holding a verdict on the account, every verdict on it healthy or not judged yet because that -// search runs — and none of those for longer than searchQuietFor. A way to root found, a search that failed or -// did not finish, any other unknown, a stale statement: false, and DA raises it. -func searchStillRunning(agent string, h inventory.NodeHealth, had bool, now time.Time) bool { +// The kinds of an agent account's verdict, for the quiet a search earns. +const ( + verdictOther = iota // anything else: a stale statement, an older engine, no verdict, another unknown + verdictPending // waiting for the search, and otherwise healthy + verdictComplete // judged: healthy, or a way to root found +) + +// rootVerdictKind reads a statement for the agent account (novox/hq ADR 0266): pending when every verdict on it +// is healthy or not judged yet because the engine's setuid search runs, at least one of them that; complete when +// every verdict is healthy or one found a way to root. The engine's own "since" is not read: it starts again at +// every restart of the engine. +func rootVerdictKind(agent string, h inventory.NodeHealth, had bool, now time.Time) int { if !had || now.Sub(h.HeardAt) > verdictFreshFor || h.Contract < link.RootContract { - return false + return verdictOther } - pending := false + pending, any := false, false for _, r := range h.Resources { if r.Kind != link.KindAccount || r.Target != agent || r.Root != link.RootNever { continue } + any = true switch { case r.State == link.StateHealthy: - case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending) && - !r.Since.IsZero() && now.Sub(r.Since) <= searchQuietFor: + case r.State == link.StateUnhealthy: + return verdictComplete + case r.State == link.StateUnknown && strings.HasPrefix(r.Reason, link.ReasonRootPending): pending = true default: - return false + return verdictOther } } - return pending + switch { + case !any: + return verdictOther + case pending: + return verdictPending + } + return verdictComplete +} + +// searchStillRunning says the one thing keeping an agent account from being judged is the node-engine's setuid +// search, and that this controller first saw it waiting less than searchQuietFor ago — kept in the store, so a +// node-engine restarted in a loop does not keep it quiet. A complete verdict forgets when it began. +func searchStillRunning(ctx context.Context, inv *inventory.Inventory, node, agent string, h inventory.NodeHealth, + had bool, now time.Time) (bool, error) { + switch rootVerdictKind(agent, h, had, now) { + case verdictComplete: + return false, inv.RootSearchJudged(ctx, node) + case verdictPending: + since, err := inv.RootSearchPending(ctx, node, now) + if err != nil { + return false, err + } + return now.Sub(since) <= searchQuietFor, nil + } + return false, nil } // probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's @@ -166,6 +198,10 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio return nil, err } now := time.Now() + quiet, err := searchStillRunning(ctx, inv, n.Name, n.AgentAccount, h, had, now) + if err != nil { + return nil, err + } confined, why := judgedConfined(n.AgentAccount, h, had, now) if confined { continue @@ -174,7 +210,7 @@ func probeAgentAccounts(ctx context.Context, d *doctor) ([]conditions.Observatio // urgent condition after every restart. The agent is still not confined — ADR 0259's router reads // agentConfined, not this — and `node show` still says not judged. Loud again once the search fails, // runs out its bound, or the statement goes stale. - if searchStillRunning(n.AgentAccount, h, had, now) { + if quiet { continue } out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "agent-root", diff --git a/cmd/mesh-controller/agent_account_test.go b/cmd/mesh-controller/agent_account_test.go index acb9a1ce..68143eca 100644 --- a/cmd/mesh-controller/agent_account_test.go +++ b/cmd/mesh-controller/agent_account_test.go @@ -1,6 +1,7 @@ package main import ( + "github.com/novox/mesh-host/rootsearch" "strings" "testing" "time" @@ -199,10 +200,14 @@ func TestTheNodeVerbOnlyShows(t *testing.T) { } // After every node-engine restart its search for setuid programs runs for up to its bound, and the agent account -// is not judged until it ends. DA does not raise that as urgent while the search is within its bound — the -// account is still not confined, and `node show` still says not judged — and raises it once the search failed, -// ran out its bound, or found a way to root. +// is not judged until it ends. DA does not raise that as urgent while the search is within its bound, counted from +// when this controller first saw it waiting — never from the engine's own "since", which a restart resets, so an +// engine restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still +// says not judged; a search that failed, or a way to root found, is urgent at once. func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { + if searchQuietFor != rootsearch.Bound { + t.Fatalf("the quiet is %s and the node-engine's bound %s: they are one value", searchQuietFor, rootsearch.Bound) + } open := aMesh(t) ctx := t.Context() inv := open.inventory @@ -215,11 +220,12 @@ func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { t.Fatal(err) } d := &doctor{open: open} - say := func(state, reason string, since time.Time) []conditions.Observation { + say := func(state, reason string) []conditions.Observation { t.Helper() + // The engine's since is always now: it was just restarted. v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, - Account: "agent", Since: since} + Account: "agent", Since: time.Now()} if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { t.Fatal(err) @@ -231,23 +237,36 @@ func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { return onlyMachine(found, "anchor") } running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" - if found := say(link.StateUnknown, running, time.Now().Add(-2*time.Minute)); len(found) != 0 { - t.Fatalf("a search two minutes into its bound was raised: %+v", found) + healthy := func() { + t.Helper() + if found := say(link.StateHealthy, ""); len(found) != 0 { + t.Fatalf("a healthy verdict raised: %+v", found) + } + } + if found := say(link.StateUnknown, running); len(found) != 0 { + t.Fatalf("a search first seen now was raised: %+v", found) } if _, confined, why, _ := agentConfined(ctx, inv, "anchor"); confined || !strings.Contains(why, "not judged") { t.Fatalf("an account whose search runs was read as confined: %q", why) } - if found := say(link.StateUnknown, running, time.Now().Add(-searchQuietFor-time.Minute)); len(found) != 1 || - found[0].Severity != conditions.Urgent { - t.Fatalf("a search past its bound was not urgent: %+v", found) + // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on. + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) } - incomplete := "not judged (search incomplete): the search for setuid programs did not finish (last tried at 19:23: " + - "timeout); it is tried again later" - if found := say(link.StateUnknown, incomplete, time.Now().Add(-time.Minute)); len(found) != 1 || - found[0].Severity != conditions.Urgent { + healthy() // a complete verdict forgets when the waiting began … + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) // … and this restart loop began past the bound + } + if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent { + t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found) + } + healthy() + incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " + + "19:23: timeout); it is tried again later" + if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent { t.Fatalf("a search that did not finish was not urgent: %+v", found) } - if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running, time.Now()); len(found) != 1 || + if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 || found[0].Severity != conditions.Urgent { t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) } diff --git a/internal/inventory/migrations/0085-the-controller-keeps-when-a-root-search-began-pending.sql b/internal/inventory/migrations/0085-the-controller-keeps-when-a-root-search-began-pending.sql new file mode 100644 index 00000000..66c10a2a --- /dev/null +++ b/internal/inventory/migrations/0085-the-controller-keeps-when-a-root-search-began-pending.sql @@ -0,0 +1,9 @@ +-- The controller keeps when it first saw an agent account's root verdict waiting for the node-engine's search +-- for setuid programs (novox/hq ADR 0266), cleared by the next complete verdict. +-- +-- The self-check does not raise `agent-can-become-root` while that search is within its bound, so a restart is +-- not an urgent condition each time. The node-engine's own "since" starts again at every restart: an agent +-- that restarted the engine in a loop kept the condition quiet for ever. This time is the controller's, kept +-- across the engine's restarts and its own, so the quiet ends once the bound has passed since the search first +-- read as pending, however often the engine started again. +alter table node add column agent_root_pending_since timestamptz; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index f71f2697..03f970b6 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -1292,3 +1292,24 @@ func (i *Inventory) Sequence(ctx context.Context, id string) (int64, error) { } return *n, nil } + +// RootSearchPending keeps when the controller first saw this node's agent account waiting for the node-engine's +// setuid search (novox/hq ADR 0266) and answers it: the first time it is seen since the last complete verdict, +// at, kept; seen again, what was kept. Kept across the engine's restarts and the controller's own. +func (i *Inventory) RootSearchPending(ctx context.Context, node string, at time.Time) (time.Time, error) { + var since time.Time + err := i.store.Pool().QueryRow(ctx, + `update node set agent_root_pending_since = coalesce(agent_root_pending_since, $2) + where name = $1 returning agent_root_pending_since`, node, at).Scan(&since) + if errors.Is(err, pgx.ErrNoRows) { + return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, node) + } + return since, err +} + +// RootSearchJudged forgets when a search began pending: a complete verdict came. +func (i *Inventory) RootSearchJudged(ctx context.Context, node string) error { + _, err := i.store.Pool().Exec(ctx, + `update node set agent_root_pending_since = null where name = $1 and agent_root_pending_since is not null`, node) + return err +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 4c1d83a9..9c73db5f 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -8,6 +8,7 @@ package link import ( "encoding/base64" + "github.com/novox/mesh-host/rootsearch" "strconv" "strings" "time" @@ -431,14 +432,13 @@ const RootContract = 3 const ReasonRoot = "can become root without a person" // ReasonRootPending starts the reason of an account verdict the node-engine cannot give yet because its search -// for setuid programs, started when the engine started, has not finished (mesh-host internal/accounts -// ReasonPending): not judged yet, said as such, never a pass. The search is bounded (RootSearchBound); one that -// fails or runs out its bound is said in other words, as not judged (search incomplete). -const ReasonRootPending = "not judged yet (search running)" +// for setuid programs, started when the engine started, has not finished: not judged yet, said as such, never a +// pass. The node-engine's own words (mesh-host rootsearch.ReasonPending), read from it rather than copied. +const ReasonRootPending = rootsearch.ReasonPending // RootSearchBound is the longest the node-engine lets one search for setuid programs run (mesh-host -// internal/accounts SearchBound). -const RootSearchBound = 15 * time.Minute +// rootsearch.Bound): the one value both read. +const RootSearchBound = rootsearch.Bound // RootNever is the value of a user's `root`, and of a verdict's Root, that the account must never become // root without a person (ADR 0266). diff --git a/vendor/github.com/novox/mesh-host/rootsearch/rootsearch.go b/vendor/github.com/novox/mesh-host/rootsearch/rootsearch.go new file mode 100644 index 00000000..a1c70c0b --- /dev/null +++ b/vendor/github.com/novox/mesh-host/rootsearch/rootsearch.go @@ -0,0 +1,19 @@ +// Package rootsearch holds what the node-engine's search for setuid programs and the controller that reads its +// verdicts must agree on (novox/hq ADR 0266): how long one search may run, and the words a verdict starts with +// while it has no answer. Public, so the controller imports these rather than keeps copies that could drift. +package rootsearch + +import "time" + +// Bound is how long one search for setuid programs may run. It governs the whole search, the walk and the +// package manager's answers together; the controller does not raise an agent account as not judged while the +// first search after a start is within it. +const Bound = 15 * time.Minute + +// The reasons of a root verdict the search could not answer yet. +const ( + // ReasonPending starts the reason while the first search since the engine started runs. + ReasonPending = "not judged yet (search running)" + // ReasonIncomplete starts the reason when no search has finished: one failed or ran out its bound. + ReasonIncomplete = "not judged (search incomplete)" +) diff --git a/vendor/modules.txt b/vendor/modules.txt index fb673637..b8440549 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,6 +78,7 @@ github.com/nats-io/nuid # github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009103656-1c61b72f354d ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration +github.com/novox/mesh-host/rootsearch github.com/novox/mesh-host/validate # go.uber.org/automaxprocs v1.6.0 ## explicit; go 1.20