mesh-self-check: DA judges each machine's agent account on its engine's own report, so the probe moves off the controller with its kind, key and words (issue 1952, ADR 0266)
The agent account a machine names comes from placement; the engine's verdicts on it from its health report (mesh-host.health, NodeHealth.Root and Contract); whether that verdict is on now from the watchman's presence, since the report is written on a change only. The judgement is the controller's: no report, not heard, an engine older than the judging, no verdict, or any verdict not healthy is not confined; a verdict waiting for the setuid search is quiet for twelve hours from when this self-check first saw it waiting. machine.<node>.agent-root, urgent, the controller's words. Rows in TestTheBusProbes; pins mesh-sdk f3677eb.
This commit is contained in:
@@ -34,7 +34,7 @@ const (
|
||||
// `reads`, which its test holds to these.
|
||||
var (
|
||||
Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb}
|
||||
Reads = []string{PlacementRead, SeatHoldersRead, PresenceRead}
|
||||
Reads = []string{PlacementRead, SeatHoldersRead, PresenceRead, HealthRead}
|
||||
)
|
||||
|
||||
// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe
|
||||
@@ -70,8 +70,7 @@ func Registry() []Probe {
|
||||
"bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects},
|
||||
{ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " +
|
||||
"statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8",
|
||||
Kind: "agent-can-become-root",
|
||||
Deferred: "the node-engines' statements of the agent account's judgement: " + storeOnly},
|
||||
Kind: "agent-can-become-root", Run: probeAgentAccounts},
|
||||
{ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " +
|
||||
"proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free",
|
||||
Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly},
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package main
|
||||
|
||||
// The account agents run as (novox/hq ADR 0266) and root where the trusted parties run (ADR 0259 §8), judged on the
|
||||
// machine's own word: each node-engine says, in its health report (mesh-host.health, key <node>), its verdict on every
|
||||
// account a module declares never to become root without a person. The judgement, its kind, key and words are the
|
||||
// controller's (mesh-controller agent_account.go). Whether the verdict is current is the watchman's presence: the
|
||||
// report is written on a change only, so a machine not heard from has no verdict on now.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The engine's words and bounds the judgement reads (mesh-host rootsearch).
|
||||
const (
|
||||
// HealthRead is the node-engines' health reports.
|
||||
HealthRead = delivery.NodeHealthRead
|
||||
// rootPending starts a verdict's reason while the engine's setuid search runs and no complete one judges.
|
||||
rootPending = "not judged yet (search running)"
|
||||
// rootQuiet is how long nothing is raised while an agent account waits for that search, counted from when this
|
||||
// self-check first saw it waiting: past it, waiting is itself the urgent condition. It never makes it confined.
|
||||
rootQuiet = 12 * time.Hour
|
||||
stateHealthy = "healthy"
|
||||
stateUnhealthy = "unhealthy"
|
||||
stateUnknown = "unknown"
|
||||
)
|
||||
|
||||
// report is a node's health report, and whether it has one.
|
||||
func (c *Checker) report(node string) (delivery.NodeHealth, bool, error) {
|
||||
raw, ok, err := c.Store.Get(HealthRead, node)
|
||||
if err != nil {
|
||||
return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if !ok {
|
||||
return delivery.NodeHealth{}, false, nil
|
||||
}
|
||||
h, err := delivery.DecodeNodeHealth(raw)
|
||||
if err != nil {
|
||||
return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s: %w", node, err)
|
||||
}
|
||||
return h, true, nil
|
||||
}
|
||||
|
||||
// pendingSince is, by node, when this self-check first saw its agent account waiting for the setuid search: kept while
|
||||
// it waits, forgotten once a verdict is complete. In this process alone: a restart of the node-engine does not restart
|
||||
// it, and a restart of the self-check gives a waiting account at most one more quiet.
|
||||
type pendingSince struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]time.Time
|
||||
}
|
||||
|
||||
func (p *pendingSince) since(node string, now time.Time) time.Time {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.seen == nil {
|
||||
p.seen = map[string]time.Time{}
|
||||
}
|
||||
if at, ok := p.seen[node]; ok {
|
||||
return at
|
||||
}
|
||||
p.seen[node] = now
|
||||
return now
|
||||
}
|
||||
|
||||
func (p *pendingSince) judged(node string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
delete(p.seen, node)
|
||||
}
|
||||
|
||||
// The kinds of an agent account's verdict, for the quiet a search earns.
|
||||
const (
|
||||
verdictOther = iota // anything else: not heard, an older engine, no verdict, another unknown
|
||||
verdictPending // waiting for the search, and otherwise healthy
|
||||
verdictComplete // judged: healthy, or a way to root found
|
||||
)
|
||||
|
||||
// judgedConfined is the controller's judgement over one machine's report: confined only when the machine is heard
|
||||
// from, its engine judges root, and every verdict on the agent account is healthy. why is said either way.
|
||||
func judgedConfined(agent string, h delivery.NodeHealth, had, heard bool) (bool, string) {
|
||||
switch {
|
||||
case !had:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+
|
||||
"nothing of what it runs", agent)
|
||||
case !heard:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine is not heard from, and a verdict "+
|
||||
"from before is not a verdict on now", agent)
|
||||
case h.Contract < delivery.RootContract:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+
|
||||
"the judging of an account's root (its statement's contract is %d, the judging is %d)",
|
||||
agent, h.Contract, delivery.RootContract)
|
||||
}
|
||||
verdicts := verdictsOn(agent, h)
|
||||
if len(verdicts) == 0 {
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+
|
||||
"verdict on it — no module there declares it never to become root, or the declaration naming it "+
|
||||
"has not been applied", agent)
|
||||
}
|
||||
for _, v := range verdicts {
|
||||
switch v.State {
|
||||
case stateHealthy:
|
||||
case stateUnhealthy:
|
||||
return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent, orNoneKnown(v.Reason),
|
||||
v.Module, v.Resource)
|
||||
default:
|
||||
return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent,
|
||||
orNoneKnown(v.Reason), v.Module, v.Resource, v.State)
|
||||
}
|
||||
}
|
||||
return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent,
|
||||
h.At.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
|
||||
// verdictsOn is every verdict on the account, sorted by module and resource.
|
||||
func verdictsOn(agent string, h delivery.NodeHealth) []delivery.RootVerdict {
|
||||
var out []delivery.RootVerdict
|
||||
for _, v := range h.Root {
|
||||
if v.Account == agent {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Resource < out[j].Module+out[j].Resource })
|
||||
return out
|
||||
}
|
||||
|
||||
// rootVerdictKind reads a report for the agent account: pending when every verdict on it is healthy or waits for the
|
||||
// setuid search, at least one of them that; complete when every verdict is healthy or one found a way to root.
|
||||
func rootVerdictKind(agent string, h delivery.NodeHealth, had, heard bool) int {
|
||||
if !had || !heard || h.Contract < delivery.RootContract {
|
||||
return verdictOther
|
||||
}
|
||||
pending, any := false, false
|
||||
for _, v := range verdictsOn(agent, h) {
|
||||
any = true
|
||||
switch {
|
||||
case v.State == stateHealthy:
|
||||
case v.State == stateUnhealthy:
|
||||
return verdictComplete
|
||||
case v.State == stateUnknown && strings.HasPrefix(v.Reason, rootPending):
|
||||
pending = true
|
||||
default:
|
||||
return verdictOther
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !any:
|
||||
return verdictOther
|
||||
case pending:
|
||||
return verdictPending
|
||||
}
|
||||
return verdictComplete
|
||||
}
|
||||
|
||||
// agentQuiet says the one thing keeping a machine's agent account unjudged is the engine's setuid search, first seen
|
||||
// waiting less than rootQuiet ago.
|
||||
func (c *Checker) agentQuiet(node, agent string, h delivery.NodeHealth, had, heard bool) bool {
|
||||
switch rootVerdictKind(agent, h, had, heard) {
|
||||
case verdictComplete:
|
||||
c.pending.judged(node)
|
||||
case verdictPending:
|
||||
now := c.clock()
|
||||
return now.Sub(c.pending.since(node, now)) <= rootQuiet
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// clock is the checker's clock, the wall clock where it has none.
|
||||
func (c *Checker) clock() time.Time {
|
||||
if c.Now != nil {
|
||||
return c.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// confinement is one machine's agent account judged: whether it names one, whether it is confined, why, and whether
|
||||
// it is unjudged only because its search runs within the quiet.
|
||||
type confinement struct {
|
||||
Named, Confined, Quiet bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// confined judges node's agent account from its placement, its report and its presence.
|
||||
func (c *Checker) confined(node string, place delivery.Placement, heard bool) (confinement, error) {
|
||||
agent := place.Decided.AgentAccount
|
||||
if agent == "" {
|
||||
return confinement{Why: fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)",
|
||||
node, orNoneKnown(place.Decided.Account))}, nil
|
||||
}
|
||||
h, had, err := c.report(node)
|
||||
if err != nil {
|
||||
return confinement{Named: true}, err
|
||||
}
|
||||
ok, why := judgedConfined(agent, h, had, heard)
|
||||
return confinement{Named: true, Confined: ok, Why: why, Quiet: !ok && c.agentQuiet(node, agent, h, had, heard)}, nil
|
||||
}
|
||||
|
||||
// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest
|
||||
// statement, unable to become root without a person (ADR 0266).
|
||||
func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) {
|
||||
places, err := c.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
heard, err := c.heard()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := make([]string, 0, len(places))
|
||||
for n := range places {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var out []Observation
|
||||
for _, n := range nodes {
|
||||
if places[n].Decided.AgentAccount == "" {
|
||||
continue
|
||||
}
|
||||
v, err := c.confined(n, places[n], heard[n])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Not judged yet only because the engine's search runs: not the urgent condition after every restart. Loud
|
||||
// again once the search fails, waits past the quiet, or the machine is not heard from.
|
||||
if v.Confined || v.Quiet {
|
||||
continue
|
||||
}
|
||||
out = append(out, Observation{Kind: "agent-can-become-root", Token: "agent-root", Severity: Urgent,
|
||||
Subject: Subject{Scope: ScopeMachine, ID: n, Machine: n},
|
||||
Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+
|
||||
"answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why),
|
||||
Said: v.Why,
|
||||
Headline: "Sessions on " + n + " could become root",
|
||||
Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.",
|
||||
Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+
|
||||
"can take over the machine without you. The machine cannot show that this holds now, so an answer "+
|
||||
"from your phone authorises nothing there until it does.", n),
|
||||
Resolved: "Sessions on " + n + " cannot become root again"})
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
|
||||
// orNoneKnown is a value, or that none is known.
|
||||
func orNoneKnown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "none known"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -152,6 +152,18 @@ func TestTheBusProbes(t *testing.T) {
|
||||
"ace": `{"banned":[]}`}}, nil, "", machines()},
|
||||
{"D8 a heard holder that does not answer is no pass", probeBans, bus{banned: map[string]string{}}, nil,
|
||||
"a ban list could not be read", machines()},
|
||||
// DA (ADR 0266): the agent account each machine names, judged on its engine's report while it is heard from.
|
||||
{"DA confined", probeAgentAccounts, bus{}, nil, "", agents(map[string]string{"novox": stateHealthy})},
|
||||
{"DA a way to root found", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "",
|
||||
agents(map[string]string{"novox": stateUnhealthy})},
|
||||
{"DA no verdict on it", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "",
|
||||
agents(map[string]string{"novox": ""})},
|
||||
{"DA the search runs, within its quiet", probeAgentAccounts, bus{}, nil, "",
|
||||
agents(map[string]string{"novox": "pending"})},
|
||||
{"DA a machine not heard from has no verdict on now", probeAgentAccounts, bus{}, []string{"machine.g14.agent-root"},
|
||||
"", agents(map[string]string{"novox": stateHealthy, "g14": stateHealthy})},
|
||||
{"DA an engine older than the judging", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "",
|
||||
agents(map[string]string{"novox": "old"})},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
c := &Checker{Ask: tc.bus.ask, Store: tc.mesh}
|
||||
@@ -193,6 +205,35 @@ func machines() *memStore {
|
||||
return s
|
||||
}
|
||||
|
||||
// agents is machines() where each node named names the agent account "agent" and its report holds the verdict on it:
|
||||
// a state, "" for none, "pending" for one waiting for its search, "old" for an engine older than the judging.
|
||||
func agents(verdicts map[string]string) *memStore {
|
||||
s := machines()
|
||||
for node, v := range verdicts {
|
||||
raw, _, _ := s.Get(PlacementRead, node)
|
||||
var p delivery.Placement
|
||||
_ = json.Unmarshal(raw, &p)
|
||||
p.Decided.AgentAccount = "agent"
|
||||
_ = s.Put(PlacementRead, node, p)
|
||||
h := delivery.NodeHealth{V: 1, Node: node, Contract: 13}
|
||||
switch v {
|
||||
case "":
|
||||
case "old":
|
||||
h.Contract = 2
|
||||
h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: stateHealthy}}
|
||||
case "pending":
|
||||
h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent",
|
||||
State: stateUnknown, Reason: rootPending + ": 3 of 9 trees"}}
|
||||
default:
|
||||
h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: v,
|
||||
Reason: "can become root without a person: a sudo rule"}}
|
||||
}
|
||||
raw, _ = delivery.EncodeNodeHealth(h)
|
||||
_ = s.Put(HealthRead, node, json.RawMessage(raw))
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what
|
||||
// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4).
|
||||
func TestTheRegistryIsADR0564sTable(t *testing.T) {
|
||||
|
||||
@@ -104,6 +104,8 @@ type Checker struct {
|
||||
seen map[string]bool
|
||||
// unsaid counts the runs whose `checked` could not be said since the start.
|
||||
unsaid int
|
||||
// pending is when each machine's agent account was first seen waiting for its setuid search (DA, D-root).
|
||||
pending pendingSince
|
||||
}
|
||||
|
||||
// LastRun is the last run, nil before the first.
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-self-check
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223859-85f5fd0b2266
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223859-85f5fd0b2266 h1:32lx0DCHm+7Ig6t8vXbfwPbk4YgbHQt1sRzmRlezFWw=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029223859-85f5fd0b2266/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e h1:h8bh7ZyAgPuFC9/lc6iFBD/hduzomLodZ0a1oJZ5X4U=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -21,7 +21,8 @@
|
||||
"reads": [
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-assigner.seat-holders",
|
||||
"mesh-delivery-watchman.presence"
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-host.health"
|
||||
],
|
||||
"state": [
|
||||
"findings",
|
||||
|
||||
Reference in New Issue
Block a user