mesh-catalog/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go

package main

// The account agents run as (novox/hq ADR 0266) and root where the trusted parties run (ADR 0259 §8), judged on the
// machine's own word: each node-engine says, in its health report (mesh-host.health, key <node>), its verdict on every
// account a module declares never to become root without a person. The judgement, its kind, key and words are the
// controller's (mesh-controller agent_account.go). Whether the verdict is current is the watchman's presence: the
// report is written on a change only, so a machine not heard from has no verdict on now.

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
	placing "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
	nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
)

// The engine's words and bounds the judgement reads (mesh-host rootsearch).
const (
	// HealthRead is the node-engines' health reports.
	HealthRead = (nodeEngineModule + "." + nodeseat.StateHealth)
	// rootPending starts a verdict's reason while the engine's setuid search runs and no complete one judges.
	rootPending = "not judged yet (search running)"
	// rootQuiet is how long nothing is raised while an agent account waits for that search, counted from when this
	// self-check first saw it waiting: past it, waiting is itself the urgent condition. It never makes it confined.
	rootQuiet      = 12 * time.Hour
	stateHealthy   = "healthy"
	stateUnhealthy = "unhealthy"
	stateUnknown   = "unknown"
)

// report is a node's health report, and whether it has one.
func (c *Checker) report(node string) (nodeseat.HealthByNodeValue, bool, error) {
	raw, ok, err := c.Store.Get(HealthRead, node)
	if err != nil {
		return nodeseat.HealthByNodeValue{}, false, fmt.Errorf("the health report of %s cannot be read: %w", node, err)
	}
	if !ok {
		return nodeseat.HealthByNodeValue{}, false, nil
	}
	h, err := engineclient.DecodeHealth(raw)
	if err != nil {
		return nodeseat.HealthByNodeValue{}, false, fmt.Errorf("the health report of %s: %w", node, err)
	}
	return h, true, nil
}

// pendingSince is, by node, when this self-check first saw its agent account waiting for the setuid search: kept while
// it waits, forgotten once a verdict is complete. In this process alone: a restart of the node-engine does not restart
// it, and a restart of the self-check gives a waiting account at most one more quiet.
type pendingSince struct {
	mu   sync.Mutex
	seen map[string]time.Time
}

func (p *pendingSince) since(node string, now time.Time) time.Time {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.seen == nil {
		p.seen = map[string]time.Time{}
	}
	if at, ok := p.seen[node]; ok {
		return at
	}
	p.seen[node] = now
	return now
}

func (p *pendingSince) judged(node string) {
	p.mu.Lock()
	defer p.mu.Unlock()
	delete(p.seen, node)
}

// The kinds of an agent account's verdict, for the quiet a search earns.
const (
	verdictOther    = iota // anything else: not heard, an older engine, no verdict, another unknown
	verdictPending         // waiting for the search, and otherwise healthy
	verdictComplete        // judged: healthy, or a way to root found
)

// judgedConfined is the controller's judgement over one machine's report: confined only when the machine is heard
// from, its engine judges root, and every verdict on the agent account is healthy. why is said either way.
func judgedConfined(agent string, h nodeseat.HealthByNodeValue, had, heard bool) (bool, string) {
	switch {
	case !had:
		return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+
			"nothing of what it runs", agent)
	case !heard:
		return false, fmt.Sprintf("the agent account %s is not judged: the machine is not heard from, and a verdict "+
			"from before is not a verdict on now", agent)
	case h.Contract < engineclient.RootContract:
		return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+
			"the judging of an account's root (its statement's contract is %d, the judging is %d)",
			agent, h.Contract, engineclient.RootContract)
	}
	verdicts := verdictsOn(agent, h)
	if len(verdicts) == 0 {
		return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+
			"verdict on it — no module there declares it never to become root, or the declaration naming it "+
			"has not been applied", agent)
	}
	for _, v := range verdicts {
		switch v.State {
		case stateHealthy:
		case stateUnhealthy:
			return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent, orNoneKnown(v.Reason),
				v.Module, v.Resource)
		default:
			return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent,
				orNoneKnown(v.Reason), v.Module, v.Resource, v.State)
		}
	}
	return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent,
		h.At.Local().Format("2006-01-02 15:04"))
}

// verdictsOn is every verdict on the account, sorted by module and resource.
func verdictsOn(agent string, h nodeseat.HealthByNodeValue) []nodeseat.RootAccountVerdict {
	var out []nodeseat.RootAccountVerdict
	for _, v := range h.Root {
		if v.Account == agent {
			out = append(out, v)
		}
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Resource < out[j].Module+out[j].Resource })
	return out
}

// rootVerdictKind reads a report for the agent account: pending when every verdict on it is healthy or waits for the
// setuid search, at least one of them that; complete when every verdict is healthy or one found a way to root.
func rootVerdictKind(agent string, h nodeseat.HealthByNodeValue, had, heard bool) int {
	if !had || !heard || h.Contract < engineclient.RootContract {
		return verdictOther
	}
	pending, any := false, false
	for _, v := range verdictsOn(agent, h) {
		any = true
		switch {
		case v.State == stateHealthy:
		case v.State == stateUnhealthy:
			return verdictComplete
		case v.State == stateUnknown && strings.HasPrefix(v.Reason, rootPending):
			pending = true
		default:
			return verdictOther
		}
	}
	switch {
	case !any:
		return verdictOther
	case pending:
		return verdictPending
	}
	return verdictComplete
}

// agentQuiet says the one thing keeping a machine's agent account unjudged is the engine's setuid search, first seen
// waiting less than rootQuiet ago.
func (c *Checker) agentQuiet(node, agent string, h nodeseat.HealthByNodeValue, had, heard bool) bool {
	switch rootVerdictKind(agent, h, had, heard) {
	case verdictComplete:
		c.pending.judged(node)
	case verdictPending:
		now := c.clock()
		return now.Sub(c.pending.since(node, now)) <= rootQuiet
	}
	return false
}

// clock is the checker's clock, the wall clock where it has none.
func (c *Checker) clock() time.Time {
	if c.Now != nil {
		return c.Now()
	}
	return time.Now()
}

// confinement is one machine's agent account judged: whether it names one, whether it is confined, why, and whether
// it is unjudged only because its search runs within the quiet.
type confinement struct {
	Named, Confined, Quiet bool
	Why                    string
}

// confined judges node's agent account from its assignments, its report and its presence.
func (c *Checker) confined(node string, place placing.Assignments, heard bool) (confinement, error) {
	agent := place.Decided.AgentAccount
	if agent == "" {
		return confinement{Why: fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)",
			node, orNoneKnown(place.Decided.Account))}, nil
	}
	h, had, err := c.report(node)
	if err != nil {
		return confinement{Named: true}, err
	}
	ok, why := judgedConfined(agent, h, had, heard)
	return confinement{Named: true, Confined: ok, Why: why, Quiet: !ok && c.agentQuiet(node, agent, h, had, heard)}, nil
}

// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest
// statement, unable to become root without a person (ADR 0266).
func probeAgentAccounts(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) {
	places, err := c.assignments()
	if err != nil {
		return nil, err
	}
	heard, err := c.heard()
	if err != nil {
		return nil, err
	}
	nodes := make([]string, 0, len(places))
	for n := range places {
		nodes = append(nodes, n)
	}
	sort.Strings(nodes)
	var out []meshconditionsclient.Observation
	for _, n := range nodes {
		if places[n].Decided.AgentAccount == "" {
			continue
		}
		v, err := c.confined(n, places[n], heard[n])
		if err != nil {
			return nil, err
		}
		// Not judged yet only because the engine's search runs: not the urgent condition after every restart. Loud
		// again once the search fails, waits past the quiet, or the machine is not heard from.
		if v.Confined || v.Quiet {
			continue
		}
		out = append(out, meshconditionsclient.Observation{Kind: "agent-can-become-root", Ref: "agent-root", Severity: meshconditionsclient.Urgent,
			Scope: ScopeMachine, ID: n, Machine: n,
			Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+
				"answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why),
			Said:     v.Why,
			Headline: "Sessions on " + n + " could become root",
			Needs:    "take the sessions' own account out of every group and rule that grants root; the details say which.",
			Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+
				"can take over the machine without you. The machine cannot show that this holds now, so an answer "+
				"from your phone authorises nothing there until it does.", n),
			Resolved: "Sessions on " + n + " cannot become root again"})
	}
	return sorted(out), nil
}

// orNoneKnown is a value, or that none is known.
func orNoneKnown(s string) string {
	if strings.TrimSpace(s) == "" {
		return "none known"
	}
	return s
}