Merge pull request 'Say every failed unit, the module's and the machine's (hq issue 315)' (#53) from fix/315-a-failed-unit-is-a-condition into main
This commit was merged in pull request #53.
This commit is contained in:
+72
-3
@@ -43,6 +43,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
"github.com/novox/mesh-host/internal/tunnel"
|
||||
"github.com/novox/mesh-host/internal/units"
|
||||
"github.com/novox/mesh-host/internal/upgrade"
|
||||
)
|
||||
|
||||
@@ -1090,6 +1091,8 @@ func runLink(ctx context.Context, opts options) error {
|
||||
return held
|
||||
}
|
||||
judging = j
|
||||
// And which units its service managers say failed, and whose each is (novox/hq issue 315).
|
||||
unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
|
||||
}
|
||||
|
||||
// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
|
||||
@@ -1384,6 +1387,10 @@ const ReconcileEvery = 5 * time.Minute
|
||||
// reports no health, and in a test.
|
||||
var judging *liveness.Judge
|
||||
|
||||
// unitJudge reads which units the machine's service managers say failed, and whose each is (novox/hq
|
||||
// issue 315); nil where judging is.
|
||||
var unitJudge *units.Judge
|
||||
|
||||
// netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a
|
||||
// one-shot command and in a test, which say nothing of the network.
|
||||
var netJudge networkJudge
|
||||
@@ -1478,6 +1485,17 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
}
|
||||
st, changed := j.Look(ctx)
|
||||
owed = owed || changed
|
||||
var unitSt *units.Statement
|
||||
if u := unitJudge; u != nil {
|
||||
us, unitsChanged := u.Look(ctx)
|
||||
unitSt = &us
|
||||
owed = owed || unitsChanged
|
||||
if unitsChanged {
|
||||
for _, f := range us.Failed {
|
||||
say(failedUnitWords(f))
|
||||
}
|
||||
}
|
||||
}
|
||||
var netSt *network.Statement
|
||||
if n := netJudge.get(); n != nil {
|
||||
ns, netChanged := n.Look(ctx)
|
||||
@@ -1500,7 +1518,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
spaced = sp
|
||||
}
|
||||
since := time.Since(lastSaid)
|
||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy)
|
||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
||||
(unitSt == nil || len(unitSt.Failed) == 0)
|
||||
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||
continue
|
||||
}
|
||||
@@ -1512,7 +1531,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
}
|
||||
}
|
||||
}
|
||||
if queue.SayHealth(ctx, *healthAsReported(st, netSt)) {
|
||||
if queue.SayHealth(ctx, *withUnits(healthAsReported(st, netSt), unitSt)) {
|
||||
lastSaid, owed = time.Now(), false
|
||||
}
|
||||
}
|
||||
@@ -1538,6 +1557,50 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health
|
||||
return h
|
||||
}
|
||||
|
||||
// withUnits adds to a statement the machine's failed units (novox/hq issue 315): each a module places,
|
||||
// among the resources as that module's unhealthy unit; the rest, the machine's own, beside them. Nil — an
|
||||
// engine not reading them — says nothing of them, which the controller reads as not known.
|
||||
func withUnits(h *link.Health, us *units.Statement) *link.Health {
|
||||
if us == nil || us.State == "" {
|
||||
return h
|
||||
}
|
||||
h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread}
|
||||
for _, f := range us.Failed {
|
||||
if f.Module == "" {
|
||||
h.Units.Failed = append(h.Units.Failed, link.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load,
|
||||
Result: f.Result, Resource: f.Resource, Since: f.Since.UTC()})
|
||||
continue
|
||||
}
|
||||
reason := "failed"
|
||||
if f.Scope == units.ScopeUser {
|
||||
reason += " in the account's own service manager"
|
||||
}
|
||||
if f.Result != "" {
|
||||
reason += " (" + f.Result + ")"
|
||||
}
|
||||
h.Resources = append(h.Resources, link.ResourceHealth{Module: f.Module, Resource: f.Resource,
|
||||
Kind: link.KindUnit, Target: f.Unit, State: link.StateUnhealthy, Reason: reason, Since: f.Since.UTC(),
|
||||
Streak: f.Streak})
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// failedUnitWords is a failed unit as the console says it.
|
||||
func failedUnitWords(f units.Failed) string {
|
||||
whose := "no module places it"
|
||||
if f.Module != "" {
|
||||
whose = fmt.Sprintf("%s's, by its %s %s", f.Module, f.Via, f.Resource)
|
||||
}
|
||||
return fmt.Sprintf("the unit %s (%s) failed%s: %s", f.Unit, f.Scope, orNothing(" ("+f.Result+")", f.Result), whose)
|
||||
}
|
||||
|
||||
func orNothing(s, unless string) string {
|
||||
if unless == "" {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq
|
||||
// to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is
|
||||
// waiting is that delivery's apply.
|
||||
@@ -1775,7 +1838,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
ns := n.Last()
|
||||
netSt = &ns
|
||||
}
|
||||
report.Health = healthAsReported(st, netSt)
|
||||
var unitSt *units.Statement
|
||||
if u := unitJudge; u != nil {
|
||||
u.Set(units.OwnedBy(declared, held))
|
||||
us := u.Last()
|
||||
unitSt = &us
|
||||
}
|
||||
report.Health = withUnits(healthAsReported(st, netSt), unitSt)
|
||||
}
|
||||
// Which of this machine's links face outside, for the filter the mesh writes around them
|
||||
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/units"
|
||||
)
|
||||
|
||||
// The machine's failed units in the engine's statement (novox/hq issue 315): a module's is among the
|
||||
// resources, unhealthy, as that module's unit, naming it; the machine's own beside them; and an engine
|
||||
// not reading them says nothing of them.
|
||||
func TestTheStatementCarriesTheFailedUnits(t *testing.T) {
|
||||
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
us := &units.Statement{At: at, State: units.Degraded, Failed: []units.Failed{
|
||||
{Unit: "openrazer-daemon.service", Scope: units.ScopeUser, Load: "loaded", Result: "exit-code",
|
||||
Module: "openrazer", Resource: "openrazer.daemon", Via: units.ViaPackage, Since: at, Streak: 2},
|
||||
{Unit: "storage-media.mount", Scope: units.ScopeSystem, Load: "loaded", Result: "timeout", Since: at, Streak: 2},
|
||||
}}
|
||||
h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), us)
|
||||
if len(h.Resources) != 1 {
|
||||
t.Fatalf("one module's unit among the resources: %+v", h.Resources)
|
||||
}
|
||||
r := h.Resources[0]
|
||||
if r.Module != "openrazer" || r.Resource != "openrazer.daemon" || r.Kind != link.KindUnit ||
|
||||
r.Target != "openrazer-daemon.service" || r.State != link.StateUnhealthy ||
|
||||
!strings.Contains(r.Reason, "account's own service manager") || !strings.Contains(r.Reason, "exit-code") {
|
||||
t.Fatalf("the module's failed unit: %+v", r)
|
||||
}
|
||||
if h.Units == nil || h.Units.State != units.Degraded || len(h.Units.Failed) != 1 ||
|
||||
h.Units.Failed[0].Unit != "storage-media.mount" || h.Units.Failed[0].Result != "timeout" {
|
||||
t.Fatalf("the machine's own: %+v", h.Units)
|
||||
}
|
||||
if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), nil); h.Units != nil {
|
||||
t.Fatal("an engine not reading units said them")
|
||||
}
|
||||
if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), &units.Statement{}); h.Units != nil {
|
||||
t.Fatal("a judge not yet given a declaration said units")
|
||||
}
|
||||
}
|
||||
@@ -242,6 +242,41 @@ type Health struct {
|
||||
// file, its names, its tunnel, its bus and its route. Absent from an engine older than that judging,
|
||||
// which the controller reads as "not known", never as healthy.
|
||||
Network *NetworkHealth `json:"network,omitempty"`
|
||||
// Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any
|
||||
// unit failed, and each failed unit no module places. A failed unit a module states, writes or
|
||||
// installs is said among Resources instead, as that module's, of kind KindUnit. Absent from an engine
|
||||
// older than that reading, which the controller reads as "not known", never as healthy.
|
||||
Units *UnitsHealth `json:"units,omitempty"`
|
||||
}
|
||||
|
||||
// KindUnit is a module's unit its service manager says failed (issue 315): a resource of the module
|
||||
// that is not long-running, or not stated running — a package's unit, a unit file the mesh wrote, a
|
||||
// service whose lifecycle is the machine's — said unhealthy for as long as it stays failed.
|
||||
const KindUnit = "unit"
|
||||
|
||||
// UnitsHealth is the machine's service managers in one statement (issue 315).
|
||||
type UnitsHealth struct {
|
||||
// State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no
|
||||
// manager could be read.
|
||||
State string `json:"state"`
|
||||
// Failed is every unit failed on two looks in a row that no module places: the machine's own.
|
||||
Failed []FailedUnit `json:"failed"`
|
||||
// Unread names a manager that could not be read, with why.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
}
|
||||
|
||||
// FailedUnit is one failed unit no module places.
|
||||
type FailedUnit struct {
|
||||
Unit string `json:"unit"`
|
||||
// Scope is system, or user: an account's own manager.
|
||||
Scope string `json:"scope"`
|
||||
// Load is loaded, not-found — a unit whose file is gone and whose failure its manager still holds…
|
||||
Load string `json:"load,omitempty"`
|
||||
// Result is how it failed: exit-code, timeout, start-limit-hit…
|
||||
Result string `json:"result,omitempty"`
|
||||
// Resource is the mesh's own resource that names it, when the mesh placed it in its own right.
|
||||
Resource string `json:"resource,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package units
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Runner runs a command and answers what it printed — the apply's own (apply.ExecRunner), so a look reads
|
||||
// the machine exactly as an apply does.
|
||||
type Runner func(ctx context.Context, name string, args ...string) (string, error)
|
||||
|
||||
// Exec reads the service managers through systemctl and the package database through the system's own
|
||||
// owner query. **Reads only**: `list-units`, `show` and the owner query are the whole of what it asks
|
||||
// (ADR 0240 rule 6, and a test holds it).
|
||||
type Exec struct {
|
||||
Run Runner
|
||||
// System is what the host was built for; it says how a file's package is asked. A system with no
|
||||
// owner query answers "no package", so a unit there is the machine's unless the declaration states it
|
||||
// or writes its file.
|
||||
System string
|
||||
}
|
||||
|
||||
// managerArgs is how systemctl is pointed at a manager: the machine's, or an account's own.
|
||||
func managerArgs(scope, user string) []string {
|
||||
if scope == ScopeUser && user != "" {
|
||||
return []string{"--user", "--machine=" + user + "@"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Failed lists the failed units of one manager, in its plain form: one per line, the unit, its load, its
|
||||
// active and sub state, and its description. A leading mark some versions print before a unit in trouble
|
||||
// is skipped.
|
||||
func (e Exec) Failed(ctx context.Context, scope, user string) ([]Listed, error) {
|
||||
args := append(managerArgs(scope, user), "list-units", "--state=failed", "--all", "--plain", "--no-legend",
|
||||
"--no-pager", "--full")
|
||||
said, err := e.Run(ctx, "systemctl", args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the service manager could not be read: %w", err)
|
||||
}
|
||||
return parseFailed(said), nil
|
||||
}
|
||||
|
||||
func parseFailed(said string) []Listed {
|
||||
var out []Listed
|
||||
for _, line := range strings.Split(said, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
for len(fields) > 0 && (fields[0] == "●" || fields[0] == "*" || fields[0] == "×") {
|
||||
fields = fields[1:]
|
||||
}
|
||||
if len(fields) < 2 || !strings.Contains(fields[0], ".") {
|
||||
continue
|
||||
}
|
||||
out = append(out, Listed{Unit: fields[0], Load: fields[1]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Show is one show of the units named: each one's file and how it failed.
|
||||
func (e Exec) Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error) {
|
||||
out := map[string]Shown{}
|
||||
if len(units) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
args := append(managerArgs(scope, user), "show", "--property=Id,FragmentPath,Result", "--")
|
||||
said, err := e.Run(ctx, "systemctl", append(args, units...)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the service manager could not be read: %w", err)
|
||||
}
|
||||
blocks := strings.Split(strings.TrimSpace(said), "\n\n")
|
||||
if len(blocks) != len(units) {
|
||||
return nil, errors.New("the service manager answered for a different number of units than were asked")
|
||||
}
|
||||
for i, block := range blocks {
|
||||
props := map[string]string{}
|
||||
for _, line := range strings.Split(block, "\n") {
|
||||
if k, v, ok := strings.Cut(line, "="); ok {
|
||||
props[strings.TrimSpace(k)] = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
out[units[i]] = Shown{FragmentPath: props["FragmentPath"], Result: props["Result"]}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PackageOwning asks the package database which package a file belongs to. pacman exits 1 both for a
|
||||
// file no package owns and for a database it cannot read; it is asked about itself first, so the second
|
||||
// is an error and only the first is "no package" (system/arch.go's two-step, for the same trap).
|
||||
func (e Exec) PackageOwning(ctx context.Context, path string) (string, bool, error) {
|
||||
switch e.System {
|
||||
case "arch":
|
||||
if _, err := e.Run(ctx, "pacman", "-Q", "pacman"); err != nil {
|
||||
return "", false, fmt.Errorf("the package database does not answer: %w", err)
|
||||
}
|
||||
said, err := e.Run(ctx, "pacman", "-Qqo", path)
|
||||
if err != nil {
|
||||
return "", false, nil
|
||||
}
|
||||
pkg := strings.TrimSpace(firstLine(said))
|
||||
return pkg, pkg != "", nil
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
// Package units is the node-engine reading which units its machine's service managers say failed, and
|
||||
// whose each is (novox/hq issue 315, under ADR 0240 rule 1 and ADR 0241 §3).
|
||||
//
|
||||
// **A failed unit of a mesh module was never raised.** Liveness judges what a module runs long-lived — a
|
||||
// container, a process, a service stated `running` — and nothing else. A module that installs a daemon as
|
||||
// a package, whose unit the package ships and D-Bus activation starts, declares no service: its unit
|
||||
// failed at every start and the machine read healthy, while the profile's `service-manager` said
|
||||
// `degraded` and nothing raised that either. Two network mounts the operator wrote into the machine's
|
||||
// own mount table, and a unit a removed package left behind, failed beside it, said by nobody.
|
||||
//
|
||||
// On every look the engine asks each service manager the mesh places units in — the machine's, and the
|
||||
// account manager of every account the declaration names — which units failed, and says each one's
|
||||
// owner:
|
||||
//
|
||||
// 1. a service or process the declaration states, by its unit and manager;
|
||||
// 2. a file the declaration writes, when the unit's file is that file;
|
||||
// 3. a package the declaration installs, when the unit's file belongs to it — asked of the package
|
||||
// manager, once per unit file;
|
||||
//
|
||||
// and otherwise nobody's in the mesh: the machine's. A unit liveness already judges is left to it, so a
|
||||
// failure is said once. **The two-look rule is the engine's** (ADR 0241 §2): a unit is said failed on
|
||||
// its second look in a row, and no longer on its first look not failed.
|
||||
//
|
||||
// **It reads; it never acts** (ADR 0240 rule 6): `list-units`, `show` and the package manager's owner
|
||||
// query are the whole of what it asks. It neither resets nor restarts anything.
|
||||
package units
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// The managers a unit is in.
|
||||
const (
|
||||
ScopeSystem = declaration.ScopeSystem
|
||||
ScopeUser = declaration.ScopeUser
|
||||
)
|
||||
|
||||
// How a unit's owner was found.
|
||||
const (
|
||||
ViaUnit = "unit"
|
||||
ViaFile = "file"
|
||||
ViaPackage = "package"
|
||||
)
|
||||
|
||||
// The machine's service managers, as the statement says them.
|
||||
const (
|
||||
// Running is every manager read and no unit failed.
|
||||
Running = "running"
|
||||
// Degraded is a unit failed in a manager read.
|
||||
Degraded = "degraded"
|
||||
// Unknown is no manager could be read.
|
||||
Unknown = "unknown"
|
||||
)
|
||||
|
||||
// owner is a module and the id of its resource that places a unit.
|
||||
type owner struct{ module, resource string }
|
||||
|
||||
// Owned is what a declaration places on the machine, as the reading needs it: the units it states, the
|
||||
// files and packages it puts there, and the accounts whose managers it places units in.
|
||||
type Owned struct {
|
||||
// Units is keyed by manager and unit (key).
|
||||
Units map[string]owner
|
||||
// Judged is every unit liveness judges, by the same key: left to it.
|
||||
Judged map[string]bool
|
||||
// Files is keyed by path; Packages by package name, only those declared present.
|
||||
Files map[string]owner
|
||||
Packages map[string]owner
|
||||
// Accounts are the accounts whose own managers are read, sorted.
|
||||
Accounts []string
|
||||
}
|
||||
|
||||
// key is a unit in one manager: "system/<unit>", or "user:<account>/<unit>".
|
||||
func key(scope, user, unit string) string {
|
||||
if scope == ScopeUser {
|
||||
return "user:" + user + "/" + unit
|
||||
}
|
||||
return "system/" + unit
|
||||
}
|
||||
|
||||
// OwnedBy reads what a declaration places. held is every resource an adopted machine holds as found,
|
||||
// by id — the machine's, not a module's. A resource the mesh declares in its own right (no module) names
|
||||
// no module: its failed unit is said with the machine's, under the resource's id.
|
||||
func OwnedBy(d *declaration.Declaration, held map[string]bool) Owned {
|
||||
o := Owned{Units: map[string]owner{}, Judged: map[string]bool{}, Files: map[string]owner{},
|
||||
Packages: map[string]owner{}}
|
||||
if d == nil {
|
||||
return o
|
||||
}
|
||||
accounts := map[string]bool{}
|
||||
for _, r := range d.Resources {
|
||||
if held[r.Identity()] || strings.HasPrefix(r.Identity(), declaration.AdoptionPrefix) {
|
||||
continue
|
||||
}
|
||||
own := ownerOf(r.Identity())
|
||||
switch v := r.(type) {
|
||||
case *declaration.Service:
|
||||
scope, user := ScopeSystem, ""
|
||||
if v.UserScoped() {
|
||||
scope, user = ScopeUser, v.User
|
||||
accounts[v.User] = true
|
||||
}
|
||||
k := key(scope, user, v.Unit)
|
||||
o.Units[k] = own
|
||||
if v.State == "running" {
|
||||
o.Judged[k] = true
|
||||
}
|
||||
case *declaration.Process:
|
||||
k := key(ScopeSystem, "", v.Name+".service")
|
||||
o.Units[k] = own
|
||||
if !v.RunOnce && v.Schedule == "" {
|
||||
o.Judged[k] = true
|
||||
}
|
||||
case *declaration.File:
|
||||
o.Files[v.Path] = own
|
||||
case *declaration.Package:
|
||||
if !v.Absent {
|
||||
o.Packages[v.Package] = own
|
||||
}
|
||||
case *declaration.User:
|
||||
accounts[v.Name] = true
|
||||
}
|
||||
}
|
||||
for a := range accounts {
|
||||
if a != "" {
|
||||
o.Accounts = append(o.Accounts, a)
|
||||
}
|
||||
}
|
||||
sort.Strings(o.Accounts)
|
||||
return o
|
||||
}
|
||||
|
||||
// ownerOf is a resource id's module and the id itself: everything before the last dot is the module (as
|
||||
// liveness.ModuleOf reads it); an id with no dot is the mesh's own, and names no module.
|
||||
func ownerOf(id string) owner {
|
||||
at := strings.LastIndex(id, ".")
|
||||
if at <= 0 {
|
||||
return owner{resource: id}
|
||||
}
|
||||
return owner{module: id[:at], resource: id}
|
||||
}
|
||||
|
||||
// Listed is one failed unit as its manager lists it.
|
||||
type Listed struct {
|
||||
Unit string
|
||||
// Load is loaded, not-found, masked, bad-setting…
|
||||
Load string
|
||||
}
|
||||
|
||||
// Shown is what the manager says of one unit's file and how it failed.
|
||||
type Shown struct {
|
||||
FragmentPath string
|
||||
// Result is how it failed: exit-code, timeout, start-limit-hit…
|
||||
Result string
|
||||
}
|
||||
|
||||
// Reader is what a look asks the machine. An error is "could not be read": that manager is said unread,
|
||||
// never healthy and never failed.
|
||||
type Reader interface {
|
||||
// Failed is every unit in failed state in a manager: the machine's (user empty), or an account's.
|
||||
Failed(ctx context.Context, scope, user string) ([]Listed, error)
|
||||
// Show is each unit's file and result, in a manager.
|
||||
Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error)
|
||||
// PackageOwning is the package a file belongs to; false when none does or the machine cannot say.
|
||||
PackageOwning(ctx context.Context, path string) (string, bool, error)
|
||||
}
|
||||
|
||||
// Failed is one failed unit as the statement says it.
|
||||
type Failed struct {
|
||||
Unit string
|
||||
Scope string
|
||||
// User is the account whose manager it is in, for a user unit: evidence inside the mesh.
|
||||
User string
|
||||
Load string
|
||||
Result string
|
||||
// Module and Resource own it; empty when no module does. Via is how that was found.
|
||||
Module string
|
||||
Resource string
|
||||
Via string
|
||||
// Since is the first look that found it failed; Streak the looks in a row since.
|
||||
Since time.Time
|
||||
Streak int
|
||||
}
|
||||
|
||||
// Statement is one look at every manager.
|
||||
type Statement struct {
|
||||
At time.Time
|
||||
// State is the worst of the managers read: running, degraded, or unknown when none was.
|
||||
State string
|
||||
// Failed is every unit failed on two looks in a row and not judged by liveness: the modules' and the
|
||||
// machine's.
|
||||
Failed []Failed
|
||||
// Unread names each manager that could not be read, with why.
|
||||
Unread []string
|
||||
}
|
||||
|
||||
// Owned answers the failures a module owns; Unowned those no module does.
|
||||
func (s Statement) Owned() []Failed { return s.filter(true) }
|
||||
func (s Statement) Unowned() []Failed { return s.filter(false) }
|
||||
|
||||
func (s Statement) filter(owned bool) []Failed {
|
||||
var out []Failed
|
||||
for _, f := range s.Failed {
|
||||
if (f.Module != "") == owned {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seen is what the judge keeps of one failed unit between looks.
|
||||
type seen struct {
|
||||
since time.Time
|
||||
streak int
|
||||
}
|
||||
|
||||
// Judge reads the machine's failed units on every look. Safe for the apply and the looking loop at once.
|
||||
type Judge struct {
|
||||
reader Reader
|
||||
Now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
owned Owned
|
||||
// known says a declaration was set: before it, nothing is read — every unit would read as the
|
||||
// machine's, and then as a module's a moment later.
|
||||
known bool
|
||||
seen map[string]*seen
|
||||
owners map[string]ownerAnswer
|
||||
said string
|
||||
last Statement
|
||||
}
|
||||
|
||||
// ownerAnswer is the package manager's answer for one unit file, kept until the declaration changes.
|
||||
type ownerAnswer struct {
|
||||
pkg string
|
||||
ok bool
|
||||
}
|
||||
|
||||
// New is a judge reading through r.
|
||||
func New(r Reader) *Judge {
|
||||
return &Judge{reader: r, Now: time.Now, seen: map[string]*seen{}, owners: map[string]ownerAnswer{}}
|
||||
}
|
||||
|
||||
// Set is what the declaration just applied places. The package manager is asked afresh after it, since a
|
||||
// package installed or removed changes whose a unit file is.
|
||||
func (j *Judge) Set(o Owned) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
j.owned, j.known = o, true
|
||||
j.owners = map[string]ownerAnswer{}
|
||||
}
|
||||
|
||||
// Last is the statement of the last look.
|
||||
func (j *Judge) Last() Statement {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.last
|
||||
}
|
||||
|
||||
// manager is one service manager read.
|
||||
type manager struct{ scope, user string }
|
||||
|
||||
func (m manager) String() string {
|
||||
if m.scope == ScopeUser {
|
||||
return "the account manager of " + m.user
|
||||
}
|
||||
return "the machine's service manager"
|
||||
}
|
||||
|
||||
// Look reads every manager once and answers the statement, and whether what it says changed since the
|
||||
// last look. Before a declaration is set it reads nothing and answers an empty statement, which says
|
||||
// nothing of the units.
|
||||
func (j *Judge) Look(ctx context.Context) (Statement, bool) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if !j.known {
|
||||
return Statement{}, false
|
||||
}
|
||||
now := j.Now()
|
||||
managers := []manager{{scope: ScopeSystem}}
|
||||
for _, a := range j.owned.Accounts {
|
||||
managers = append(managers, manager{scope: ScopeUser, user: a})
|
||||
}
|
||||
st := Statement{At: now, State: Unknown}
|
||||
read := 0
|
||||
failedNow := map[string]bool{}
|
||||
for _, m := range managers {
|
||||
listed, err := j.reader.Failed(ctx, m.scope, m.user)
|
||||
if err != nil {
|
||||
st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error())))
|
||||
// What it held is neither cleared nor counted while it cannot be read.
|
||||
for k := range j.seen {
|
||||
if strings.HasPrefix(k, key(m.scope, m.user, "")) {
|
||||
failedNow[k] = true
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
read++
|
||||
if st.State == Unknown {
|
||||
st.State = Running
|
||||
}
|
||||
if len(listed) > 0 {
|
||||
st.State = Degraded
|
||||
}
|
||||
var names []string
|
||||
for _, l := range listed {
|
||||
names = append(names, l.Unit)
|
||||
}
|
||||
var shown map[string]Shown
|
||||
if len(names) > 0 {
|
||||
if shown, err = j.reader.Show(ctx, m.scope, m.user, names); err != nil {
|
||||
shown = map[string]Shown{}
|
||||
}
|
||||
}
|
||||
for _, l := range listed {
|
||||
k := key(m.scope, m.user, l.Unit)
|
||||
failedNow[k] = true
|
||||
s := j.seen[k]
|
||||
if s == nil {
|
||||
s = &seen{since: now}
|
||||
j.seen[k] = s
|
||||
}
|
||||
s.streak++
|
||||
if j.owned.Judged[k] || s.streak < 2 {
|
||||
continue
|
||||
}
|
||||
f := Failed{Unit: l.Unit, Scope: m.scope, User: m.user, Load: l.Load, Result: shown[l.Unit].Result,
|
||||
Since: s.since, Streak: s.streak}
|
||||
if own, via, ok := j.ownerOfUnit(ctx, k, shown[l.Unit].FragmentPath); ok {
|
||||
f.Module, f.Resource, f.Via = own.module, own.resource, via
|
||||
}
|
||||
st.Failed = append(st.Failed, f)
|
||||
}
|
||||
}
|
||||
for k := range j.seen {
|
||||
if !failedNow[k] {
|
||||
delete(j.seen, k)
|
||||
}
|
||||
}
|
||||
sort.Slice(st.Failed, func(a, b int) bool {
|
||||
if st.Failed[a].Scope != st.Failed[b].Scope {
|
||||
return st.Failed[a].Scope < st.Failed[b].Scope
|
||||
}
|
||||
return st.Failed[a].Unit < st.Failed[b].Unit
|
||||
})
|
||||
if read == 0 {
|
||||
st.State = Unknown
|
||||
}
|
||||
word := st.State
|
||||
for _, f := range st.Failed {
|
||||
word += "|" + f.Scope + "/" + f.Unit + "/" + f.Module
|
||||
}
|
||||
changed := word != j.said
|
||||
j.said, j.last = word, st
|
||||
return st, changed
|
||||
}
|
||||
|
||||
// ownerOfUnit is whose a failed unit is: the declaration's unit, then the file the unit is, then the
|
||||
// package the file belongs to.
|
||||
func (j *Judge) ownerOfUnit(ctx context.Context, k, fragment string) (owner, string, bool) {
|
||||
if o, ok := j.owned.Units[k]; ok {
|
||||
return o, ViaUnit, true
|
||||
}
|
||||
if fragment == "" {
|
||||
return owner{}, "", false
|
||||
}
|
||||
if o, ok := j.owned.Files[fragment]; ok {
|
||||
return o, ViaFile, true
|
||||
}
|
||||
if len(j.owned.Packages) == 0 {
|
||||
return owner{}, "", false
|
||||
}
|
||||
a, asked := j.owners[fragment]
|
||||
if !asked {
|
||||
pkg, ok, err := j.reader.PackageOwning(ctx, fragment)
|
||||
if err != nil {
|
||||
// Not kept: asked again on the next look.
|
||||
return owner{}, "", false
|
||||
}
|
||||
a = ownerAnswer{pkg: pkg, ok: ok}
|
||||
j.owners[fragment] = a
|
||||
}
|
||||
if !a.ok {
|
||||
return owner{}, "", false
|
||||
}
|
||||
if o, ok := j.owned.Packages[a.pkg]; ok {
|
||||
return o, ViaPackage, true
|
||||
}
|
||||
return owner{}, "", false
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
return line
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package units
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// The reading of failed units (novox/hq issue 315, "how it is checked"): a failed unit a module states,
|
||||
// writes or installs is that module's, in either manager; one no module places is the machine's; a unit
|
||||
// liveness judges is left to it; the two-look rule holds both ways; a manager that cannot be read is
|
||||
// unread, never healthy; and only reads are asked.
|
||||
|
||||
type fakeReader struct {
|
||||
failed map[string][]Listed // by "system" or "user:<account>"
|
||||
shown map[string]Shown // by unit
|
||||
owners map[string]string // by path
|
||||
unread map[string]error
|
||||
ownerQs int
|
||||
}
|
||||
|
||||
func mgr(scope, user string) string {
|
||||
if scope == ScopeUser {
|
||||
return "user:" + user
|
||||
}
|
||||
return "system"
|
||||
}
|
||||
|
||||
func (f *fakeReader) Failed(_ context.Context, scope, user string) ([]Listed, error) {
|
||||
if err := f.unread[mgr(scope, user)]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.failed[mgr(scope, user)], nil
|
||||
}
|
||||
|
||||
func (f *fakeReader) Show(_ context.Context, _, _ string, units []string) (map[string]Shown, error) {
|
||||
out := map[string]Shown{}
|
||||
for _, u := range units {
|
||||
out[u] = f.shown[u]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeReader) PackageOwning(_ context.Context, path string) (string, bool, error) {
|
||||
f.ownerQs++
|
||||
p, ok := f.owners[path]
|
||||
return p, ok, nil
|
||||
}
|
||||
|
||||
// shanks is the desktop as found on 2026-10-08: two mounts of the machine's own mount table, the
|
||||
// Razer daemon's packaged user unit, and a unit a removed package left behind.
|
||||
func shanks() *fakeReader {
|
||||
return &fakeReader{
|
||||
failed: map[string][]Listed{
|
||||
"system": {{Unit: "mnt-recalbox.mount", Load: "loaded"}, {Unit: "storage-media.mount", Load: "loaded"}},
|
||||
"user:operator": {{Unit: "greenclip.service", Load: "not-found"},
|
||||
{Unit: "openrazer-daemon.service", Load: "loaded"}},
|
||||
},
|
||||
shown: map[string]Shown{
|
||||
"mnt-recalbox.mount": {FragmentPath: "/run/systemd/generator/mnt-recalbox.mount", Result: "exit-code"},
|
||||
"storage-media.mount": {FragmentPath: "/run/systemd/generator/storage-media.mount", Result: "timeout"},
|
||||
"greenclip.service": {},
|
||||
"openrazer-daemon.service": {FragmentPath: "/usr/lib/systemd/user/openrazer-daemon.service", Result: "exit-code"},
|
||||
},
|
||||
owners: map[string]string{"/usr/lib/systemd/user/openrazer-daemon.service": "openrazer-daemon"},
|
||||
}
|
||||
}
|
||||
|
||||
func declared() *declaration.Declaration {
|
||||
return &declaration.Declaration{Resources: []declaration.Resource{
|
||||
&declaration.Package{ID: "openrazer.daemon", Type: declaration.TypePackage, Package: "openrazer-daemon"},
|
||||
&declaration.Package{ID: "clipmenu.greenclip", Type: declaration.TypePackage, Package: "rofi-greenclip", Absent: true},
|
||||
&declaration.User{ID: "zsh.account", Type: declaration.TypeUser, Name: "operator"},
|
||||
&declaration.Service{ID: "sshd.run", Type: declaration.TypeService, Unit: "sshd.service", State: "running"},
|
||||
&declaration.Service{ID: "power.after-boot", Type: declaration.TypeService, Unit: "mesh-power-after-boot.service"},
|
||||
&declaration.File{ID: "watcher.unit", Type: declaration.TypeFile, Path: "/home/operator/.config/systemd/user/watcher.service"},
|
||||
}}
|
||||
}
|
||||
|
||||
var t0 = time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func lookTwice(t *testing.T, j *Judge) Statement {
|
||||
t.Helper()
|
||||
now := t0
|
||||
j.Now = func() time.Time { return now }
|
||||
first, _ := j.Look(context.Background())
|
||||
if len(first.Failed) != 0 {
|
||||
t.Fatalf("a unit was said failed on its first look: %+v", first.Failed)
|
||||
}
|
||||
now = now.Add(15 * time.Second)
|
||||
st, _ := j.Look(context.Background())
|
||||
return st
|
||||
}
|
||||
|
||||
func TestTheModulesAndTheMachinesFailures(t *testing.T) {
|
||||
r := shanks()
|
||||
j := New(r)
|
||||
j.Set(OwnedBy(declared(), nil))
|
||||
st := lookTwice(t, j)
|
||||
if st.State != Degraded {
|
||||
t.Fatalf("state %q, want degraded", st.State)
|
||||
}
|
||||
owned := st.Owned()
|
||||
if len(owned) != 1 || owned[0].Unit != "openrazer-daemon.service" || owned[0].Module != "openrazer" ||
|
||||
owned[0].Resource != "openrazer.daemon" || owned[0].Via != ViaPackage || owned[0].Scope != ScopeUser {
|
||||
t.Fatalf("the Razer daemon's packaged user unit is openrazer's: %+v", owned)
|
||||
}
|
||||
if owned[0].Since != t0 || owned[0].Streak != 2 || owned[0].Result != "exit-code" {
|
||||
t.Fatalf("since the first look that found it, two in a row, how it failed: %+v", owned[0])
|
||||
}
|
||||
var machine []string
|
||||
for _, f := range st.Unowned() {
|
||||
machine = append(machine, f.Scope+"/"+f.Unit)
|
||||
}
|
||||
want := "system/mnt-recalbox.mount system/storage-media.mount user/greenclip.service"
|
||||
if strings.Join(machine, " ") != want {
|
||||
t.Fatalf("the machine's own: %v, want %s", machine, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeclaredUnitAndAWrittenFileAreTheirModules(t *testing.T) {
|
||||
r := &fakeReader{
|
||||
failed: map[string][]Listed{
|
||||
"system": {{Unit: "mesh-power-after-boot.service", Load: "loaded"}, {Unit: "sshd.service", Load: "loaded"}},
|
||||
"user:operator": {{Unit: "watcher.service", Load: "loaded"}},
|
||||
},
|
||||
shown: map[string]Shown{
|
||||
"watcher.service": {FragmentPath: "/home/operator/.config/systemd/user/watcher.service"},
|
||||
},
|
||||
}
|
||||
j := New(r)
|
||||
j.Set(OwnedBy(declared(), nil))
|
||||
st := lookTwice(t, j)
|
||||
got := map[string]string{}
|
||||
for _, f := range st.Failed {
|
||||
got[f.Unit] = f.Module + " " + f.Via
|
||||
}
|
||||
if got["mesh-power-after-boot.service"] != "power unit" {
|
||||
t.Errorf("a stateless service the declaration names is its module's: %q", got["mesh-power-after-boot.service"])
|
||||
}
|
||||
if got["watcher.service"] != "watcher file" {
|
||||
t.Errorf("a user unit the mesh wrote is its module's: %q", got["watcher.service"])
|
||||
}
|
||||
if _, said := got["sshd.service"]; said {
|
||||
t.Errorf("a service stated running is liveness's to judge, and said once: %v", got)
|
||||
}
|
||||
if r.ownerQs != 0 {
|
||||
t.Errorf("the package database was asked %d time(s) for units the declaration already names", r.ownerQs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoLooksEachWay(t *testing.T) {
|
||||
r := shanks()
|
||||
j := New(r)
|
||||
j.Set(OwnedBy(declared(), nil))
|
||||
now := t0
|
||||
j.Now = func() time.Time { return now }
|
||||
st, changed := j.Look(context.Background())
|
||||
if len(st.Failed) != 0 || !changed {
|
||||
t.Fatalf("first look: nothing said failed, the state said: %+v %v", st.Failed, changed)
|
||||
}
|
||||
now = now.Add(15 * time.Second)
|
||||
if st, changed = j.Look(context.Background()); len(st.Failed) != 4 || !changed {
|
||||
t.Fatalf("second look: four said failed: %+v %v", st.Failed, changed)
|
||||
}
|
||||
now = now.Add(15 * time.Second)
|
||||
if _, changed = j.Look(context.Background()); changed {
|
||||
t.Fatal("a third look the same is no change")
|
||||
}
|
||||
// The operator mounts the media library again: no longer failed on the first look that says so.
|
||||
r.failed["system"] = r.failed["system"][:1]
|
||||
now = now.Add(15 * time.Second)
|
||||
st, changed = j.Look(context.Background())
|
||||
if len(st.Failed) != 3 || !changed {
|
||||
t.Fatalf("a unit no longer failed is no longer said: %+v", st.Failed)
|
||||
}
|
||||
// Every manager clean: running.
|
||||
r.failed = map[string][]Listed{}
|
||||
now = now.Add(15 * time.Second)
|
||||
if st, _ = j.Look(context.Background()); st.State != Running || len(st.Failed) != 0 {
|
||||
t.Fatalf("no failed unit is running: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManagerThatCannotBeReadIsUnread(t *testing.T) {
|
||||
r := shanks()
|
||||
j := New(r)
|
||||
j.Set(OwnedBy(declared(), nil))
|
||||
lookTwice(t, j)
|
||||
r.unread = map[string]error{"user:operator": errors.New("Failed to connect to bus: No medium found")}
|
||||
st, _ := j.Look(context.Background())
|
||||
if len(st.Unread) != 1 || !strings.Contains(st.Unread[0], "operator") {
|
||||
t.Fatalf("the account manager is said unread: %v", st.Unread)
|
||||
}
|
||||
// Its failures are not counted while unread, and not forgotten either: back on the next read, said
|
||||
// at once, from when they were first found.
|
||||
r.unread = nil
|
||||
st, _ = j.Look(context.Background())
|
||||
for _, f := range st.Failed {
|
||||
if f.Unit == "openrazer-daemon.service" && f.Since == t0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("a manager read again says what it held, from when it was first found: %+v", st.Failed)
|
||||
}
|
||||
|
||||
func TestNoManagerReadIsUnknown(t *testing.T) {
|
||||
r := &fakeReader{unread: map[string]error{"system": errors.New("systemctl: not found"),
|
||||
"user:operator": errors.New("systemctl: not found")}}
|
||||
j := New(r)
|
||||
j.Set(OwnedBy(declared(), nil))
|
||||
if st, _ := j.Look(context.Background()); st.State != Unknown {
|
||||
t.Fatalf("no manager read is unknown, never running: %q", st.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedMachinesHoldingIsTheMachines(t *testing.T) {
|
||||
d := &declaration.Declaration{Resources: []declaration.Resource{
|
||||
&declaration.Service{ID: "found.cups", Type: declaration.TypeService, Unit: "cups.service"},
|
||||
}}
|
||||
o := OwnedBy(d, map[string]bool{"found.cups": true})
|
||||
if _, ok := o.Units["system/cups.service"]; ok {
|
||||
t.Fatal("a unit an adopted machine holds as found is the machine's, not a module's")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheReaderOnlyReads(t *testing.T) {
|
||||
var asked []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
asked = append(asked, name+" "+strings.Join(args, " "))
|
||||
switch {
|
||||
case name == "systemctl" && contains(args, "list-units"):
|
||||
return "● greenclip.service not-found failed failed greenclip.service\n" +
|
||||
"openrazer-daemon.service loaded failed failed Daemon to manage razer devices in userspace\n", nil
|
||||
case name == "systemctl" && contains(args, "show"):
|
||||
return "Id=greenclip.service\nFragmentPath=\nResult=start-limit-hit\n\n" +
|
||||
"Id=openrazer-daemon.service\nFragmentPath=/usr/lib/systemd/user/openrazer-daemon.service\nResult=exit-code\n", nil
|
||||
case name == "pacman" && contains(args, "-Qqo"):
|
||||
return "openrazer-daemon\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
e := Exec{Run: run, System: "arch"}
|
||||
listed, err := e.Failed(context.Background(), ScopeUser, "operator")
|
||||
if err != nil || len(listed) != 2 || listed[0].Unit != "greenclip.service" || listed[0].Load != "not-found" {
|
||||
t.Fatalf("the list, its mark skipped: %+v %v", listed, err)
|
||||
}
|
||||
shown, err := e.Show(context.Background(), ScopeUser, "operator", []string{"greenclip.service", "openrazer-daemon.service"})
|
||||
if err != nil || shown["openrazer-daemon.service"].FragmentPath == "" || shown["greenclip.service"].Result != "start-limit-hit" {
|
||||
t.Fatalf("the show: %+v %v", shown, err)
|
||||
}
|
||||
if pkg, ok, err := e.PackageOwning(context.Background(), "/usr/lib/systemd/user/openrazer-daemon.service"); !ok ||
|
||||
pkg != "openrazer-daemon" || err != nil {
|
||||
t.Fatalf("the owner: %q %v %v", pkg, ok, err)
|
||||
}
|
||||
for _, a := range asked {
|
||||
if !strings.Contains(a, "list-units") && !strings.Contains(a, " show ") && !strings.HasPrefix(a, "pacman -Q") {
|
||||
t.Errorf("asked something that is not a read: %q", a)
|
||||
}
|
||||
if !strings.HasPrefix(a, "pacman") && !strings.Contains(a, "--machine=operator@") {
|
||||
t.Errorf("an account's manager is asked as that account's: %q", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(args []string, s string) bool {
|
||||
for _, a := range args {
|
||||
if a == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestNothingIsReadBeforeADeclaration(t *testing.T) {
|
||||
r := shanks()
|
||||
j := New(r)
|
||||
if st, changed := j.Look(context.Background()); st.State != "" || changed {
|
||||
t.Fatalf("before a declaration every unit would read as the machine's: %+v", st)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user