Say every failed unit, the module's and the machine's (hq issue 315)
mesh/merge-gate pass: builds mesh-host → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group fix/315-a-failed-unit-is-a-condition delivered: every member is delivered

Liveness judged only what a module runs long-lived, so a module whose
daemon is a package's unit started by D-Bus activation failed at every
start while its machine read healthy, and a degraded service manager was
said by nothing but the profile.

The engine now reads the failed units of the machine's manager and of
every account manager the declaration names, on the two-look rule, and
says whose each is: a declared service or process, a unit file the mesh
writes, or a package the mesh installs makes it that module's, said as
an unhealthy resource of kind unit; anything else is the machine's own,
said in the statement's new units field. It reads; it never acts.
This commit is contained in:
jochen
2026-10-08 11:28:51 +02:00
parent f863adb741
commit 14e5d91c9a
6 changed files with 942 additions and 3 deletions
+72 -3
View File
@@ -43,6 +43,7 @@ import (
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
"github.com/novox/mesh-host/internal/tunnel"
"github.com/novox/mesh-host/internal/units"
"github.com/novox/mesh-host/internal/upgrade"
)
@@ -1090,6 +1091,8 @@ func runLink(ctx context.Context, opts options) error {
return held
}
judging = j
// And which units its service managers say failed, and whose each is (novox/hq issue 315).
unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
}
// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
@@ -1384,6 +1387,10 @@ const ReconcileEvery = 5 * time.Minute
// reports no health, and in a test.
var judging *liveness.Judge
// unitJudge reads which units the machine's service managers say failed, and whose each is (novox/hq
// issue 315); nil where judging is.
var unitJudge *units.Judge
// netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a
// one-shot command and in a test, which say nothing of the network.
var netJudge networkJudge
@@ -1478,6 +1485,17 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
}
st, changed := j.Look(ctx)
owed = owed || changed
var unitSt *units.Statement
if u := unitJudge; u != nil {
us, unitsChanged := u.Look(ctx)
unitSt = &us
owed = owed || unitsChanged
if unitsChanged {
for _, f := range us.Failed {
say(failedUnitWords(f))
}
}
}
var netSt *network.Statement
if n := netJudge.get(); n != nil {
ns, netChanged := n.Look(ctx)
@@ -1500,7 +1518,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
spaced = sp
}
since := time.Since(lastSaid)
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy)
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
(unitSt == nil || len(unitSt.Failed) == 0)
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
continue
}
@@ -1512,7 +1531,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
}
}
}
if queue.SayHealth(ctx, *healthAsReported(st, netSt)) {
if queue.SayHealth(ctx, *withUnits(healthAsReported(st, netSt), unitSt)) {
lastSaid, owed = time.Now(), false
}
}
@@ -1538,6 +1557,50 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health
return h
}
// withUnits adds to a statement the machine's failed units (novox/hq issue 315): each a module places,
// among the resources as that module's unhealthy unit; the rest, the machine's own, beside them. Nil — an
// engine not reading them — says nothing of them, which the controller reads as not known.
func withUnits(h *link.Health, us *units.Statement) *link.Health {
if us == nil || us.State == "" {
return h
}
h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread}
for _, f := range us.Failed {
if f.Module == "" {
h.Units.Failed = append(h.Units.Failed, link.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load,
Result: f.Result, Resource: f.Resource, Since: f.Since.UTC()})
continue
}
reason := "failed"
if f.Scope == units.ScopeUser {
reason += " in the account's own service manager"
}
if f.Result != "" {
reason += " (" + f.Result + ")"
}
h.Resources = append(h.Resources, link.ResourceHealth{Module: f.Module, Resource: f.Resource,
Kind: link.KindUnit, Target: f.Unit, State: link.StateUnhealthy, Reason: reason, Since: f.Since.UTC(),
Streak: f.Streak})
}
return h
}
// failedUnitWords is a failed unit as the console says it.
func failedUnitWords(f units.Failed) string {
whose := "no module places it"
if f.Module != "" {
whose = fmt.Sprintf("%s's, by its %s %s", f.Module, f.Via, f.Resource)
}
return fmt.Sprintf("the unit %s (%s) failed%s: %s", f.Unit, f.Scope, orNothing(" ("+f.Result+")", f.Result), whose)
}
func orNothing(s, unless string) string {
if unless == "" {
return ""
}
return s
}
// holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq
// to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is
// waiting is that delivery's apply.
@@ -1775,7 +1838,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
ns := n.Last()
netSt = &ns
}
report.Health = healthAsReported(st, netSt)
var unitSt *units.Statement
if u := unitJudge; u != nil {
u.Set(units.OwnedBy(declared, held))
us := u.Last()
unitSt = &us
}
report.Health = withUnits(healthAsReported(st, netSt), unitSt)
}
// Which of this machine's links face outside, for the filter the mesh writes around them
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/liveness"
"github.com/novox/mesh-host/internal/units"
)
// The machine's failed units in the engine's statement (novox/hq issue 315): a module's is among the
// resources, unhealthy, as that module's unit, naming it; the machine's own beside them; and an engine
// not reading them says nothing of them.
func TestTheStatementCarriesTheFailedUnits(t *testing.T) {
at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
us := &units.Statement{At: at, State: units.Degraded, Failed: []units.Failed{
{Unit: "openrazer-daemon.service", Scope: units.ScopeUser, Load: "loaded", Result: "exit-code",
Module: "openrazer", Resource: "openrazer.daemon", Via: units.ViaPackage, Since: at, Streak: 2},
{Unit: "storage-media.mount", Scope: units.ScopeSystem, Load: "loaded", Result: "timeout", Since: at, Streak: 2},
}}
h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), us)
if len(h.Resources) != 1 {
t.Fatalf("one module's unit among the resources: %+v", h.Resources)
}
r := h.Resources[0]
if r.Module != "openrazer" || r.Resource != "openrazer.daemon" || r.Kind != link.KindUnit ||
r.Target != "openrazer-daemon.service" || r.State != link.StateUnhealthy ||
!strings.Contains(r.Reason, "account's own service manager") || !strings.Contains(r.Reason, "exit-code") {
t.Fatalf("the module's failed unit: %+v", r)
}
if h.Units == nil || h.Units.State != units.Degraded || len(h.Units.Failed) != 1 ||
h.Units.Failed[0].Unit != "storage-media.mount" || h.Units.Failed[0].Result != "timeout" {
t.Fatalf("the machine's own: %+v", h.Units)
}
if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), nil); h.Units != nil {
t.Fatal("an engine not reading units said them")
}
if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), &units.Statement{}); h.Units != nil {
t.Fatal("a judge not yet given a declaration said units")
}
}
+35
View File
@@ -242,6 +242,41 @@ type Health struct {
// file, its names, its tunnel, its bus and its route. Absent from an engine older than that judging,
// which the controller reads as "not known", never as healthy.
Network *NetworkHealth `json:"network,omitempty"`
// Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any
// unit failed, and each failed unit no module places. A failed unit a module states, writes or
// installs is said among Resources instead, as that module's, of kind KindUnit. Absent from an engine
// older than that reading, which the controller reads as "not known", never as healthy.
Units *UnitsHealth `json:"units,omitempty"`
}
// KindUnit is a module's unit its service manager says failed (issue 315): a resource of the module
// that is not long-running, or not stated running — a package's unit, a unit file the mesh wrote, a
// service whose lifecycle is the machine's — said unhealthy for as long as it stays failed.
const KindUnit = "unit"
// UnitsHealth is the machine's service managers in one statement (issue 315).
type UnitsHealth struct {
// State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no
// manager could be read.
State string `json:"state"`
// Failed is every unit failed on two looks in a row that no module places: the machine's own.
Failed []FailedUnit `json:"failed"`
// Unread names a manager that could not be read, with why.
Unread []string `json:"unread,omitempty"`
}
// FailedUnit is one failed unit no module places.
type FailedUnit struct {
Unit string `json:"unit"`
// Scope is system, or user: an account's own manager.
Scope string `json:"scope"`
// Load is loaded, not-found — a unit whose file is gone and whose failure its manager still holds…
Load string `json:"load,omitempty"`
// Result is how it failed: exit-code, timeout, start-limit-hit…
Result string `json:"result,omitempty"`
// Resource is the mesh's own resource that names it, when the mesh placed it in its own right.
Resource string `json:"resource,omitempty"`
Since time.Time `json:"since"`
}
// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since
+105
View File
@@ -0,0 +1,105 @@
package units
import (
"context"
"errors"
"fmt"
"strings"
)
// Runner runs a command and answers what it printed — the apply's own (apply.ExecRunner), so a look reads
// the machine exactly as an apply does.
type Runner func(ctx context.Context, name string, args ...string) (string, error)
// Exec reads the service managers through systemctl and the package database through the system's own
// owner query. **Reads only**: `list-units`, `show` and the owner query are the whole of what it asks
// (ADR 0240 rule 6, and a test holds it).
type Exec struct {
Run Runner
// System is what the host was built for; it says how a file's package is asked. A system with no
// owner query answers "no package", so a unit there is the machine's unless the declaration states it
// or writes its file.
System string
}
// managerArgs is how systemctl is pointed at a manager: the machine's, or an account's own.
func managerArgs(scope, user string) []string {
if scope == ScopeUser && user != "" {
return []string{"--user", "--machine=" + user + "@"}
}
return nil
}
// Failed lists the failed units of one manager, in its plain form: one per line, the unit, its load, its
// active and sub state, and its description. A leading mark some versions print before a unit in trouble
// is skipped.
func (e Exec) Failed(ctx context.Context, scope, user string) ([]Listed, error) {
args := append(managerArgs(scope, user), "list-units", "--state=failed", "--all", "--plain", "--no-legend",
"--no-pager", "--full")
said, err := e.Run(ctx, "systemctl", args...)
if err != nil {
return nil, fmt.Errorf("the service manager could not be read: %w", err)
}
return parseFailed(said), nil
}
func parseFailed(said string) []Listed {
var out []Listed
for _, line := range strings.Split(said, "\n") {
fields := strings.Fields(line)
for len(fields) > 0 && (fields[0] == "●" || fields[0] == "*" || fields[0] == "×") {
fields = fields[1:]
}
if len(fields) < 2 || !strings.Contains(fields[0], ".") {
continue
}
out = append(out, Listed{Unit: fields[0], Load: fields[1]})
}
return out
}
// Show is one show of the units named: each one's file and how it failed.
func (e Exec) Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error) {
out := map[string]Shown{}
if len(units) == 0 {
return out, nil
}
args := append(managerArgs(scope, user), "show", "--property=Id,FragmentPath,Result", "--")
said, err := e.Run(ctx, "systemctl", append(args, units...)...)
if err != nil {
return nil, fmt.Errorf("the service manager could not be read: %w", err)
}
blocks := strings.Split(strings.TrimSpace(said), "\n\n")
if len(blocks) != len(units) {
return nil, errors.New("the service manager answered for a different number of units than were asked")
}
for i, block := range blocks {
props := map[string]string{}
for _, line := range strings.Split(block, "\n") {
if k, v, ok := strings.Cut(line, "="); ok {
props[strings.TrimSpace(k)] = strings.TrimSpace(v)
}
}
out[units[i]] = Shown{FragmentPath: props["FragmentPath"], Result: props["Result"]}
}
return out, nil
}
// PackageOwning asks the package database which package a file belongs to. pacman exits 1 both for a
// file no package owns and for a database it cannot read; it is asked about itself first, so the second
// is an error and only the first is "no package" (system/arch.go's two-step, for the same trap).
func (e Exec) PackageOwning(ctx context.Context, path string) (string, bool, error) {
switch e.System {
case "arch":
if _, err := e.Run(ctx, "pacman", "-Q", "pacman"); err != nil {
return "", false, fmt.Errorf("the package database does not answer: %w", err)
}
said, err := e.Run(ctx, "pacman", "-Qqo", path)
if err != nil {
return "", false, nil
}
pkg := strings.TrimSpace(firstLine(said))
return pkg, pkg != "", nil
}
return "", false, nil
}
+402
View File
@@ -0,0 +1,402 @@
// Package units is the node-engine reading which units its machine's service managers say failed, and
// whose each is (novox/hq issue 315, under ADR 0240 rule 1 and ADR 0241 §3).
//
// **A failed unit of a mesh module was never raised.** Liveness judges what a module runs long-lived — a
// container, a process, a service stated `running` — and nothing else. A module that installs a daemon as
// a package, whose unit the package ships and D-Bus activation starts, declares no service: its unit
// failed at every start and the machine read healthy, while the profile's `service-manager` said
// `degraded` and nothing raised that either. Two network mounts the operator wrote into the machine's
// own mount table, and a unit a removed package left behind, failed beside it, said by nobody.
//
// On every look the engine asks each service manager the mesh places units in — the machine's, and the
// account manager of every account the declaration names — which units failed, and says each one's
// owner:
//
// 1. a service or process the declaration states, by its unit and manager;
// 2. a file the declaration writes, when the unit's file is that file;
// 3. a package the declaration installs, when the unit's file belongs to it — asked of the package
// manager, once per unit file;
//
// and otherwise nobody's in the mesh: the machine's. A unit liveness already judges is left to it, so a
// failure is said once. **The two-look rule is the engine's** (ADR 0241 §2): a unit is said failed on
// its second look in a row, and no longer on its first look not failed.
//
// **It reads; it never acts** (ADR 0240 rule 6): `list-units`, `show` and the package manager's owner
// query are the whole of what it asks. It neither resets nor restarts anything.
package units
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// The managers a unit is in.
const (
ScopeSystem = declaration.ScopeSystem
ScopeUser = declaration.ScopeUser
)
// How a unit's owner was found.
const (
ViaUnit = "unit"
ViaFile = "file"
ViaPackage = "package"
)
// The machine's service managers, as the statement says them.
const (
// Running is every manager read and no unit failed.
Running = "running"
// Degraded is a unit failed in a manager read.
Degraded = "degraded"
// Unknown is no manager could be read.
Unknown = "unknown"
)
// owner is a module and the id of its resource that places a unit.
type owner struct{ module, resource string }
// Owned is what a declaration places on the machine, as the reading needs it: the units it states, the
// files and packages it puts there, and the accounts whose managers it places units in.
type Owned struct {
// Units is keyed by manager and unit (key).
Units map[string]owner
// Judged is every unit liveness judges, by the same key: left to it.
Judged map[string]bool
// Files is keyed by path; Packages by package name, only those declared present.
Files map[string]owner
Packages map[string]owner
// Accounts are the accounts whose own managers are read, sorted.
Accounts []string
}
// key is a unit in one manager: "system/<unit>", or "user:<account>/<unit>".
func key(scope, user, unit string) string {
if scope == ScopeUser {
return "user:" + user + "/" + unit
}
return "system/" + unit
}
// OwnedBy reads what a declaration places. held is every resource an adopted machine holds as found,
// by id — the machine's, not a module's. A resource the mesh declares in its own right (no module) names
// no module: its failed unit is said with the machine's, under the resource's id.
func OwnedBy(d *declaration.Declaration, held map[string]bool) Owned {
o := Owned{Units: map[string]owner{}, Judged: map[string]bool{}, Files: map[string]owner{},
Packages: map[string]owner{}}
if d == nil {
return o
}
accounts := map[string]bool{}
for _, r := range d.Resources {
if held[r.Identity()] || strings.HasPrefix(r.Identity(), declaration.AdoptionPrefix) {
continue
}
own := ownerOf(r.Identity())
switch v := r.(type) {
case *declaration.Service:
scope, user := ScopeSystem, ""
if v.UserScoped() {
scope, user = ScopeUser, v.User
accounts[v.User] = true
}
k := key(scope, user, v.Unit)
o.Units[k] = own
if v.State == "running" {
o.Judged[k] = true
}
case *declaration.Process:
k := key(ScopeSystem, "", v.Name+".service")
o.Units[k] = own
if !v.RunOnce && v.Schedule == "" {
o.Judged[k] = true
}
case *declaration.File:
o.Files[v.Path] = own
case *declaration.Package:
if !v.Absent {
o.Packages[v.Package] = own
}
case *declaration.User:
accounts[v.Name] = true
}
}
for a := range accounts {
if a != "" {
o.Accounts = append(o.Accounts, a)
}
}
sort.Strings(o.Accounts)
return o
}
// ownerOf is a resource id's module and the id itself: everything before the last dot is the module (as
// liveness.ModuleOf reads it); an id with no dot is the mesh's own, and names no module.
func ownerOf(id string) owner {
at := strings.LastIndex(id, ".")
if at <= 0 {
return owner{resource: id}
}
return owner{module: id[:at], resource: id}
}
// Listed is one failed unit as its manager lists it.
type Listed struct {
Unit string
// Load is loaded, not-found, masked, bad-setting…
Load string
}
// Shown is what the manager says of one unit's file and how it failed.
type Shown struct {
FragmentPath string
// Result is how it failed: exit-code, timeout, start-limit-hit…
Result string
}
// Reader is what a look asks the machine. An error is "could not be read": that manager is said unread,
// never healthy and never failed.
type Reader interface {
// Failed is every unit in failed state in a manager: the machine's (user empty), or an account's.
Failed(ctx context.Context, scope, user string) ([]Listed, error)
// Show is each unit's file and result, in a manager.
Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error)
// PackageOwning is the package a file belongs to; false when none does or the machine cannot say.
PackageOwning(ctx context.Context, path string) (string, bool, error)
}
// Failed is one failed unit as the statement says it.
type Failed struct {
Unit string
Scope string
// User is the account whose manager it is in, for a user unit: evidence inside the mesh.
User string
Load string
Result string
// Module and Resource own it; empty when no module does. Via is how that was found.
Module string
Resource string
Via string
// Since is the first look that found it failed; Streak the looks in a row since.
Since time.Time
Streak int
}
// Statement is one look at every manager.
type Statement struct {
At time.Time
// State is the worst of the managers read: running, degraded, or unknown when none was.
State string
// Failed is every unit failed on two looks in a row and not judged by liveness: the modules' and the
// machine's.
Failed []Failed
// Unread names each manager that could not be read, with why.
Unread []string
}
// Owned answers the failures a module owns; Unowned those no module does.
func (s Statement) Owned() []Failed { return s.filter(true) }
func (s Statement) Unowned() []Failed { return s.filter(false) }
func (s Statement) filter(owned bool) []Failed {
var out []Failed
for _, f := range s.Failed {
if (f.Module != "") == owned {
out = append(out, f)
}
}
return out
}
// seen is what the judge keeps of one failed unit between looks.
type seen struct {
since time.Time
streak int
}
// Judge reads the machine's failed units on every look. Safe for the apply and the looking loop at once.
type Judge struct {
reader Reader
Now func() time.Time
mu sync.Mutex
owned Owned
// known says a declaration was set: before it, nothing is read — every unit would read as the
// machine's, and then as a module's a moment later.
known bool
seen map[string]*seen
owners map[string]ownerAnswer
said string
last Statement
}
// ownerAnswer is the package manager's answer for one unit file, kept until the declaration changes.
type ownerAnswer struct {
pkg string
ok bool
}
// New is a judge reading through r.
func New(r Reader) *Judge {
return &Judge{reader: r, Now: time.Now, seen: map[string]*seen{}, owners: map[string]ownerAnswer{}}
}
// Set is what the declaration just applied places. The package manager is asked afresh after it, since a
// package installed or removed changes whose a unit file is.
func (j *Judge) Set(o Owned) {
j.mu.Lock()
defer j.mu.Unlock()
j.owned, j.known = o, true
j.owners = map[string]ownerAnswer{}
}
// Last is the statement of the last look.
func (j *Judge) Last() Statement {
j.mu.Lock()
defer j.mu.Unlock()
return j.last
}
// manager is one service manager read.
type manager struct{ scope, user string }
func (m manager) String() string {
if m.scope == ScopeUser {
return "the account manager of " + m.user
}
return "the machine's service manager"
}
// Look reads every manager once and answers the statement, and whether what it says changed since the
// last look. Before a declaration is set it reads nothing and answers an empty statement, which says
// nothing of the units.
func (j *Judge) Look(ctx context.Context) (Statement, bool) {
j.mu.Lock()
defer j.mu.Unlock()
if !j.known {
return Statement{}, false
}
now := j.Now()
managers := []manager{{scope: ScopeSystem}}
for _, a := range j.owned.Accounts {
managers = append(managers, manager{scope: ScopeUser, user: a})
}
st := Statement{At: now, State: Unknown}
read := 0
failedNow := map[string]bool{}
for _, m := range managers {
listed, err := j.reader.Failed(ctx, m.scope, m.user)
if err != nil {
st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error())))
// What it held is neither cleared nor counted while it cannot be read.
for k := range j.seen {
if strings.HasPrefix(k, key(m.scope, m.user, "")) {
failedNow[k] = true
}
}
continue
}
read++
if st.State == Unknown {
st.State = Running
}
if len(listed) > 0 {
st.State = Degraded
}
var names []string
for _, l := range listed {
names = append(names, l.Unit)
}
var shown map[string]Shown
if len(names) > 0 {
if shown, err = j.reader.Show(ctx, m.scope, m.user, names); err != nil {
shown = map[string]Shown{}
}
}
for _, l := range listed {
k := key(m.scope, m.user, l.Unit)
failedNow[k] = true
s := j.seen[k]
if s == nil {
s = &seen{since: now}
j.seen[k] = s
}
s.streak++
if j.owned.Judged[k] || s.streak < 2 {
continue
}
f := Failed{Unit: l.Unit, Scope: m.scope, User: m.user, Load: l.Load, Result: shown[l.Unit].Result,
Since: s.since, Streak: s.streak}
if own, via, ok := j.ownerOfUnit(ctx, k, shown[l.Unit].FragmentPath); ok {
f.Module, f.Resource, f.Via = own.module, own.resource, via
}
st.Failed = append(st.Failed, f)
}
}
for k := range j.seen {
if !failedNow[k] {
delete(j.seen, k)
}
}
sort.Slice(st.Failed, func(a, b int) bool {
if st.Failed[a].Scope != st.Failed[b].Scope {
return st.Failed[a].Scope < st.Failed[b].Scope
}
return st.Failed[a].Unit < st.Failed[b].Unit
})
if read == 0 {
st.State = Unknown
}
word := st.State
for _, f := range st.Failed {
word += "|" + f.Scope + "/" + f.Unit + "/" + f.Module
}
changed := word != j.said
j.said, j.last = word, st
return st, changed
}
// ownerOfUnit is whose a failed unit is: the declaration's unit, then the file the unit is, then the
// package the file belongs to.
func (j *Judge) ownerOfUnit(ctx context.Context, k, fragment string) (owner, string, bool) {
if o, ok := j.owned.Units[k]; ok {
return o, ViaUnit, true
}
if fragment == "" {
return owner{}, "", false
}
if o, ok := j.owned.Files[fragment]; ok {
return o, ViaFile, true
}
if len(j.owned.Packages) == 0 {
return owner{}, "", false
}
a, asked := j.owners[fragment]
if !asked {
pkg, ok, err := j.reader.PackageOwning(ctx, fragment)
if err != nil {
// Not kept: asked again on the next look.
return owner{}, "", false
}
a = ownerAnswer{pkg: pkg, ok: ok}
j.owners[fragment] = a
}
if !a.ok {
return owner{}, "", false
}
if o, ok := j.owned.Packages[a.pkg]; ok {
return o, ViaPackage, true
}
return owner{}, "", false
}
func firstLine(s string) string {
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
return line
}
+285
View File
@@ -0,0 +1,285 @@
package units
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// The reading of failed units (novox/hq issue 315, "how it is checked"): a failed unit a module states,
// writes or installs is that module's, in either manager; one no module places is the machine's; a unit
// liveness judges is left to it; the two-look rule holds both ways; a manager that cannot be read is
// unread, never healthy; and only reads are asked.
type fakeReader struct {
failed map[string][]Listed // by "system" or "user:<account>"
shown map[string]Shown // by unit
owners map[string]string // by path
unread map[string]error
ownerQs int
}
func mgr(scope, user string) string {
if scope == ScopeUser {
return "user:" + user
}
return "system"
}
func (f *fakeReader) Failed(_ context.Context, scope, user string) ([]Listed, error) {
if err := f.unread[mgr(scope, user)]; err != nil {
return nil, err
}
return f.failed[mgr(scope, user)], nil
}
func (f *fakeReader) Show(_ context.Context, _, _ string, units []string) (map[string]Shown, error) {
out := map[string]Shown{}
for _, u := range units {
out[u] = f.shown[u]
}
return out, nil
}
func (f *fakeReader) PackageOwning(_ context.Context, path string) (string, bool, error) {
f.ownerQs++
p, ok := f.owners[path]
return p, ok, nil
}
// shanks is the desktop as found on 2026-10-08: two mounts of the machine's own mount table, the
// Razer daemon's packaged user unit, and a unit a removed package left behind.
func shanks() *fakeReader {
return &fakeReader{
failed: map[string][]Listed{
"system": {{Unit: "mnt-recalbox.mount", Load: "loaded"}, {Unit: "storage-media.mount", Load: "loaded"}},
"user:operator": {{Unit: "greenclip.service", Load: "not-found"},
{Unit: "openrazer-daemon.service", Load: "loaded"}},
},
shown: map[string]Shown{
"mnt-recalbox.mount": {FragmentPath: "/run/systemd/generator/mnt-recalbox.mount", Result: "exit-code"},
"storage-media.mount": {FragmentPath: "/run/systemd/generator/storage-media.mount", Result: "timeout"},
"greenclip.service": {},
"openrazer-daemon.service": {FragmentPath: "/usr/lib/systemd/user/openrazer-daemon.service", Result: "exit-code"},
},
owners: map[string]string{"/usr/lib/systemd/user/openrazer-daemon.service": "openrazer-daemon"},
}
}
func declared() *declaration.Declaration {
return &declaration.Declaration{Resources: []declaration.Resource{
&declaration.Package{ID: "openrazer.daemon", Type: declaration.TypePackage, Package: "openrazer-daemon"},
&declaration.Package{ID: "clipmenu.greenclip", Type: declaration.TypePackage, Package: "rofi-greenclip", Absent: true},
&declaration.User{ID: "zsh.account", Type: declaration.TypeUser, Name: "operator"},
&declaration.Service{ID: "sshd.run", Type: declaration.TypeService, Unit: "sshd.service", State: "running"},
&declaration.Service{ID: "power.after-boot", Type: declaration.TypeService, Unit: "mesh-power-after-boot.service"},
&declaration.File{ID: "watcher.unit", Type: declaration.TypeFile, Path: "/home/operator/.config/systemd/user/watcher.service"},
}}
}
var t0 = time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
func lookTwice(t *testing.T, j *Judge) Statement {
t.Helper()
now := t0
j.Now = func() time.Time { return now }
first, _ := j.Look(context.Background())
if len(first.Failed) != 0 {
t.Fatalf("a unit was said failed on its first look: %+v", first.Failed)
}
now = now.Add(15 * time.Second)
st, _ := j.Look(context.Background())
return st
}
func TestTheModulesAndTheMachinesFailures(t *testing.T) {
r := shanks()
j := New(r)
j.Set(OwnedBy(declared(), nil))
st := lookTwice(t, j)
if st.State != Degraded {
t.Fatalf("state %q, want degraded", st.State)
}
owned := st.Owned()
if len(owned) != 1 || owned[0].Unit != "openrazer-daemon.service" || owned[0].Module != "openrazer" ||
owned[0].Resource != "openrazer.daemon" || owned[0].Via != ViaPackage || owned[0].Scope != ScopeUser {
t.Fatalf("the Razer daemon's packaged user unit is openrazer's: %+v", owned)
}
if owned[0].Since != t0 || owned[0].Streak != 2 || owned[0].Result != "exit-code" {
t.Fatalf("since the first look that found it, two in a row, how it failed: %+v", owned[0])
}
var machine []string
for _, f := range st.Unowned() {
machine = append(machine, f.Scope+"/"+f.Unit)
}
want := "system/mnt-recalbox.mount system/storage-media.mount user/greenclip.service"
if strings.Join(machine, " ") != want {
t.Fatalf("the machine's own: %v, want %s", machine, want)
}
}
func TestADeclaredUnitAndAWrittenFileAreTheirModules(t *testing.T) {
r := &fakeReader{
failed: map[string][]Listed{
"system": {{Unit: "mesh-power-after-boot.service", Load: "loaded"}, {Unit: "sshd.service", Load: "loaded"}},
"user:operator": {{Unit: "watcher.service", Load: "loaded"}},
},
shown: map[string]Shown{
"watcher.service": {FragmentPath: "/home/operator/.config/systemd/user/watcher.service"},
},
}
j := New(r)
j.Set(OwnedBy(declared(), nil))
st := lookTwice(t, j)
got := map[string]string{}
for _, f := range st.Failed {
got[f.Unit] = f.Module + " " + f.Via
}
if got["mesh-power-after-boot.service"] != "power unit" {
t.Errorf("a stateless service the declaration names is its module's: %q", got["mesh-power-after-boot.service"])
}
if got["watcher.service"] != "watcher file" {
t.Errorf("a user unit the mesh wrote is its module's: %q", got["watcher.service"])
}
if _, said := got["sshd.service"]; said {
t.Errorf("a service stated running is liveness's to judge, and said once: %v", got)
}
if r.ownerQs != 0 {
t.Errorf("the package database was asked %d time(s) for units the declaration already names", r.ownerQs)
}
}
func TestTwoLooksEachWay(t *testing.T) {
r := shanks()
j := New(r)
j.Set(OwnedBy(declared(), nil))
now := t0
j.Now = func() time.Time { return now }
st, changed := j.Look(context.Background())
if len(st.Failed) != 0 || !changed {
t.Fatalf("first look: nothing said failed, the state said: %+v %v", st.Failed, changed)
}
now = now.Add(15 * time.Second)
if st, changed = j.Look(context.Background()); len(st.Failed) != 4 || !changed {
t.Fatalf("second look: four said failed: %+v %v", st.Failed, changed)
}
now = now.Add(15 * time.Second)
if _, changed = j.Look(context.Background()); changed {
t.Fatal("a third look the same is no change")
}
// The operator mounts the media library again: no longer failed on the first look that says so.
r.failed["system"] = r.failed["system"][:1]
now = now.Add(15 * time.Second)
st, changed = j.Look(context.Background())
if len(st.Failed) != 3 || !changed {
t.Fatalf("a unit no longer failed is no longer said: %+v", st.Failed)
}
// Every manager clean: running.
r.failed = map[string][]Listed{}
now = now.Add(15 * time.Second)
if st, _ = j.Look(context.Background()); st.State != Running || len(st.Failed) != 0 {
t.Fatalf("no failed unit is running: %+v", st)
}
}
func TestAManagerThatCannotBeReadIsUnread(t *testing.T) {
r := shanks()
j := New(r)
j.Set(OwnedBy(declared(), nil))
lookTwice(t, j)
r.unread = map[string]error{"user:operator": errors.New("Failed to connect to bus: No medium found")}
st, _ := j.Look(context.Background())
if len(st.Unread) != 1 || !strings.Contains(st.Unread[0], "operator") {
t.Fatalf("the account manager is said unread: %v", st.Unread)
}
// Its failures are not counted while unread, and not forgotten either: back on the next read, said
// at once, from when they were first found.
r.unread = nil
st, _ = j.Look(context.Background())
for _, f := range st.Failed {
if f.Unit == "openrazer-daemon.service" && f.Since == t0 {
return
}
}
t.Fatalf("a manager read again says what it held, from when it was first found: %+v", st.Failed)
}
func TestNoManagerReadIsUnknown(t *testing.T) {
r := &fakeReader{unread: map[string]error{"system": errors.New("systemctl: not found"),
"user:operator": errors.New("systemctl: not found")}}
j := New(r)
j.Set(OwnedBy(declared(), nil))
if st, _ := j.Look(context.Background()); st.State != Unknown {
t.Fatalf("no manager read is unknown, never running: %q", st.State)
}
}
func TestAnAdoptedMachinesHoldingIsTheMachines(t *testing.T) {
d := &declaration.Declaration{Resources: []declaration.Resource{
&declaration.Service{ID: "found.cups", Type: declaration.TypeService, Unit: "cups.service"},
}}
o := OwnedBy(d, map[string]bool{"found.cups": true})
if _, ok := o.Units["system/cups.service"]; ok {
t.Fatal("a unit an adopted machine holds as found is the machine's, not a module's")
}
}
func TestTheReaderOnlyReads(t *testing.T) {
var asked []string
run := func(_ context.Context, name string, args ...string) (string, error) {
asked = append(asked, name+" "+strings.Join(args, " "))
switch {
case name == "systemctl" && contains(args, "list-units"):
return "● greenclip.service not-found failed failed greenclip.service\n" +
"openrazer-daemon.service loaded failed failed Daemon to manage razer devices in userspace\n", nil
case name == "systemctl" && contains(args, "show"):
return "Id=greenclip.service\nFragmentPath=\nResult=start-limit-hit\n\n" +
"Id=openrazer-daemon.service\nFragmentPath=/usr/lib/systemd/user/openrazer-daemon.service\nResult=exit-code\n", nil
case name == "pacman" && contains(args, "-Qqo"):
return "openrazer-daemon\n", nil
}
return "", nil
}
e := Exec{Run: run, System: "arch"}
listed, err := e.Failed(context.Background(), ScopeUser, "operator")
if err != nil || len(listed) != 2 || listed[0].Unit != "greenclip.service" || listed[0].Load != "not-found" {
t.Fatalf("the list, its mark skipped: %+v %v", listed, err)
}
shown, err := e.Show(context.Background(), ScopeUser, "operator", []string{"greenclip.service", "openrazer-daemon.service"})
if err != nil || shown["openrazer-daemon.service"].FragmentPath == "" || shown["greenclip.service"].Result != "start-limit-hit" {
t.Fatalf("the show: %+v %v", shown, err)
}
if pkg, ok, err := e.PackageOwning(context.Background(), "/usr/lib/systemd/user/openrazer-daemon.service"); !ok ||
pkg != "openrazer-daemon" || err != nil {
t.Fatalf("the owner: %q %v %v", pkg, ok, err)
}
for _, a := range asked {
if !strings.Contains(a, "list-units") && !strings.Contains(a, " show ") && !strings.HasPrefix(a, "pacman -Q") {
t.Errorf("asked something that is not a read: %q", a)
}
if !strings.HasPrefix(a, "pacman") && !strings.Contains(a, "--machine=operator@") {
t.Errorf("an account's manager is asked as that account's: %q", a)
}
}
}
func contains(args []string, s string) bool {
for _, a := range args {
if a == s {
return true
}
}
return false
}
func TestNothingIsReadBeforeADeclaration(t *testing.T) {
r := shanks()
j := New(r)
if st, changed := j.Look(context.Background()); st.State != "" || changed {
t.Fatalf("before a declaration every unit would read as the machine's: %+v", st)
}
}