diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 9ad714c..580e1e5 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -43,6 +43,7 @@ import ( "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" "github.com/novox/mesh-host/internal/tunnel" + "github.com/novox/mesh-host/internal/units" "github.com/novox/mesh-host/internal/upgrade" ) @@ -1090,6 +1091,8 @@ func runLink(ctx context.Context, opts options) error { return held } judging = j + // And which units its service managers say failed, and whose each is (novox/hq issue 315). + unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor}) } // **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR @@ -1384,6 +1387,10 @@ const ReconcileEvery = 5 * time.Minute // reports no health, and in a test. var judging *liveness.Judge +// unitJudge reads which units the machine's service managers say failed, and whose each is (novox/hq +// issue 315); nil where judging is. +var unitJudge *units.Judge + // netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a // one-shot command and in a test, which say nothing of the network. var netJudge networkJudge @@ -1478,6 +1485,17 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } st, changed := j.Look(ctx) owed = owed || changed + var unitSt *units.Statement + if u := unitJudge; u != nil { + us, unitsChanged := u.Look(ctx) + unitSt = &us + owed = owed || unitsChanged + if unitsChanged { + for _, f := range us.Failed { + say(failedUnitWords(f)) + } + } + } var netSt *network.Statement if n := netJudge.get(); n != nil { ns, netChanged := n.Look(ctx) @@ -1500,7 +1518,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa spaced = sp } since := time.Since(lastSaid) - healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) + healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) && + (unitSt == nil || len(unitSt.Failed) == 0) if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway { continue } @@ -1512,7 +1531,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } } } - if queue.SayHealth(ctx, *healthAsReported(st, netSt)) { + if queue.SayHealth(ctx, *withUnits(healthAsReported(st, netSt), unitSt)) { lastSaid, owed = time.Now(), false } } @@ -1538,6 +1557,50 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health return h } +// withUnits adds to a statement the machine's failed units (novox/hq issue 315): each a module places, +// among the resources as that module's unhealthy unit; the rest, the machine's own, beside them. Nil — an +// engine not reading them — says nothing of them, which the controller reads as not known. +func withUnits(h *link.Health, us *units.Statement) *link.Health { + if us == nil || us.State == "" { + return h + } + h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread} + for _, f := range us.Failed { + if f.Module == "" { + h.Units.Failed = append(h.Units.Failed, link.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, + Result: f.Result, Resource: f.Resource, Since: f.Since.UTC()}) + continue + } + reason := "failed" + if f.Scope == units.ScopeUser { + reason += " in the account's own service manager" + } + if f.Result != "" { + reason += " (" + f.Result + ")" + } + h.Resources = append(h.Resources, link.ResourceHealth{Module: f.Module, Resource: f.Resource, + Kind: link.KindUnit, Target: f.Unit, State: link.StateUnhealthy, Reason: reason, Since: f.Since.UTC(), + Streak: f.Streak}) + } + return h +} + +// failedUnitWords is a failed unit as the console says it. +func failedUnitWords(f units.Failed) string { + whose := "no module places it" + if f.Module != "" { + whose = fmt.Sprintf("%s's, by its %s %s", f.Module, f.Via, f.Resource) + } + return fmt.Sprintf("the unit %s (%s) failed%s: %s", f.Unit, f.Scope, orNothing(" ("+f.Result+")", f.Result), whose) +} + +func orNothing(s, unless string) string { + if unless == "" { + return "" + } + return s +} + // holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq // to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is // waiting is that delivery's apply. @@ -1775,7 +1838,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto ns := n.Last() netSt = &ns } - report.Health = healthAsReported(st, netSt) + var unitSt *units.Statement + if u := unitJudge; u != nil { + u.Set(units.OwnedBy(declared, held)) + us := u.Last() + unitSt = &us + } + report.Health = withUnits(healthAsReported(st, netSt), unitSt) } // Which of this machine's links face outside, for the filter the mesh writes around them // (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it, diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go new file mode 100644 index 0000000..e4e6f2b --- /dev/null +++ b/cmd/mesh-host/units_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/units" +) + +// The machine's failed units in the engine's statement (novox/hq issue 315): a module's is among the +// resources, unhealthy, as that module's unit, naming it; the machine's own beside them; and an engine +// not reading them says nothing of them. +func TestTheStatementCarriesTheFailedUnits(t *testing.T) { + at := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + us := &units.Statement{At: at, State: units.Degraded, Failed: []units.Failed{ + {Unit: "openrazer-daemon.service", Scope: units.ScopeUser, Load: "loaded", Result: "exit-code", + Module: "openrazer", Resource: "openrazer.daemon", Via: units.ViaPackage, Since: at, Streak: 2}, + {Unit: "storage-media.mount", Scope: units.ScopeSystem, Load: "loaded", Result: "timeout", Since: at, Streak: 2}, + }} + h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), us) + if len(h.Resources) != 1 { + t.Fatalf("one module's unit among the resources: %+v", h.Resources) + } + r := h.Resources[0] + if r.Module != "openrazer" || r.Resource != "openrazer.daemon" || r.Kind != link.KindUnit || + r.Target != "openrazer-daemon.service" || r.State != link.StateUnhealthy || + !strings.Contains(r.Reason, "account's own service manager") || !strings.Contains(r.Reason, "exit-code") { + t.Fatalf("the module's failed unit: %+v", r) + } + if h.Units == nil || h.Units.State != units.Degraded || len(h.Units.Failed) != 1 || + h.Units.Failed[0].Unit != "storage-media.mount" || h.Units.Failed[0].Result != "timeout" { + t.Fatalf("the machine's own: %+v", h.Units) + } + if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), nil); h.Units != nil { + t.Fatal("an engine not reading units said them") + } + if h := withUnits(healthAsReported(liveness.Statement{At: at}, nil), &units.Statement{}); h.Units != nil { + t.Fatal("a judge not yet given a declaration said units") + } +} diff --git a/internal/link/messages.go b/internal/link/messages.go index 783d8d2..b95938a 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -242,6 +242,41 @@ type Health struct { // file, its names, its tunnel, its bus and its route. Absent from an engine older than that judging, // which the controller reads as "not known", never as healthy. Network *NetworkHealth `json:"network,omitempty"` + // Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any + // unit failed, and each failed unit no module places. A failed unit a module states, writes or + // installs is said among Resources instead, as that module's, of kind KindUnit. Absent from an engine + // older than that reading, which the controller reads as "not known", never as healthy. + Units *UnitsHealth `json:"units,omitempty"` +} + +// KindUnit is a module's unit its service manager says failed (issue 315): a resource of the module +// that is not long-running, or not stated running — a package's unit, a unit file the mesh wrote, a +// service whose lifecycle is the machine's — said unhealthy for as long as it stays failed. +const KindUnit = "unit" + +// UnitsHealth is the machine's service managers in one statement (issue 315). +type UnitsHealth struct { + // State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no + // manager could be read. + State string `json:"state"` + // Failed is every unit failed on two looks in a row that no module places: the machine's own. + Failed []FailedUnit `json:"failed"` + // Unread names a manager that could not be read, with why. + Unread []string `json:"unread,omitempty"` +} + +// FailedUnit is one failed unit no module places. +type FailedUnit struct { + Unit string `json:"unit"` + // Scope is system, or user: an account's own manager. + Scope string `json:"scope"` + // Load is loaded, not-found — a unit whose file is gone and whose failure its manager still holds… + Load string `json:"load,omitempty"` + // Result is how it failed: exit-code, timeout, start-limit-hit… + Result string `json:"result,omitempty"` + // Resource is the mesh's own resource that names it, when the mesh placed it in its own right. + Resource string `json:"resource,omitempty"` + Since time.Time `json:"since"` } // NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since diff --git a/internal/units/exec.go b/internal/units/exec.go new file mode 100644 index 0000000..8d8f73e --- /dev/null +++ b/internal/units/exec.go @@ -0,0 +1,105 @@ +package units + +import ( + "context" + "errors" + "fmt" + "strings" +) + +// Runner runs a command and answers what it printed — the apply's own (apply.ExecRunner), so a look reads +// the machine exactly as an apply does. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// Exec reads the service managers through systemctl and the package database through the system's own +// owner query. **Reads only**: `list-units`, `show` and the owner query are the whole of what it asks +// (ADR 0240 rule 6, and a test holds it). +type Exec struct { + Run Runner + // System is what the host was built for; it says how a file's package is asked. A system with no + // owner query answers "no package", so a unit there is the machine's unless the declaration states it + // or writes its file. + System string +} + +// managerArgs is how systemctl is pointed at a manager: the machine's, or an account's own. +func managerArgs(scope, user string) []string { + if scope == ScopeUser && user != "" { + return []string{"--user", "--machine=" + user + "@"} + } + return nil +} + +// Failed lists the failed units of one manager, in its plain form: one per line, the unit, its load, its +// active and sub state, and its description. A leading mark some versions print before a unit in trouble +// is skipped. +func (e Exec) Failed(ctx context.Context, scope, user string) ([]Listed, error) { + args := append(managerArgs(scope, user), "list-units", "--state=failed", "--all", "--plain", "--no-legend", + "--no-pager", "--full") + said, err := e.Run(ctx, "systemctl", args...) + if err != nil { + return nil, fmt.Errorf("the service manager could not be read: %w", err) + } + return parseFailed(said), nil +} + +func parseFailed(said string) []Listed { + var out []Listed + for _, line := range strings.Split(said, "\n") { + fields := strings.Fields(line) + for len(fields) > 0 && (fields[0] == "●" || fields[0] == "*" || fields[0] == "×") { + fields = fields[1:] + } + if len(fields) < 2 || !strings.Contains(fields[0], ".") { + continue + } + out = append(out, Listed{Unit: fields[0], Load: fields[1]}) + } + return out +} + +// Show is one show of the units named: each one's file and how it failed. +func (e Exec) Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error) { + out := map[string]Shown{} + if len(units) == 0 { + return out, nil + } + args := append(managerArgs(scope, user), "show", "--property=Id,FragmentPath,Result", "--") + said, err := e.Run(ctx, "systemctl", append(args, units...)...) + if err != nil { + return nil, fmt.Errorf("the service manager could not be read: %w", err) + } + blocks := strings.Split(strings.TrimSpace(said), "\n\n") + if len(blocks) != len(units) { + return nil, errors.New("the service manager answered for a different number of units than were asked") + } + for i, block := range blocks { + props := map[string]string{} + for _, line := range strings.Split(block, "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + props[strings.TrimSpace(k)] = strings.TrimSpace(v) + } + } + out[units[i]] = Shown{FragmentPath: props["FragmentPath"], Result: props["Result"]} + } + return out, nil +} + +// PackageOwning asks the package database which package a file belongs to. pacman exits 1 both for a +// file no package owns and for a database it cannot read; it is asked about itself first, so the second +// is an error and only the first is "no package" (system/arch.go's two-step, for the same trap). +func (e Exec) PackageOwning(ctx context.Context, path string) (string, bool, error) { + switch e.System { + case "arch": + if _, err := e.Run(ctx, "pacman", "-Q", "pacman"); err != nil { + return "", false, fmt.Errorf("the package database does not answer: %w", err) + } + said, err := e.Run(ctx, "pacman", "-Qqo", path) + if err != nil { + return "", false, nil + } + pkg := strings.TrimSpace(firstLine(said)) + return pkg, pkg != "", nil + } + return "", false, nil +} diff --git a/internal/units/units.go b/internal/units/units.go new file mode 100644 index 0000000..f796b5f --- /dev/null +++ b/internal/units/units.go @@ -0,0 +1,402 @@ +// Package units is the node-engine reading which units its machine's service managers say failed, and +// whose each is (novox/hq issue 315, under ADR 0240 rule 1 and ADR 0241 §3). +// +// **A failed unit of a mesh module was never raised.** Liveness judges what a module runs long-lived — a +// container, a process, a service stated `running` — and nothing else. A module that installs a daemon as +// a package, whose unit the package ships and D-Bus activation starts, declares no service: its unit +// failed at every start and the machine read healthy, while the profile's `service-manager` said +// `degraded` and nothing raised that either. Two network mounts the operator wrote into the machine's +// own mount table, and a unit a removed package left behind, failed beside it, said by nobody. +// +// On every look the engine asks each service manager the mesh places units in — the machine's, and the +// account manager of every account the declaration names — which units failed, and says each one's +// owner: +// +// 1. a service or process the declaration states, by its unit and manager; +// 2. a file the declaration writes, when the unit's file is that file; +// 3. a package the declaration installs, when the unit's file belongs to it — asked of the package +// manager, once per unit file; +// +// and otherwise nobody's in the mesh: the machine's. A unit liveness already judges is left to it, so a +// failure is said once. **The two-look rule is the engine's** (ADR 0241 §2): a unit is said failed on +// its second look in a row, and no longer on its first look not failed. +// +// **It reads; it never acts** (ADR 0240 rule 6): `list-units`, `show` and the package manager's owner +// query are the whole of what it asks. It neither resets nor restarts anything. +package units + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// The managers a unit is in. +const ( + ScopeSystem = declaration.ScopeSystem + ScopeUser = declaration.ScopeUser +) + +// How a unit's owner was found. +const ( + ViaUnit = "unit" + ViaFile = "file" + ViaPackage = "package" +) + +// The machine's service managers, as the statement says them. +const ( + // Running is every manager read and no unit failed. + Running = "running" + // Degraded is a unit failed in a manager read. + Degraded = "degraded" + // Unknown is no manager could be read. + Unknown = "unknown" +) + +// owner is a module and the id of its resource that places a unit. +type owner struct{ module, resource string } + +// Owned is what a declaration places on the machine, as the reading needs it: the units it states, the +// files and packages it puts there, and the accounts whose managers it places units in. +type Owned struct { + // Units is keyed by manager and unit (key). + Units map[string]owner + // Judged is every unit liveness judges, by the same key: left to it. + Judged map[string]bool + // Files is keyed by path; Packages by package name, only those declared present. + Files map[string]owner + Packages map[string]owner + // Accounts are the accounts whose own managers are read, sorted. + Accounts []string +} + +// key is a unit in one manager: "system/", or "user:/". +func key(scope, user, unit string) string { + if scope == ScopeUser { + return "user:" + user + "/" + unit + } + return "system/" + unit +} + +// OwnedBy reads what a declaration places. held is every resource an adopted machine holds as found, +// by id — the machine's, not a module's. A resource the mesh declares in its own right (no module) names +// no module: its failed unit is said with the machine's, under the resource's id. +func OwnedBy(d *declaration.Declaration, held map[string]bool) Owned { + o := Owned{Units: map[string]owner{}, Judged: map[string]bool{}, Files: map[string]owner{}, + Packages: map[string]owner{}} + if d == nil { + return o + } + accounts := map[string]bool{} + for _, r := range d.Resources { + if held[r.Identity()] || strings.HasPrefix(r.Identity(), declaration.AdoptionPrefix) { + continue + } + own := ownerOf(r.Identity()) + switch v := r.(type) { + case *declaration.Service: + scope, user := ScopeSystem, "" + if v.UserScoped() { + scope, user = ScopeUser, v.User + accounts[v.User] = true + } + k := key(scope, user, v.Unit) + o.Units[k] = own + if v.State == "running" { + o.Judged[k] = true + } + case *declaration.Process: + k := key(ScopeSystem, "", v.Name+".service") + o.Units[k] = own + if !v.RunOnce && v.Schedule == "" { + o.Judged[k] = true + } + case *declaration.File: + o.Files[v.Path] = own + case *declaration.Package: + if !v.Absent { + o.Packages[v.Package] = own + } + case *declaration.User: + accounts[v.Name] = true + } + } + for a := range accounts { + if a != "" { + o.Accounts = append(o.Accounts, a) + } + } + sort.Strings(o.Accounts) + return o +} + +// ownerOf is a resource id's module and the id itself: everything before the last dot is the module (as +// liveness.ModuleOf reads it); an id with no dot is the mesh's own, and names no module. +func ownerOf(id string) owner { + at := strings.LastIndex(id, ".") + if at <= 0 { + return owner{resource: id} + } + return owner{module: id[:at], resource: id} +} + +// Listed is one failed unit as its manager lists it. +type Listed struct { + Unit string + // Load is loaded, not-found, masked, bad-setting… + Load string +} + +// Shown is what the manager says of one unit's file and how it failed. +type Shown struct { + FragmentPath string + // Result is how it failed: exit-code, timeout, start-limit-hit… + Result string +} + +// Reader is what a look asks the machine. An error is "could not be read": that manager is said unread, +// never healthy and never failed. +type Reader interface { + // Failed is every unit in failed state in a manager: the machine's (user empty), or an account's. + Failed(ctx context.Context, scope, user string) ([]Listed, error) + // Show is each unit's file and result, in a manager. + Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error) + // PackageOwning is the package a file belongs to; false when none does or the machine cannot say. + PackageOwning(ctx context.Context, path string) (string, bool, error) +} + +// Failed is one failed unit as the statement says it. +type Failed struct { + Unit string + Scope string + // User is the account whose manager it is in, for a user unit: evidence inside the mesh. + User string + Load string + Result string + // Module and Resource own it; empty when no module does. Via is how that was found. + Module string + Resource string + Via string + // Since is the first look that found it failed; Streak the looks in a row since. + Since time.Time + Streak int +} + +// Statement is one look at every manager. +type Statement struct { + At time.Time + // State is the worst of the managers read: running, degraded, or unknown when none was. + State string + // Failed is every unit failed on two looks in a row and not judged by liveness: the modules' and the + // machine's. + Failed []Failed + // Unread names each manager that could not be read, with why. + Unread []string +} + +// Owned answers the failures a module owns; Unowned those no module does. +func (s Statement) Owned() []Failed { return s.filter(true) } +func (s Statement) Unowned() []Failed { return s.filter(false) } + +func (s Statement) filter(owned bool) []Failed { + var out []Failed + for _, f := range s.Failed { + if (f.Module != "") == owned { + out = append(out, f) + } + } + return out +} + +// seen is what the judge keeps of one failed unit between looks. +type seen struct { + since time.Time + streak int +} + +// Judge reads the machine's failed units on every look. Safe for the apply and the looking loop at once. +type Judge struct { + reader Reader + Now func() time.Time + + mu sync.Mutex + owned Owned + // known says a declaration was set: before it, nothing is read — every unit would read as the + // machine's, and then as a module's a moment later. + known bool + seen map[string]*seen + owners map[string]ownerAnswer + said string + last Statement +} + +// ownerAnswer is the package manager's answer for one unit file, kept until the declaration changes. +type ownerAnswer struct { + pkg string + ok bool +} + +// New is a judge reading through r. +func New(r Reader) *Judge { + return &Judge{reader: r, Now: time.Now, seen: map[string]*seen{}, owners: map[string]ownerAnswer{}} +} + +// Set is what the declaration just applied places. The package manager is asked afresh after it, since a +// package installed or removed changes whose a unit file is. +func (j *Judge) Set(o Owned) { + j.mu.Lock() + defer j.mu.Unlock() + j.owned, j.known = o, true + j.owners = map[string]ownerAnswer{} +} + +// Last is the statement of the last look. +func (j *Judge) Last() Statement { + j.mu.Lock() + defer j.mu.Unlock() + return j.last +} + +// manager is one service manager read. +type manager struct{ scope, user string } + +func (m manager) String() string { + if m.scope == ScopeUser { + return "the account manager of " + m.user + } + return "the machine's service manager" +} + +// Look reads every manager once and answers the statement, and whether what it says changed since the +// last look. Before a declaration is set it reads nothing and answers an empty statement, which says +// nothing of the units. +func (j *Judge) Look(ctx context.Context) (Statement, bool) { + j.mu.Lock() + defer j.mu.Unlock() + if !j.known { + return Statement{}, false + } + now := j.Now() + managers := []manager{{scope: ScopeSystem}} + for _, a := range j.owned.Accounts { + managers = append(managers, manager{scope: ScopeUser, user: a}) + } + st := Statement{At: now, State: Unknown} + read := 0 + failedNow := map[string]bool{} + for _, m := range managers { + listed, err := j.reader.Failed(ctx, m.scope, m.user) + if err != nil { + st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error()))) + // What it held is neither cleared nor counted while it cannot be read. + for k := range j.seen { + if strings.HasPrefix(k, key(m.scope, m.user, "")) { + failedNow[k] = true + } + } + continue + } + read++ + if st.State == Unknown { + st.State = Running + } + if len(listed) > 0 { + st.State = Degraded + } + var names []string + for _, l := range listed { + names = append(names, l.Unit) + } + var shown map[string]Shown + if len(names) > 0 { + if shown, err = j.reader.Show(ctx, m.scope, m.user, names); err != nil { + shown = map[string]Shown{} + } + } + for _, l := range listed { + k := key(m.scope, m.user, l.Unit) + failedNow[k] = true + s := j.seen[k] + if s == nil { + s = &seen{since: now} + j.seen[k] = s + } + s.streak++ + if j.owned.Judged[k] || s.streak < 2 { + continue + } + f := Failed{Unit: l.Unit, Scope: m.scope, User: m.user, Load: l.Load, Result: shown[l.Unit].Result, + Since: s.since, Streak: s.streak} + if own, via, ok := j.ownerOfUnit(ctx, k, shown[l.Unit].FragmentPath); ok { + f.Module, f.Resource, f.Via = own.module, own.resource, via + } + st.Failed = append(st.Failed, f) + } + } + for k := range j.seen { + if !failedNow[k] { + delete(j.seen, k) + } + } + sort.Slice(st.Failed, func(a, b int) bool { + if st.Failed[a].Scope != st.Failed[b].Scope { + return st.Failed[a].Scope < st.Failed[b].Scope + } + return st.Failed[a].Unit < st.Failed[b].Unit + }) + if read == 0 { + st.State = Unknown + } + word := st.State + for _, f := range st.Failed { + word += "|" + f.Scope + "/" + f.Unit + "/" + f.Module + } + changed := word != j.said + j.said, j.last = word, st + return st, changed +} + +// ownerOfUnit is whose a failed unit is: the declaration's unit, then the file the unit is, then the +// package the file belongs to. +func (j *Judge) ownerOfUnit(ctx context.Context, k, fragment string) (owner, string, bool) { + if o, ok := j.owned.Units[k]; ok { + return o, ViaUnit, true + } + if fragment == "" { + return owner{}, "", false + } + if o, ok := j.owned.Files[fragment]; ok { + return o, ViaFile, true + } + if len(j.owned.Packages) == 0 { + return owner{}, "", false + } + a, asked := j.owners[fragment] + if !asked { + pkg, ok, err := j.reader.PackageOwning(ctx, fragment) + if err != nil { + // Not kept: asked again on the next look. + return owner{}, "", false + } + a = ownerAnswer{pkg: pkg, ok: ok} + j.owners[fragment] = a + } + if !a.ok { + return owner{}, "", false + } + if o, ok := j.owned.Packages[a.pkg]; ok { + return o, ViaPackage, true + } + return owner{}, "", false +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line +} diff --git a/internal/units/units_test.go b/internal/units/units_test.go new file mode 100644 index 0000000..71f7cf5 --- /dev/null +++ b/internal/units/units_test.go @@ -0,0 +1,285 @@ +package units + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// The reading of failed units (novox/hq issue 315, "how it is checked"): a failed unit a module states, +// writes or installs is that module's, in either manager; one no module places is the machine's; a unit +// liveness judges is left to it; the two-look rule holds both ways; a manager that cannot be read is +// unread, never healthy; and only reads are asked. + +type fakeReader struct { + failed map[string][]Listed // by "system" or "user:" + shown map[string]Shown // by unit + owners map[string]string // by path + unread map[string]error + ownerQs int +} + +func mgr(scope, user string) string { + if scope == ScopeUser { + return "user:" + user + } + return "system" +} + +func (f *fakeReader) Failed(_ context.Context, scope, user string) ([]Listed, error) { + if err := f.unread[mgr(scope, user)]; err != nil { + return nil, err + } + return f.failed[mgr(scope, user)], nil +} + +func (f *fakeReader) Show(_ context.Context, _, _ string, units []string) (map[string]Shown, error) { + out := map[string]Shown{} + for _, u := range units { + out[u] = f.shown[u] + } + return out, nil +} + +func (f *fakeReader) PackageOwning(_ context.Context, path string) (string, bool, error) { + f.ownerQs++ + p, ok := f.owners[path] + return p, ok, nil +} + +// shanks is the desktop as found on 2026-10-08: two mounts of the machine's own mount table, the +// Razer daemon's packaged user unit, and a unit a removed package left behind. +func shanks() *fakeReader { + return &fakeReader{ + failed: map[string][]Listed{ + "system": {{Unit: "mnt-recalbox.mount", Load: "loaded"}, {Unit: "storage-media.mount", Load: "loaded"}}, + "user:operator": {{Unit: "greenclip.service", Load: "not-found"}, + {Unit: "openrazer-daemon.service", Load: "loaded"}}, + }, + shown: map[string]Shown{ + "mnt-recalbox.mount": {FragmentPath: "/run/systemd/generator/mnt-recalbox.mount", Result: "exit-code"}, + "storage-media.mount": {FragmentPath: "/run/systemd/generator/storage-media.mount", Result: "timeout"}, + "greenclip.service": {}, + "openrazer-daemon.service": {FragmentPath: "/usr/lib/systemd/user/openrazer-daemon.service", Result: "exit-code"}, + }, + owners: map[string]string{"/usr/lib/systemd/user/openrazer-daemon.service": "openrazer-daemon"}, + } +} + +func declared() *declaration.Declaration { + return &declaration.Declaration{Resources: []declaration.Resource{ + &declaration.Package{ID: "openrazer.daemon", Type: declaration.TypePackage, Package: "openrazer-daemon"}, + &declaration.Package{ID: "clipmenu.greenclip", Type: declaration.TypePackage, Package: "rofi-greenclip", Absent: true}, + &declaration.User{ID: "zsh.account", Type: declaration.TypeUser, Name: "operator"}, + &declaration.Service{ID: "sshd.run", Type: declaration.TypeService, Unit: "sshd.service", State: "running"}, + &declaration.Service{ID: "power.after-boot", Type: declaration.TypeService, Unit: "mesh-power-after-boot.service"}, + &declaration.File{ID: "watcher.unit", Type: declaration.TypeFile, Path: "/home/operator/.config/systemd/user/watcher.service"}, + }} +} + +var t0 = time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + +func lookTwice(t *testing.T, j *Judge) Statement { + t.Helper() + now := t0 + j.Now = func() time.Time { return now } + first, _ := j.Look(context.Background()) + if len(first.Failed) != 0 { + t.Fatalf("a unit was said failed on its first look: %+v", first.Failed) + } + now = now.Add(15 * time.Second) + st, _ := j.Look(context.Background()) + return st +} + +func TestTheModulesAndTheMachinesFailures(t *testing.T) { + r := shanks() + j := New(r) + j.Set(OwnedBy(declared(), nil)) + st := lookTwice(t, j) + if st.State != Degraded { + t.Fatalf("state %q, want degraded", st.State) + } + owned := st.Owned() + if len(owned) != 1 || owned[0].Unit != "openrazer-daemon.service" || owned[0].Module != "openrazer" || + owned[0].Resource != "openrazer.daemon" || owned[0].Via != ViaPackage || owned[0].Scope != ScopeUser { + t.Fatalf("the Razer daemon's packaged user unit is openrazer's: %+v", owned) + } + if owned[0].Since != t0 || owned[0].Streak != 2 || owned[0].Result != "exit-code" { + t.Fatalf("since the first look that found it, two in a row, how it failed: %+v", owned[0]) + } + var machine []string + for _, f := range st.Unowned() { + machine = append(machine, f.Scope+"/"+f.Unit) + } + want := "system/mnt-recalbox.mount system/storage-media.mount user/greenclip.service" + if strings.Join(machine, " ") != want { + t.Fatalf("the machine's own: %v, want %s", machine, want) + } +} + +func TestADeclaredUnitAndAWrittenFileAreTheirModules(t *testing.T) { + r := &fakeReader{ + failed: map[string][]Listed{ + "system": {{Unit: "mesh-power-after-boot.service", Load: "loaded"}, {Unit: "sshd.service", Load: "loaded"}}, + "user:operator": {{Unit: "watcher.service", Load: "loaded"}}, + }, + shown: map[string]Shown{ + "watcher.service": {FragmentPath: "/home/operator/.config/systemd/user/watcher.service"}, + }, + } + j := New(r) + j.Set(OwnedBy(declared(), nil)) + st := lookTwice(t, j) + got := map[string]string{} + for _, f := range st.Failed { + got[f.Unit] = f.Module + " " + f.Via + } + if got["mesh-power-after-boot.service"] != "power unit" { + t.Errorf("a stateless service the declaration names is its module's: %q", got["mesh-power-after-boot.service"]) + } + if got["watcher.service"] != "watcher file" { + t.Errorf("a user unit the mesh wrote is its module's: %q", got["watcher.service"]) + } + if _, said := got["sshd.service"]; said { + t.Errorf("a service stated running is liveness's to judge, and said once: %v", got) + } + if r.ownerQs != 0 { + t.Errorf("the package database was asked %d time(s) for units the declaration already names", r.ownerQs) + } +} + +func TestTwoLooksEachWay(t *testing.T) { + r := shanks() + j := New(r) + j.Set(OwnedBy(declared(), nil)) + now := t0 + j.Now = func() time.Time { return now } + st, changed := j.Look(context.Background()) + if len(st.Failed) != 0 || !changed { + t.Fatalf("first look: nothing said failed, the state said: %+v %v", st.Failed, changed) + } + now = now.Add(15 * time.Second) + if st, changed = j.Look(context.Background()); len(st.Failed) != 4 || !changed { + t.Fatalf("second look: four said failed: %+v %v", st.Failed, changed) + } + now = now.Add(15 * time.Second) + if _, changed = j.Look(context.Background()); changed { + t.Fatal("a third look the same is no change") + } + // The operator mounts the media library again: no longer failed on the first look that says so. + r.failed["system"] = r.failed["system"][:1] + now = now.Add(15 * time.Second) + st, changed = j.Look(context.Background()) + if len(st.Failed) != 3 || !changed { + t.Fatalf("a unit no longer failed is no longer said: %+v", st.Failed) + } + // Every manager clean: running. + r.failed = map[string][]Listed{} + now = now.Add(15 * time.Second) + if st, _ = j.Look(context.Background()); st.State != Running || len(st.Failed) != 0 { + t.Fatalf("no failed unit is running: %+v", st) + } +} + +func TestAManagerThatCannotBeReadIsUnread(t *testing.T) { + r := shanks() + j := New(r) + j.Set(OwnedBy(declared(), nil)) + lookTwice(t, j) + r.unread = map[string]error{"user:operator": errors.New("Failed to connect to bus: No medium found")} + st, _ := j.Look(context.Background()) + if len(st.Unread) != 1 || !strings.Contains(st.Unread[0], "operator") { + t.Fatalf("the account manager is said unread: %v", st.Unread) + } + // Its failures are not counted while unread, and not forgotten either: back on the next read, said + // at once, from when they were first found. + r.unread = nil + st, _ = j.Look(context.Background()) + for _, f := range st.Failed { + if f.Unit == "openrazer-daemon.service" && f.Since == t0 { + return + } + } + t.Fatalf("a manager read again says what it held, from when it was first found: %+v", st.Failed) +} + +func TestNoManagerReadIsUnknown(t *testing.T) { + r := &fakeReader{unread: map[string]error{"system": errors.New("systemctl: not found"), + "user:operator": errors.New("systemctl: not found")}} + j := New(r) + j.Set(OwnedBy(declared(), nil)) + if st, _ := j.Look(context.Background()); st.State != Unknown { + t.Fatalf("no manager read is unknown, never running: %q", st.State) + } +} + +func TestAnAdoptedMachinesHoldingIsTheMachines(t *testing.T) { + d := &declaration.Declaration{Resources: []declaration.Resource{ + &declaration.Service{ID: "found.cups", Type: declaration.TypeService, Unit: "cups.service"}, + }} + o := OwnedBy(d, map[string]bool{"found.cups": true}) + if _, ok := o.Units["system/cups.service"]; ok { + t.Fatal("a unit an adopted machine holds as found is the machine's, not a module's") + } +} + +func TestTheReaderOnlyReads(t *testing.T) { + var asked []string + run := func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + switch { + case name == "systemctl" && contains(args, "list-units"): + return "● greenclip.service not-found failed failed greenclip.service\n" + + "openrazer-daemon.service loaded failed failed Daemon to manage razer devices in userspace\n", nil + case name == "systemctl" && contains(args, "show"): + return "Id=greenclip.service\nFragmentPath=\nResult=start-limit-hit\n\n" + + "Id=openrazer-daemon.service\nFragmentPath=/usr/lib/systemd/user/openrazer-daemon.service\nResult=exit-code\n", nil + case name == "pacman" && contains(args, "-Qqo"): + return "openrazer-daemon\n", nil + } + return "", nil + } + e := Exec{Run: run, System: "arch"} + listed, err := e.Failed(context.Background(), ScopeUser, "operator") + if err != nil || len(listed) != 2 || listed[0].Unit != "greenclip.service" || listed[0].Load != "not-found" { + t.Fatalf("the list, its mark skipped: %+v %v", listed, err) + } + shown, err := e.Show(context.Background(), ScopeUser, "operator", []string{"greenclip.service", "openrazer-daemon.service"}) + if err != nil || shown["openrazer-daemon.service"].FragmentPath == "" || shown["greenclip.service"].Result != "start-limit-hit" { + t.Fatalf("the show: %+v %v", shown, err) + } + if pkg, ok, err := e.PackageOwning(context.Background(), "/usr/lib/systemd/user/openrazer-daemon.service"); !ok || + pkg != "openrazer-daemon" || err != nil { + t.Fatalf("the owner: %q %v %v", pkg, ok, err) + } + for _, a := range asked { + if !strings.Contains(a, "list-units") && !strings.Contains(a, " show ") && !strings.HasPrefix(a, "pacman -Q") { + t.Errorf("asked something that is not a read: %q", a) + } + if !strings.HasPrefix(a, "pacman") && !strings.Contains(a, "--machine=operator@") { + t.Errorf("an account's manager is asked as that account's: %q", a) + } + } +} + +func contains(args []string, s string) bool { + for _, a := range args { + if a == s { + return true + } + } + return false +} + +func TestNothingIsReadBeforeADeclaration(t *testing.T) { + r := shanks() + j := New(r) + if st, changed := j.Look(context.Background()); st.State != "" || changed { + t.Fatalf("before a declaration every unit would read as the machine's: %+v", st) + } +}