// Package units is the node-engine reading which units its machine's service managers say failed, and // whose each is (novox/hq issue 315, under ADR 0240 rule 1 and ADR 0241 §3). // // **A failed unit of a mesh module was never raised.** Liveness judges what a module runs long-lived — a // container, a process, a service stated `running` — and nothing else. A module that installs a daemon as // a package, whose unit the package ships and D-Bus activation starts, declares no service: its unit // failed at every start and the machine read healthy, while the profile's `service-manager` said // `degraded` and nothing raised that either. Two network mounts the operator wrote into the machine's // own mount table, and a unit a removed package left behind, failed beside it, said by nobody. // // On every look the engine asks each service manager the mesh places units in — the machine's, and the // account manager of every account the declaration names — which units failed, and says each one's // owner: // // 1. a service or process the declaration states, by its unit and manager; // 2. a file the declaration writes, when the unit's file is that file; // 3. a package the declaration installs, when the unit's file belongs to it — asked of the package // manager, once per unit file; // // and otherwise nobody's in the mesh: the machine's. A unit liveness already judges is left to it, so a // failure is said once. **The two-look rule is the engine's** (ADR 0241 §2): a unit is said failed on // its second look in a row, and no longer on its first look not failed. // // **It reads; it never acts** (ADR 0240 rule 6): `list-units`, `show` and the package manager's owner // query are the whole of what it asks. It neither resets nor restarts anything. package units import ( "context" "fmt" "sort" "strings" "sync" "time" "github.com/novox/mesh-host/internal/declaration" ) // The managers a unit is in. const ( ScopeSystem = declaration.ScopeSystem ScopeUser = declaration.ScopeUser ) // How a unit's owner was found. const ( ViaUnit = "unit" ViaFile = "file" ViaPackage = "package" ) // The machine's service managers, as the statement says them. const ( // Running is every manager read and no unit failed. Running = "running" // Degraded is a unit failed in a manager read. Degraded = "degraded" // Unknown is no manager could be read. Unknown = "unknown" ) // owner is a module and the id of its resource that places a unit. type owner struct{ module, resource string } // Owned is what a declaration places on the machine, as the reading needs it: the units it states, the // files and packages it puts there, and the accounts whose managers it places units in. type Owned struct { // Units is keyed by manager and unit (key). Units map[string]owner // Judged is every unit liveness judges, by the same key: left to it. Judged map[string]bool // Files is keyed by path; Packages by package name, only those declared present. Files map[string]owner Packages map[string]owner // Accounts are the accounts whose own managers are read, sorted. Accounts []string } // key is a unit in one manager: "system/", or "user:/". func key(scope, user, unit string) string { if scope == ScopeUser { return "user:" + user + "/" + unit } return "system/" + unit } // OwnedBy reads what a declaration places. held is every resource an adopted machine holds as found, // by id — the machine's, not a module's. A resource the mesh declares in its own right (no module) names // no module: its failed unit is said with the machine's, under the resource's id. func OwnedBy(d *declaration.Declaration, held map[string]bool) Owned { o := Owned{Units: map[string]owner{}, Judged: map[string]bool{}, Files: map[string]owner{}, Packages: map[string]owner{}} if d == nil { return o } accounts := map[string]bool{} for _, r := range d.Resources { if held[r.Identity()] || strings.HasPrefix(r.Identity(), declaration.AdoptionPrefix) { continue } own := ownerOf(r.Identity()) switch v := r.(type) { case *declaration.Service: scope, user := ScopeSystem, "" if v.UserScoped() { scope, user = ScopeUser, v.User accounts[v.User] = true } k := key(scope, user, v.Unit) o.Units[k] = own if v.State == "running" { o.Judged[k] = true } case *declaration.Process: k := key(ScopeSystem, "", v.Name+".service") o.Units[k] = own if !v.RunOnce && v.Schedule == "" { o.Judged[k] = true } case *declaration.File: o.Files[v.Path] = own case *declaration.Package: if !v.Absent { o.Packages[v.Package] = own } case *declaration.User: accounts[v.Name] = true } } for a := range accounts { if a != "" { o.Accounts = append(o.Accounts, a) } } sort.Strings(o.Accounts) return o } // ownerOf is a resource id's module and the id itself: everything before the last dot is the module (as // liveness.ModuleOf reads it); an id with no dot is the mesh's own, and names no module. func ownerOf(id string) owner { at := strings.LastIndex(id, ".") if at <= 0 { return owner{resource: id} } return owner{module: id[:at], resource: id} } // Listed is one failed unit as its manager lists it. type Listed struct { Unit string // Load is loaded, not-found, masked, bad-setting… Load string } // Shown is what the manager says of one unit's file and how it failed. type Shown struct { FragmentPath string // Result is how it failed: exit-code, timeout, start-limit-hit… Result string } // Reader is what a look asks the machine. An error is "could not be read": that manager is said unread, // never healthy and never failed. type Reader interface { // Failed is every unit in failed state in a manager: the machine's (user empty), or an account's. Failed(ctx context.Context, scope, user string) ([]Listed, error) // Show is each unit's file and result, in a manager. Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error) // PackageOwning is the package a file belongs to; false when none does or the machine cannot say. PackageOwning(ctx context.Context, path string) (string, bool, error) } // Failed is one failed unit as the statement says it. type Failed struct { Unit string Scope string // User is the account whose manager it is in, for a user unit: evidence inside the mesh. User string Load string Result string // Module and Resource own it; empty when no module does. Via is how that was found. Module string Resource string Via string // Since is the first look that found it failed; Streak the looks in a row since. Since time.Time Streak int } // Statement is one look at every manager. type Statement struct { At time.Time // State is the worst of the managers read: running, degraded, or unknown when none was. State string // Failed is every unit failed on two looks in a row and not judged by liveness: the modules' and the // machine's. Failed []Failed // Unread names each manager that could not be read, with why. Unread []string } // Owned answers the failures a module owns; Unowned those no module does. func (s Statement) Owned() []Failed { return s.filter(true) } func (s Statement) Unowned() []Failed { return s.filter(false) } func (s Statement) filter(owned bool) []Failed { var out []Failed for _, f := range s.Failed { if (f.Module != "") == owned { out = append(out, f) } } return out } // seen is what the judge keeps of one failed unit between looks. type seen struct { since time.Time streak int } // Judge reads the machine's failed units on every look. Safe for the apply and the looking loop at once. type Judge struct { reader Reader Now func() time.Time mu sync.Mutex owned Owned // known says a declaration was set: before it, nothing is read — every unit would read as the // machine's, and then as a module's a moment later. known bool seen map[string]*seen owners map[string]ownerAnswer said string last Statement } // ownerAnswer is the package manager's answer for one unit file, kept until the declaration changes. type ownerAnswer struct { pkg string ok bool } // New is a judge reading through r. func New(r Reader) *Judge { return &Judge{reader: r, Now: time.Now, seen: map[string]*seen{}, owners: map[string]ownerAnswer{}} } // Set is what the declaration just applied places. The package manager is asked afresh after it, since a // package installed or removed changes whose a unit file is. func (j *Judge) Set(o Owned) { j.mu.Lock() defer j.mu.Unlock() j.owned, j.known = o, true j.owners = map[string]ownerAnswer{} } // Last is the statement of the last look. func (j *Judge) Last() Statement { j.mu.Lock() defer j.mu.Unlock() return j.last } // manager is one service manager read. type manager struct{ scope, user string } func (m manager) String() string { if m.scope == ScopeUser { return "the account manager of " + m.user } return "the machine's service manager" } // Look reads every manager once and answers the statement, and whether what it says changed since the // last look. Before a declaration is set it reads nothing and answers an empty statement, which says // nothing of the units. func (j *Judge) Look(ctx context.Context) (Statement, bool) { j.mu.Lock() defer j.mu.Unlock() if !j.known { return Statement{}, false } now := j.Now() managers := []manager{{scope: ScopeSystem}} for _, a := range j.owned.Accounts { managers = append(managers, manager{scope: ScopeUser, user: a}) } st := Statement{At: now, State: Unknown} read := 0 failedNow := map[string]bool{} for _, m := range managers { listed, err := j.reader.Failed(ctx, m.scope, m.user) if err != nil { st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error()))) // What it held is neither cleared nor counted while it cannot be read. for k := range j.seen { if strings.HasPrefix(k, key(m.scope, m.user, "")) { failedNow[k] = true } } continue } read++ if st.State == Unknown { st.State = Running } if len(listed) > 0 { st.State = Degraded } var names []string for _, l := range listed { names = append(names, l.Unit) } var shown map[string]Shown if len(names) > 0 { if shown, err = j.reader.Show(ctx, m.scope, m.user, names); err != nil { shown = map[string]Shown{} } } for _, l := range listed { k := key(m.scope, m.user, l.Unit) failedNow[k] = true s := j.seen[k] if s == nil { s = &seen{since: now} j.seen[k] = s } s.streak++ if j.owned.Judged[k] || s.streak < 2 { continue } f := Failed{Unit: l.Unit, Scope: m.scope, User: m.user, Load: l.Load, Result: shown[l.Unit].Result, Since: s.since, Streak: s.streak} if own, via, ok := j.ownerOfUnit(ctx, k, shown[l.Unit].FragmentPath); ok { f.Module, f.Resource, f.Via = own.module, own.resource, via } st.Failed = append(st.Failed, f) } } for k := range j.seen { if !failedNow[k] { delete(j.seen, k) } } sort.Slice(st.Failed, func(a, b int) bool { if st.Failed[a].Scope != st.Failed[b].Scope { return st.Failed[a].Scope < st.Failed[b].Scope } return st.Failed[a].Unit < st.Failed[b].Unit }) if read == 0 { st.State = Unknown } word := st.State for _, f := range st.Failed { word += "|" + f.Scope + "/" + f.Unit + "/" + f.Module } changed := word != j.said j.said, j.last = word, st return st, changed } // ownerOfUnit is whose a failed unit is: the declaration's unit, then the file the unit is, then the // package the file belongs to. func (j *Judge) ownerOfUnit(ctx context.Context, k, fragment string) (owner, string, bool) { if o, ok := j.owned.Units[k]; ok { return o, ViaUnit, true } if fragment == "" { return owner{}, "", false } if o, ok := j.owned.Files[fragment]; ok { return o, ViaFile, true } if len(j.owned.Packages) == 0 { return owner{}, "", false } a, asked := j.owners[fragment] if !asked { pkg, ok, err := j.reader.PackageOwning(ctx, fragment) if err != nil { // Not kept: asked again on the next look. return owner{}, "", false } a = ownerAnswer{pkg: pkg, ok: ok} j.owners[fragment] = a } if !a.ok { return owner{}, "", false } if o, ok := j.owned.Packages[a.pkg]; ok { return o, ViaPackage, true } return owner{}, "", false } func firstLine(s string) string { line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") return line }