mesh-host/internal/units/units.go

// Package units is the node-engine reading which units its machine's service managers say failed, and
// whose each is (novox/hq issue 315, under ADR 0240 rule 1 and ADR 0241 §3).
//
// **A failed unit of a mesh module was never raised.** Liveness judges what a module runs long-lived — a
// container, a process, a service stated `running` — and nothing else. A module that installs a daemon as
// a package, whose unit the package ships and D-Bus activation starts, declares no service: its unit
// failed at every start and the machine read healthy, while the profile's `service-manager` said
// `degraded` and nothing raised that either. Two network mounts the operator wrote into the machine's
// own mount table, and a unit a removed package left behind, failed beside it, said by nobody.
//
// On every look the engine asks each service manager the mesh places units in — the machine's, and the
// account manager of every account the declaration places a user-scoped unit for — which units failed,
// and says each one's owner:
//
//  1. a service or process the declaration states, by its unit and manager;
//  2. a file the declaration writes, when the unit's file is that file;
//  3. a package the declaration installs, when the unit's file belongs to it — asked of the package
//     manager, once per unit file;
//
// and otherwise nobody's in the mesh: the machine's. A unit liveness already judges is left to it, so a
// failure is said once. **The two-look rule is the engine's** (ADR 0241 §2): a unit is said failed on
// its second look in a row, and no longer on its first look not failed.
//
// **It reads; it never acts** (ADR 0240 rule 6): `list-units`, `show` and the package manager's owner
// query are the whole of what it asks. It neither resets nor restarts anything.
//
// **A look starts no account's manager** (novox/hq issue 764, ADR 0177). Reading an account's manager is
// a login, and a login starts the manager when none runs; so the machine's own manager is asked first
// whether user@<uid>.service is active, and the account's is read only then. A manager that is not
// running is away: no unit in it is failed, none is read, and the statement says it away — the apply's
// word for the same absence ("waiting" for its manager).
package units

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
)

// The managers a unit is in.
const (
	ScopeSystem = declaration.ScopeSystem
	ScopeUser   = declaration.ScopeUser
)

// How a unit's owner was found.
const (
	ViaUnit    = "unit"
	ViaFile    = "file"
	ViaPackage = "package"
)

// The machine's service managers, as the statement says them.
const (
	// Running is every manager read and no unit failed.
	Running = "running"
	// Degraded is a unit failed in a manager read.
	Degraded = "degraded"
	// Unknown is no manager could be read.
	Unknown = "unknown"
)

// owner is a module and the id of its resource that places a unit.
type owner struct{ module, resource string }

// Owned is what a declaration places on the machine, as the reading needs it: the units it states, the
// files and packages it puts there, and the accounts whose managers it places units in.
type Owned struct {
	// Units is keyed by manager and unit (key).
	Units map[string]owner
	// Judged is every unit liveness judges, by the same key: left to it.
	Judged map[string]bool
	// Files is keyed by path; Packages by package name, only those declared present.
	Files    map[string]owner
	Packages map[string]owner
	// Accounts are the accounts whose own managers are read, sorted: each one the declaration places a
	// user-scoped unit for, and no other (novox/hq issue 764). An account the declaration only creates
	// has no unit of the mesh's in its manager to fail, and asking that manager starts it.
	Accounts []string
}

// key is a unit in one manager: "system/<unit>", or "user:<account>/<unit>".
func key(scope, user, unit string) string {
	if scope == ScopeUser {
		return "user:" + user + "/" + unit
	}
	return "system/" + unit
}

// OwnedBy reads what a declaration places. held is every resource an adopted machine holds as found,
// by id — the machine's, not a module's. A resource the mesh declares in its own right (no module) names
// no module: its failed unit is said with the machine's, under the resource's id.
func OwnedBy(d *declaration.Declaration, held map[string]bool) Owned {
	o := Owned{Units: map[string]owner{}, Judged: map[string]bool{}, Files: map[string]owner{},
		Packages: map[string]owner{}}
	if d == nil {
		return o
	}
	accounts := map[string]bool{}
	for _, r := range d.Resources {
		if held[r.Identity()] || strings.HasPrefix(r.Identity(), declaration.AdoptionPrefix) {
			continue
		}
		own := ownerOf(r.Identity())
		switch v := r.(type) {
		case *declaration.Service:
			scope, user := ScopeSystem, ""
			if v.UserScoped() {
				scope, user = ScopeUser, v.User
				accounts[v.User] = true
			}
			k := key(scope, user, v.Unit)
			o.Units[k] = own
			if v.State == "running" {
				o.Judged[k] = true
			}
		case *declaration.Process:
			k := key(ScopeSystem, "", v.Name+".service")
			o.Units[k] = own
			if !v.RunOnce && v.Schedule == "" {
				o.Judged[k] = true
			}
		case *declaration.File:
			o.Files[v.Path] = own
		case *declaration.Package:
			if !v.Absent {
				o.Packages[v.Package] = own
			}
		}
	}
	for a := range accounts {
		if a != "" {
			o.Accounts = append(o.Accounts, a)
		}
	}
	sort.Strings(o.Accounts)
	return o
}

// ownerOf is a resource id's module and the id itself: everything before the last dot is the module (as
// liveness.ModuleOf reads it); an id with no dot is the mesh's own, and names no module.
func ownerOf(id string) owner {
	at := strings.LastIndex(id, ".")
	if at <= 0 {
		return owner{resource: id}
	}
	return owner{module: id[:at], resource: id}
}

// Listed is one failed unit as its manager lists it.
type Listed struct {
	Unit string
	// Load is loaded, not-found, masked, bad-setting…
	Load string
}

// Shown is what the manager says of one unit's file and how it failed.
type Shown struct {
	FragmentPath string
	// Result is how it failed: exit-code, timeout, start-limit-hit…
	Result string
}

// Reader is what a look asks the machine. An error is "could not be read": that manager is said unread,
// never healthy and never failed.
type Reader interface {
	// Failed is every unit in failed state in a manager: the machine's (user empty), or an account's.
	Failed(ctx context.Context, scope, user string) ([]Listed, error)
	// Show is each unit's file and result, in a manager.
	Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error)
	// PackageOwning is the package a file belongs to; false when none does or the machine cannot say.
	PackageOwning(ctx context.Context, path string) (string, bool, error)
	// UserManagerRunning is whether an account's own manager runs now, asked of the machine's manager
	// and never of the account's, which a question would start (novox/hq issue 764); exists is false
	// for an account the machine does not have.
	UserManagerRunning(ctx context.Context, account string) (running, exists bool, err error)
}

// Failed is one failed unit as the statement says it.
type Failed struct {
	Unit  string
	Scope string
	// User is the account whose manager it is in, for a user unit: evidence inside the mesh.
	User   string
	Load   string
	Result string
	// Module and Resource own it; empty when no module does. Via is how that was found.
	Module   string
	Resource string
	Via      string
	// Since is the first look that found it failed; Streak the looks in a row since.
	Since  time.Time
	Streak int
}

// Statement is one look at every manager.
type Statement struct {
	At time.Time
	// State is the worst of the managers read: running, degraded, or unknown when none was.
	State string
	// Failed is every unit failed on two looks in a row and not judged by liveness: the modules' and the
	// machine's.
	Failed []Failed
	// Unread names each manager that could not be read, with why.
	Unread []string
	// Away names each account manager not running, so not read: its account is not logged in and does
	// not linger. Nothing in it is failed (novox/hq issue 764).
	Away []string
}

// Owned answers the failures a module owns; Unowned those no module does.
func (s Statement) Owned() []Failed   { return s.filter(true) }
func (s Statement) Unowned() []Failed { return s.filter(false) }

func (s Statement) filter(owned bool) []Failed {
	var out []Failed
	for _, f := range s.Failed {
		if (f.Module != "") == owned {
			out = append(out, f)
		}
	}
	return out
}

// seen is what the judge keeps of one failed unit between looks.
type seen struct {
	since  time.Time
	streak int
}

// Judge reads the machine's failed units on every look. Safe for the apply and the looking loop at once.
type Judge struct {
	reader Reader
	Now    func() time.Time

	mu    sync.Mutex
	owned Owned
	// known says a declaration was set: before it, nothing is read — every unit would read as the
	// machine's, and then as a module's a moment later.
	known  bool
	seen   map[string]*seen
	owners map[string]ownerAnswer
	said   string
	last   Statement
}

// ownerAnswer is the package manager's answer for one unit file, kept until the declaration changes.
type ownerAnswer struct {
	pkg string
	ok  bool
}

// New is a judge reading through r.
func New(r Reader) *Judge {
	return &Judge{reader: r, Now: time.Now, seen: map[string]*seen{}, owners: map[string]ownerAnswer{}}
}

// Set is what the declaration just applied places. The package manager is asked afresh after it, since a
// package installed or removed changes whose a unit file is.
func (j *Judge) Set(o Owned) {
	j.mu.Lock()
	defer j.mu.Unlock()
	j.owned, j.known = o, true
	j.owners = map[string]ownerAnswer{}
}

// Last is the statement of the last look.
func (j *Judge) Last() Statement {
	j.mu.Lock()
	defer j.mu.Unlock()
	return j.last
}

// manager is one service manager read.
type manager struct{ scope, user string }

func (m manager) String() string {
	if m.scope == ScopeUser {
		return "the account manager of " + m.user
	}
	return "the machine's service manager"
}

// Look reads every manager once and answers the statement, and whether what it says changed since the
// last look. Before a declaration is set it reads nothing and answers an empty statement, which says
// nothing of the units.
func (j *Judge) Look(ctx context.Context) (Statement, bool) {
	j.mu.Lock()
	defer j.mu.Unlock()
	if !j.known {
		return Statement{}, false
	}
	now := j.Now()
	managers := []manager{{scope: ScopeSystem}}
	for _, a := range j.owned.Accounts {
		managers = append(managers, manager{scope: ScopeUser, user: a})
	}
	st := Statement{At: now, State: Unknown}
	read := 0
	failedNow := map[string]bool{}
	keep := func(m manager) {
		// What it held is neither cleared nor counted while it cannot be read.
		for k := range j.seen {
			if strings.HasPrefix(k, key(m.scope, m.user, "")) {
				failedNow[k] = true
			}
		}
	}
	for _, m := range managers {
		if m.scope == ScopeUser {
			// Asked of the machine's manager first: reading the account's own would start it.
			running, exists, err := j.reader.UserManagerRunning(ctx, m.user)
			switch {
			case err != nil:
				st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error())))
				keep(m)
				continue
			case !exists:
				st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: the machine has no account %q",
					m, m.user))
				keep(m)
				continue
			case !running:
				// Away: nothing in it runs, so nothing in it is failed; what it held is cleared.
				st.Away = append(st.Away, m.user+"'s own service manager is not running: the account is not "+
					"logged in and does not linger, so its units are not read")
				continue
			}
		}
		listed, err := j.reader.Failed(ctx, m.scope, m.user)
		if err != nil {
			st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error())))
			keep(m)
			continue
		}
		read++
		if st.State == Unknown {
			st.State = Running
		}
		if len(listed) > 0 {
			st.State = Degraded
		}
		var names []string
		for _, l := range listed {
			names = append(names, l.Unit)
		}
		var shown map[string]Shown
		if len(names) > 0 {
			if shown, err = j.reader.Show(ctx, m.scope, m.user, names); err != nil {
				shown = map[string]Shown{}
			}
		}
		for _, l := range listed {
			k := key(m.scope, m.user, l.Unit)
			failedNow[k] = true
			s := j.seen[k]
			if s == nil {
				s = &seen{since: now}
				j.seen[k] = s
			}
			s.streak++
			if j.owned.Judged[k] || s.streak < 2 {
				continue
			}
			f := Failed{Unit: l.Unit, Scope: m.scope, User: m.user, Load: l.Load, Result: shown[l.Unit].Result,
				Since: s.since, Streak: s.streak}
			if own, via, ok := j.ownerOfUnit(ctx, k, shown[l.Unit].FragmentPath); ok {
				f.Module, f.Resource, f.Via = own.module, own.resource, via
			}
			st.Failed = append(st.Failed, f)
		}
	}
	for k := range j.seen {
		if !failedNow[k] {
			delete(j.seen, k)
		}
	}
	sort.Slice(st.Failed, func(a, b int) bool {
		if st.Failed[a].Scope != st.Failed[b].Scope {
			return st.Failed[a].Scope < st.Failed[b].Scope
		}
		return st.Failed[a].Unit < st.Failed[b].Unit
	})
	if read == 0 {
		st.State = Unknown
	}
	word := st.State + "|away:" + strings.Join(st.Away, ",")
	for _, f := range st.Failed {
		word += "|" + f.Scope + "/" + f.Unit + "/" + f.Module
	}
	changed := word != j.said
	j.said, j.last = word, st
	return st, changed
}

// ownerOfUnit is whose a failed unit is: the declaration's unit, then the file the unit is, then the
// package the file belongs to.
func (j *Judge) ownerOfUnit(ctx context.Context, k, fragment string) (owner, string, bool) {
	if o, ok := j.owned.Units[k]; ok {
		return o, ViaUnit, true
	}
	if fragment == "" {
		return owner{}, "", false
	}
	if o, ok := j.owned.Files[fragment]; ok {
		return o, ViaFile, true
	}
	if len(j.owned.Packages) == 0 {
		return owner{}, "", false
	}
	a, asked := j.owners[fragment]
	if !asked {
		pkg, ok, err := j.reader.PackageOwning(ctx, fragment)
		if err != nil {
			// Not kept: asked again on the next look.
			return owner{}, "", false
		}
		a = ownerAnswer{pkg: pkg, ok: ok}
		j.owners[fragment] = a
	}
	if !a.ok {
		return owner{}, "", false
	}
	if o, ok := j.owned.Packages[a.pkg]; ok {
		return o, ViaPackage, true
	}
	return owner{}, "", false
}

func firstLine(s string) string {
	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
	return line
}