pacman writes its errors to stderr, which the runner folds into the error rather than the output; the stale-index classifier read the output alone and never saw a single 'failed retrieving file', so a ten-week-old database on the control node reported as a wall of 404s from the mirrors. The classifier now reads everything pacman said, knows a failed signature as the same staleness, tells a mirror outage with a fresh database apart from it, and names the database's date and age beside the remedy: a full upgrade by the operator, never a one-package sync, which on this distribution is a partial upgrade. Whose job keeping the database current is stays issue 205's question.
367 lines
15 KiB
Go
367 lines
15 KiB
Go
package system
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
)
|
|
|
|
// arch is pacman and systemd.
|
|
type arch struct{}
|
|
|
|
func (arch) Name() string { return "arch" }
|
|
func (arch) Shapes() []declaration.Type { return everyShape() }
|
|
|
|
func (a arch) Confirm(ctx context.Context, run Runner) error {
|
|
if _, err := run(ctx, "pacman", "-Q", "pacman"); err != nil {
|
|
return fmt.Errorf(
|
|
"this is the arch host and pacman does not answer here. Either this machine is not "+
|
|
"Arch, or its package database is broken: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// PackageInstalled asks the package database, having first established that it answers.
|
|
//
|
|
// The two-step is the trap this file exists to remember. `pacman -Q name` exits non-zero for a
|
|
// package that is not installed AND for a database that cannot be read, so believing the first
|
|
// answer reports a broken package manager as "nothing is installed" — absence read as fact.
|
|
// Proving the tool answers about something that certainly exists separates them.
|
|
func (a arch) PackageInstalled(ctx context.Context, run Runner, name string) (bool, error) {
|
|
if err := a.Confirm(ctx, run); err != nil {
|
|
return false, fmt.Errorf("nothing can be said about %q: %w", name, err)
|
|
}
|
|
if _, err := run(ctx, "pacman", "-Q", name); err != nil {
|
|
return false, nil
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// RemovePackage removes one package and nothing it depends on: `-R`, not `-Rs`, because what else
|
|
// relied on a dependency is not this declaration's to know. pacman keeps a configuration file the
|
|
// operator changed as `.pacsave`, which is what "never flushed" comes to once the front end is gone.
|
|
func (arch) RemovePackage(ctx context.Context, run Runner, name string) error {
|
|
_, err := run(ctx, "pacman", "-R", "--noconfirm", name)
|
|
return err
|
|
}
|
|
|
|
func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
|
|
out, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", name)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
// **The package manager's own words, and a name for the case that looks like a bug in the
|
|
// declaration and is not.** A stale index asks the mirrors for a version they have already
|
|
// superseded and gets a 404 from every one of them — so the package exists, the declaration is
|
|
// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as
|
|
// old as the index fails one step later, on the signature of whatever a mirror still had.
|
|
//
|
|
// **Read from everything pacman said.** Its errors go to stderr, which the runner folds into
|
|
// the error rather than the output; this classifier read the output alone and so never saw a
|
|
// single "failed retrieving file", and the control node reported a ten-week-old database as
|
|
// a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205).
|
|
//
|
|
// **It is not fixed by syncing here.** `pacman -Sy <pkg>` installs a package built against
|
|
// libraries this machine does not have: a partial upgrade, which Arch does not support and
|
|
// which breaks the machine in a way that surfaces much later as something unrelated. The
|
|
// remedy is a full upgrade, and it is a decision about the whole machine rather than
|
|
// something to do silently in the middle of applying one resource. Whose decision, and on
|
|
// what schedule, is issue 205's question; until it is answered the host says what it sees.
|
|
said := strings.TrimSpace(out + "\n" + err.Error())
|
|
switch classifyInstallFailure(said) {
|
|
case installStale:
|
|
return fmt.Errorf(
|
|
"%s could not be fetched from any mirror, which is what a stale package index looks like: "+
|
|
"the package database on this machine is %s and the mirrors no longer serve what it "+
|
|
"names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+
|
|
"operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+
|
|
"host does not sync one package by itself, because on this distribution that is a "+
|
|
"partial upgrade (novox/hq 04-ISSUES/205).\n\n%s",
|
|
name, syncDatabaseAge(), name, said)
|
|
case installMirrors:
|
|
return fmt.Errorf(
|
|
"no mirror could be reached to fetch %s, and the package database on this machine is "+
|
|
"%s: this reads as the mirrors or the network, not as this machine being out of "+
|
|
"date — try again when they answer.\n\n%s",
|
|
name, syncDatabaseAge(), said)
|
|
}
|
|
return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out))
|
|
}
|
|
|
|
// How a failed install is read, from what the package manager said.
|
|
type installFailure int
|
|
|
|
const (
|
|
installOther installFailure = iota
|
|
// installStale: the machine's package database or keyring is older than what the mirrors
|
|
// serve — every mirror 404s the file the database names, or a package that did arrive fails
|
|
// its signature against a keyring that never saw the key.
|
|
installStale
|
|
// installMirrors: no mirror could be reached at all, and nothing says the database is old.
|
|
installMirrors
|
|
)
|
|
|
|
// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit
|
|
// code is the same for every one of these.
|
|
func classifyInstallFailure(said string) installFailure {
|
|
lower := strings.ToLower(said)
|
|
gone := strings.Count(lower, "returned error: 404")
|
|
fetching := strings.Contains(lower, "failed retrieving file")
|
|
badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") ||
|
|
strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") ||
|
|
strings.Contains(lower, "is unknown trust") ||
|
|
strings.Contains(lower, "could not be looked up remotely")
|
|
switch {
|
|
case badSignature:
|
|
return installStale
|
|
case fetching && gone > 0:
|
|
// Every mirror, not one: a single mirror failing is an ordinary transient thing and
|
|
// retrying is the answer. pacman walks its whole mirror list before giving up, so more
|
|
// than one 404 among the lines is the index being old rather than one host being wrong.
|
|
if gone > 1 || !strings.Contains(lower, "could not resolve host") &&
|
|
!strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") {
|
|
return installStale
|
|
}
|
|
return installMirrors
|
|
case fetching:
|
|
return installMirrors
|
|
}
|
|
return installOther
|
|
}
|
|
|
|
// staleIndex is the yes-or-no form older callers and tests use.
|
|
func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale }
|
|
|
|
// syncDatabaseAge says how old this machine's package database is, in words a person acts on:
|
|
// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed
|
|
// install so a ten-week-old database is told apart from a mirror outage by reading one line.
|
|
//
|
|
// A variable so a test can say what the machine's database looks like without having one.
|
|
var syncDatabaseAge = func() string {
|
|
entries, err := filepath.Glob("/var/lib/pacman/sync/*.db")
|
|
if err != nil || len(entries) == 0 {
|
|
return "of unknown age (no sync database found under /var/lib/pacman/sync)"
|
|
}
|
|
var newest time.Time
|
|
for _, e := range entries {
|
|
info, err := os.Stat(e)
|
|
if err == nil && info.ModTime().After(newest) {
|
|
newest = info.ModTime()
|
|
}
|
|
}
|
|
if newest.IsZero() {
|
|
return "of unknown age"
|
|
}
|
|
return describeAge(newest, time.Now())
|
|
}
|
|
|
|
// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye.
|
|
func describeAge(when, now time.Time) string {
|
|
days := int(now.Sub(when).Hours() / 24)
|
|
span := fmt.Sprintf("%d days old", days)
|
|
switch {
|
|
case days < 1:
|
|
span = "less than a day old"
|
|
case days >= 14:
|
|
span = fmt.Sprintf("%d weeks old", days/7)
|
|
}
|
|
return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span)
|
|
}
|
|
|
|
// ServiceState reads what systemd says about a unit.
|
|
//
|
|
// Two traps, and both were hit before this read what it now reads.
|
|
//
|
|
// The exit code is not the answer: `is-active` exits non-zero for every state except active.
|
|
//
|
|
// And "inactive" does not mean stopped. `systemctl is-active` says "inactive" for a unit that
|
|
// DOES NOT EXIST exactly as it does for one installed and stopped, so declaring a unit stopped
|
|
// reported success for a unit the host cannot manage at all. LoadState is what separates them,
|
|
// so LoadState is what is read — and it is the thing an interface spanning systemd and OpenRC
|
|
// would have had to drop.
|
|
func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string, error) {
|
|
out, _ := run(ctx, "systemctl", "show", unit,
|
|
"--property=LoadState", "--property=ActiveState", "--property=Type",
|
|
"--property=RemainAfterExit", "--property=ExecMainStatus")
|
|
|
|
var load, active, kind, remains, exited string
|
|
for _, line := range strings.Split(out, "\n") {
|
|
key, value, found := strings.Cut(strings.TrimSpace(line), "=")
|
|
if !found {
|
|
continue
|
|
}
|
|
switch key {
|
|
case "LoadState":
|
|
load = value
|
|
case "ActiveState":
|
|
active = value
|
|
case "Type":
|
|
kind = value
|
|
case "RemainAfterExit":
|
|
remains = value
|
|
case "ExecMainStatus":
|
|
exited = value
|
|
}
|
|
}
|
|
|
|
switch load {
|
|
case "":
|
|
return "", fmt.Errorf("the service manager said nothing about %s", unit)
|
|
case "not-found":
|
|
return "", fmt.Errorf(
|
|
"%s does not exist on this machine. A declaration naming a unit that is not "+
|
|
"installed cannot be satisfied, and reporting it stopped would be reporting "+
|
|
"absence as success", unit)
|
|
case "masked":
|
|
return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit)
|
|
case "error", "bad-setting":
|
|
return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load)
|
|
}
|
|
|
|
// **A one-shot that finished is not stopped.** A unit whose whole job is to apply something
|
|
// and exit — load a rule set, set a sysctl — is reported inactive the moment it succeeds, and
|
|
// unless it is told to linger there is no state in which it is ever "active". Reading that as
|
|
// "stopped" makes such a unit permanently unsatisfiable: the host starts it, it does its work,
|
|
// it exits, the host reads back "stopped" and reports failure — for ever, on every apply,
|
|
// while the thing it configured is in place and working.
|
|
//
|
|
// That is not hypothetical. It is what the firewall did on every machine it was ever assigned
|
|
// to: rules loaded, service reported failed, the mesh reported a machine not doing what it was
|
|
// told, and the only visible symptom was a red line about a unit nobody could see anything
|
|
// wrong with.
|
|
//
|
|
// So for that shape, what "running" means is "it ran, and it worked".
|
|
if kind == "oneshot" && remains != "yes" && active == "inactive" {
|
|
if exited == "0" || exited == "" {
|
|
return "running", nil
|
|
}
|
|
return "stopped", nil
|
|
}
|
|
|
|
switch active {
|
|
case "active", "activating", "reloading":
|
|
return "running", nil
|
|
case "inactive", "failed", "deactivating":
|
|
return "stopped", nil
|
|
default:
|
|
return "", fmt.Errorf(
|
|
"the service manager reports %s as %q, which is neither running nor stopped", unit, active)
|
|
}
|
|
}
|
|
|
|
func (arch) SetServiceState(ctx context.Context, run Runner, unit, state string) error {
|
|
verb := "start"
|
|
if state == "stopped" {
|
|
verb = "stop"
|
|
}
|
|
_, err := run(ctx, "systemctl", verb, unit)
|
|
return err
|
|
}
|
|
|
|
// ServiceBoot reads whether a unit starts at boot.
|
|
//
|
|
// `is-enabled` has more than two answers, and `static` is the one that matters: the unit has no
|
|
// install section and CANNOT be enabled. Reading it as "disabled" would have the host try, fail,
|
|
// and blame the wrong thing — the same shape as reading a missing unit as "stopped".
|
|
func (arch) ServiceBoot(ctx context.Context, run Runner, unit string) (string, error) {
|
|
out, _ := run(ctx, "systemctl", "is-enabled", unit)
|
|
switch state := strings.TrimSpace(out); state {
|
|
case "enabled", "enabled-runtime", "alias":
|
|
return "enabled", nil
|
|
case "disabled":
|
|
return "disabled", nil
|
|
case "":
|
|
return "", fmt.Errorf("the service manager said nothing about whether %s starts at boot", unit)
|
|
case "static":
|
|
return "", fmt.Errorf(
|
|
"%s is static — it has no install section, so it cannot be enabled or disabled. "+
|
|
"Something else pulls it in, and that is what a declaration should name", unit)
|
|
case "masked", "masked-runtime":
|
|
return "", fmt.Errorf("%s is masked, so its boot state cannot be declared", unit)
|
|
default:
|
|
return "", fmt.Errorf(
|
|
"the service manager reports %s as %q at boot, which is neither enabled nor disabled",
|
|
unit, state)
|
|
}
|
|
}
|
|
|
|
func (arch) SetServiceBoot(ctx context.Context, run Runner, unit, boot string) error {
|
|
verb := "enable"
|
|
if boot == "disabled" {
|
|
verb = "disable"
|
|
}
|
|
_, err := run(ctx, "systemctl", verb, unit)
|
|
return err
|
|
}
|
|
|
|
// CreateUser makes a login with useradd.
|
|
//
|
|
// `--create-home` because a user whose home does not exist is a user nothing can be delivered
|
|
// to, and delivering a shell's configuration is most of why the mesh knows about users at all.
|
|
func (arch) CreateUser(ctx context.Context, run Runner, name, home, shell string) error {
|
|
args := []string{"--create-home"}
|
|
if home != "" {
|
|
args = append(args, "--home-dir", home)
|
|
}
|
|
if shell != "" {
|
|
args = append(args, "--shell", shell)
|
|
}
|
|
if _, err := run(ctx, "useradd", append(args, name)...); err != nil {
|
|
return fmt.Errorf("cannot create the user %q: %w", name, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (arch) SetUserShell(ctx context.Context, run Runner, name, shell string) error {
|
|
if _, err := run(ctx, "usermod", "--shell", shell, name); err != nil {
|
|
return fmt.Errorf("cannot set %q's shell to %q: %w", name, shell, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AddUserToGroup appends, and `--append` is the whole point: without it usermod REPLACES the
|
|
// user's supplementary groups, so a declaration naming one group would silently remove every
|
|
// other — including the ones that make a login able to use a machine at all.
|
|
func (arch) AddUserToGroup(ctx context.Context, run Runner, name, group string) error {
|
|
if _, err := run(ctx, "usermod", "--append", "--groups", group, name); err != nil {
|
|
return fmt.Errorf("cannot put %q in the group %q: %w", name, group, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ServiceUnitFile says where the service manager loads a unit from — systemd's FragmentPath. It
|
|
// is how the host tells a unit an administrator installed, under /etc or /run, from one a package
|
|
// ships under /usr (novox/hq ADR 0103). Empty, with no error, for a unit that loads from nowhere.
|
|
func (arch) ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) {
|
|
out, err := run(ctx, "systemctl", "show", unit, "--property=FragmentPath")
|
|
if err != nil {
|
|
return "", fmt.Errorf("the service manager did not say where %s comes from: %w", unit, err)
|
|
}
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if path, ok := strings.CutPrefix(strings.TrimSpace(line), "FragmentPath="); ok {
|
|
return strings.TrimSpace(path), nil
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
// ReloadUnits has systemd read its unit files again. A unit file that changed on disk is otherwise
|
|
// ignored: a restart runs the unit systemd already loaded, and the new text only takes effect
|
|
// after a reload nobody asked for.
|
|
func (arch) ReloadUnits(ctx context.Context, run Runner) error {
|
|
_, err := run(ctx, "systemctl", "daemon-reload")
|
|
return err
|
|
}
|
|
|
|
// ReloadService tells a running unit to read its configuration again, without stopping it.
|
|
func (arch) ReloadService(ctx context.Context, run Runner, unit string) error {
|
|
_, err := run(ctx, "systemctl", "reload", unit)
|
|
return err
|
|
}
|