Files
mesh-host/internal/system/arch.go
T
jochen c9b963f8b9 A failed install says whether the package database is stale, how old it is, and what fixes it (hq issue 205)
pacman writes its errors to stderr, which the runner folds into the error rather than the output;
the stale-index classifier read the output alone and never saw a single 'failed retrieving file',
so a ten-week-old database on the control node reported as a wall of 404s from the mirrors. The
classifier now reads everything pacman said, knows a failed signature as the same staleness, tells a
mirror outage with a fresh database apart from it, and names the database's date and age beside the
remedy: a full upgrade by the operator, never a one-package sync, which on this distribution is a
partial upgrade. Whose job keeping the database current is stays issue 205's question.
2026-10-03 03:54:17 +02:00

367 lines
15 KiB
Go

package system
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// arch is pacman and systemd.
type arch struct{}
func (arch) Name() string { return "arch" }
func (arch) Shapes() []declaration.Type { return everyShape() }
func (a arch) Confirm(ctx context.Context, run Runner) error {
if _, err := run(ctx, "pacman", "-Q", "pacman"); err != nil {
return fmt.Errorf(
"this is the arch host and pacman does not answer here. Either this machine is not "+
"Arch, or its package database is broken: %w", err)
}
return nil
}
// PackageInstalled asks the package database, having first established that it answers.
//
// The two-step is the trap this file exists to remember. `pacman -Q name` exits non-zero for a
// package that is not installed AND for a database that cannot be read, so believing the first
// answer reports a broken package manager as "nothing is installed" — absence read as fact.
// Proving the tool answers about something that certainly exists separates them.
func (a arch) PackageInstalled(ctx context.Context, run Runner, name string) (bool, error) {
if err := a.Confirm(ctx, run); err != nil {
return false, fmt.Errorf("nothing can be said about %q: %w", name, err)
}
if _, err := run(ctx, "pacman", "-Q", name); err != nil {
return false, nil
}
return true, nil
}
// RemovePackage removes one package and nothing it depends on: `-R`, not `-Rs`, because what else
// relied on a dependency is not this declaration's to know. pacman keeps a configuration file the
// operator changed as `.pacsave`, which is what "never flushed" comes to once the front end is gone.
func (arch) RemovePackage(ctx context.Context, run Runner, name string) error {
_, err := run(ctx, "pacman", "-R", "--noconfirm", name)
return err
}
func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
out, err := run(ctx, "pacman", "-S", "--noconfirm", "--needed", name)
if err == nil {
return nil
}
// **The package manager's own words, and a name for the case that looks like a bug in the
// declaration and is not.** A stale index asks the mirrors for a version they have already
// superseded and gets a 404 from every one of them — so the package exists, the declaration is
// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as
// old as the index fails one step later, on the signature of whatever a mirror still had.
//
// **Read from everything pacman said.** Its errors go to stderr, which the runner folds into
// the error rather than the output; this classifier read the output alone and so never saw a
// single "failed retrieving file", and the control node reported a ten-week-old database as
// a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205).
//
// **It is not fixed by syncing here.** `pacman -Sy <pkg>` installs a package built against
// libraries this machine does not have: a partial upgrade, which Arch does not support and
// which breaks the machine in a way that surfaces much later as something unrelated. The
// remedy is a full upgrade, and it is a decision about the whole machine rather than
// something to do silently in the middle of applying one resource. Whose decision, and on
// what schedule, is issue 205's question; until it is answered the host says what it sees.
said := strings.TrimSpace(out + "\n" + err.Error())
switch classifyInstallFailure(said) {
case installStale:
return fmt.Errorf(
"%s could not be fetched from any mirror, which is what a stale package index looks like: "+
"the package database on this machine is %s and the mirrors no longer serve what it "+
"names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+
"operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+
"host does not sync one package by itself, because on this distribution that is a "+
"partial upgrade (novox/hq 04-ISSUES/205).\n\n%s",
name, syncDatabaseAge(), name, said)
case installMirrors:
return fmt.Errorf(
"no mirror could be reached to fetch %s, and the package database on this machine is "+
"%s: this reads as the mirrors or the network, not as this machine being out of "+
"date — try again when they answer.\n\n%s",
name, syncDatabaseAge(), said)
}
return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out))
}
// How a failed install is read, from what the package manager said.
type installFailure int
const (
installOther installFailure = iota
// installStale: the machine's package database or keyring is older than what the mirrors
// serve — every mirror 404s the file the database names, or a package that did arrive fails
// its signature against a keyring that never saw the key.
installStale
// installMirrors: no mirror could be reached at all, and nothing says the database is old.
installMirrors
)
// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit
// code is the same for every one of these.
func classifyInstallFailure(said string) installFailure {
lower := strings.ToLower(said)
gone := strings.Count(lower, "returned error: 404")
fetching := strings.Contains(lower, "failed retrieving file")
badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") ||
strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") ||
strings.Contains(lower, "is unknown trust") ||
strings.Contains(lower, "could not be looked up remotely")
switch {
case badSignature:
return installStale
case fetching && gone > 0:
// Every mirror, not one: a single mirror failing is an ordinary transient thing and
// retrying is the answer. pacman walks its whole mirror list before giving up, so more
// than one 404 among the lines is the index being old rather than one host being wrong.
if gone > 1 || !strings.Contains(lower, "could not resolve host") &&
!strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") {
return installStale
}
return installMirrors
case fetching:
return installMirrors
}
return installOther
}
// staleIndex is the yes-or-no form older callers and tests use.
func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale }
// syncDatabaseAge says how old this machine's package database is, in words a person acts on:
// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed
// install so a ten-week-old database is told apart from a mirror outage by reading one line.
//
// A variable so a test can say what the machine's database looks like without having one.
var syncDatabaseAge = func() string {
entries, err := filepath.Glob("/var/lib/pacman/sync/*.db")
if err != nil || len(entries) == 0 {
return "of unknown age (no sync database found under /var/lib/pacman/sync)"
}
var newest time.Time
for _, e := range entries {
info, err := os.Stat(e)
if err == nil && info.ModTime().After(newest) {
newest = info.ModTime()
}
}
if newest.IsZero() {
return "of unknown age"
}
return describeAge(newest, time.Now())
}
// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye.
func describeAge(when, now time.Time) string {
days := int(now.Sub(when).Hours() / 24)
span := fmt.Sprintf("%d days old", days)
switch {
case days < 1:
span = "less than a day old"
case days >= 14:
span = fmt.Sprintf("%d weeks old", days/7)
}
return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span)
}
// ServiceState reads what systemd says about a unit.
//
// Two traps, and both were hit before this read what it now reads.
//
// The exit code is not the answer: `is-active` exits non-zero for every state except active.
//
// And "inactive" does not mean stopped. `systemctl is-active` says "inactive" for a unit that
// DOES NOT EXIST exactly as it does for one installed and stopped, so declaring a unit stopped
// reported success for a unit the host cannot manage at all. LoadState is what separates them,
// so LoadState is what is read — and it is the thing an interface spanning systemd and OpenRC
// would have had to drop.
func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string, error) {
out, _ := run(ctx, "systemctl", "show", unit,
"--property=LoadState", "--property=ActiveState", "--property=Type",
"--property=RemainAfterExit", "--property=ExecMainStatus")
var load, active, kind, remains, exited string
for _, line := range strings.Split(out, "\n") {
key, value, found := strings.Cut(strings.TrimSpace(line), "=")
if !found {
continue
}
switch key {
case "LoadState":
load = value
case "ActiveState":
active = value
case "Type":
kind = value
case "RemainAfterExit":
remains = value
case "ExecMainStatus":
exited = value
}
}
switch load {
case "":
return "", fmt.Errorf("the service manager said nothing about %s", unit)
case "not-found":
return "", fmt.Errorf(
"%s does not exist on this machine. A declaration naming a unit that is not "+
"installed cannot be satisfied, and reporting it stopped would be reporting "+
"absence as success", unit)
case "masked":
return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit)
case "error", "bad-setting":
return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load)
}
// **A one-shot that finished is not stopped.** A unit whose whole job is to apply something
// and exit — load a rule set, set a sysctl — is reported inactive the moment it succeeds, and
// unless it is told to linger there is no state in which it is ever "active". Reading that as
// "stopped" makes such a unit permanently unsatisfiable: the host starts it, it does its work,
// it exits, the host reads back "stopped" and reports failure — for ever, on every apply,
// while the thing it configured is in place and working.
//
// That is not hypothetical. It is what the firewall did on every machine it was ever assigned
// to: rules loaded, service reported failed, the mesh reported a machine not doing what it was
// told, and the only visible symptom was a red line about a unit nobody could see anything
// wrong with.
//
// So for that shape, what "running" means is "it ran, and it worked".
if kind == "oneshot" && remains != "yes" && active == "inactive" {
if exited == "0" || exited == "" {
return "running", nil
}
return "stopped", nil
}
switch active {
case "active", "activating", "reloading":
return "running", nil
case "inactive", "failed", "deactivating":
return "stopped", nil
default:
return "", fmt.Errorf(
"the service manager reports %s as %q, which is neither running nor stopped", unit, active)
}
}
func (arch) SetServiceState(ctx context.Context, run Runner, unit, state string) error {
verb := "start"
if state == "stopped" {
verb = "stop"
}
_, err := run(ctx, "systemctl", verb, unit)
return err
}
// ServiceBoot reads whether a unit starts at boot.
//
// `is-enabled` has more than two answers, and `static` is the one that matters: the unit has no
// install section and CANNOT be enabled. Reading it as "disabled" would have the host try, fail,
// and blame the wrong thing — the same shape as reading a missing unit as "stopped".
func (arch) ServiceBoot(ctx context.Context, run Runner, unit string) (string, error) {
out, _ := run(ctx, "systemctl", "is-enabled", unit)
switch state := strings.TrimSpace(out); state {
case "enabled", "enabled-runtime", "alias":
return "enabled", nil
case "disabled":
return "disabled", nil
case "":
return "", fmt.Errorf("the service manager said nothing about whether %s starts at boot", unit)
case "static":
return "", fmt.Errorf(
"%s is static — it has no install section, so it cannot be enabled or disabled. "+
"Something else pulls it in, and that is what a declaration should name", unit)
case "masked", "masked-runtime":
return "", fmt.Errorf("%s is masked, so its boot state cannot be declared", unit)
default:
return "", fmt.Errorf(
"the service manager reports %s as %q at boot, which is neither enabled nor disabled",
unit, state)
}
}
func (arch) SetServiceBoot(ctx context.Context, run Runner, unit, boot string) error {
verb := "enable"
if boot == "disabled" {
verb = "disable"
}
_, err := run(ctx, "systemctl", verb, unit)
return err
}
// CreateUser makes a login with useradd.
//
// `--create-home` because a user whose home does not exist is a user nothing can be delivered
// to, and delivering a shell's configuration is most of why the mesh knows about users at all.
func (arch) CreateUser(ctx context.Context, run Runner, name, home, shell string) error {
args := []string{"--create-home"}
if home != "" {
args = append(args, "--home-dir", home)
}
if shell != "" {
args = append(args, "--shell", shell)
}
if _, err := run(ctx, "useradd", append(args, name)...); err != nil {
return fmt.Errorf("cannot create the user %q: %w", name, err)
}
return nil
}
func (arch) SetUserShell(ctx context.Context, run Runner, name, shell string) error {
if _, err := run(ctx, "usermod", "--shell", shell, name); err != nil {
return fmt.Errorf("cannot set %q's shell to %q: %w", name, shell, err)
}
return nil
}
// AddUserToGroup appends, and `--append` is the whole point: without it usermod REPLACES the
// user's supplementary groups, so a declaration naming one group would silently remove every
// other — including the ones that make a login able to use a machine at all.
func (arch) AddUserToGroup(ctx context.Context, run Runner, name, group string) error {
if _, err := run(ctx, "usermod", "--append", "--groups", group, name); err != nil {
return fmt.Errorf("cannot put %q in the group %q: %w", name, group, err)
}
return nil
}
// ServiceUnitFile says where the service manager loads a unit from — systemd's FragmentPath. It
// is how the host tells a unit an administrator installed, under /etc or /run, from one a package
// ships under /usr (novox/hq ADR 0103). Empty, with no error, for a unit that loads from nowhere.
func (arch) ServiceUnitFile(ctx context.Context, run Runner, unit string) (string, error) {
out, err := run(ctx, "systemctl", "show", unit, "--property=FragmentPath")
if err != nil {
return "", fmt.Errorf("the service manager did not say where %s comes from: %w", unit, err)
}
for _, line := range strings.Split(out, "\n") {
if path, ok := strings.CutPrefix(strings.TrimSpace(line), "FragmentPath="); ok {
return strings.TrimSpace(path), nil
}
}
return "", nil
}
// ReloadUnits has systemd read its unit files again. A unit file that changed on disk is otherwise
// ignored: a restart runs the unit systemd already loaded, and the new text only takes effect
// after a reload nobody asked for.
func (arch) ReloadUnits(ctx context.Context, run Runner) error {
_, err := run(ctx, "systemctl", "daemon-reload")
return err
}
// ReloadService tells a running unit to read its configuration again, without stopping it.
func (arch) ReloadService(ctx context.Context, run Runner, unit string) error {
_, err := run(ctx, "systemctl", "reload", unit)
return err
}