A failed install says whether the package database is stale, how old it is, and what fixes it (hq issue 205)

pacman writes its errors to stderr, which the runner folds into the error rather than the output;
the stale-index classifier read the output alone and never saw a single 'failed retrieving file',
so a ten-week-old database on the control node reported as a wall of 404s from the mirrors. The
classifier now reads everything pacman said, knows a failed signature as the same staleness, tells a
mirror outage with a fresh database apart from it, and names the database's date and age beside the
remedy: a full upgrade by the operator, never a one-package sync, which on this distribution is a
partial upgrade. Whose job keeping the database current is stays issue 205's question.

internal/system/arch.go

@@ -3,7 +3,10 @@ package system
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/novox/mesh-host/internal/declaration"
 )
@@ -56,42 +59,119 @@ func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
 	// **The package manager's own words, and a name for the case that looks like a bug in the
 	// declaration and is not.** A stale index asks the mirrors for a version they have already
 	// superseded and gets a 404 from every one of them — so the package exists, the declaration is
-	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002).
+	// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as
+	// old as the index fails one step later, on the signature of whatever a mirror still had.
+	//
+	// **Read from everything pacman said.** Its errors go to stderr, which the runner folds into
+	// the error rather than the output; this classifier read the output alone and so never saw a
+	// single "failed retrieving file", and the control node reported a ten-week-old database as
+	// a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205).
 	//
 	// **It is not fixed by syncing here.** `pacman -Sy <pkg>` installs a package built against
 	// libraries this machine does not have: a partial upgrade, which Arch does not support and
 	// which breaks the machine in a way that surfaces much later as something unrelated. The
 	// remedy is a full upgrade, and it is a decision about the whole machine rather than
-	// something to do silently in the middle of applying one resource.
+	// something to do silently in the middle of applying one resource. Whose decision, and on
-	//
+	// what schedule, is issue 205's question; until it is answered the host says what it sees.
-	// So this says which of the two it is looking at. A declaration that is wrong and a machine
+	said := strings.TrimSpace(out + "\n" + err.Error())
-	// that is out of date fail identically otherwise, and they are fixed in completely different
+	switch classifyInstallFailure(said) {
-	// places.
+	case installStale:
-	if staleIndex(out) {
 		return fmt.Errorf(
-			"%s could not be fetched from any mirror, which is what a stale package index looks "+
+			"%s could not be fetched from any mirror, which is what a stale package index looks like: "+
-				"like: this machine is asking for a version the mirrors have replaced. The "+
+				"the package database on this machine is %s and the mirrors no longer serve what it "+
-				"package and the declaration are probably both fine. It is fixed by upgrading "+
+				"names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+
-				"the machine, not by this host syncing one package — that would be a partial "+
+				"operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+
-				"upgrade, which this distribution does not support.\n\n%s",
+				"host does not sync one package by itself, because on this distribution that is a "+
-			name, strings.TrimSpace(out))
+				"partial upgrade (novox/hq 04-ISSUES/205).\n\n%s",
+			name, syncDatabaseAge(), name, said)
+	case installMirrors:
+		return fmt.Errorf(
+			"no mirror could be reached to fetch %s, and the package database on this machine is "+
+				"%s: this reads as the mirrors or the network, not as this machine being out of "+
+				"date — try again when they answer.\n\n%s",
+			name, syncDatabaseAge(), said)
 	}
 	return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out))
 }
 
-// staleIndex reports whether a failed install looks like the machine's view being old rather than
+// How a failed install is read, from what the package manager said.
-// the package being wrong.
+type installFailure int
-//
+
-// By what the package manager said, because there is nothing else to go on: the exit code is the
+const (
-// same for both.
+	installOther installFailure = iota
-func staleIndex(out string) bool {
+	// installStale: the machine's package database or keyring is older than what the mirrors
-	said := strings.ToLower(out)
+	// serve — every mirror 404s the file the database names, or a package that did arrive fails
-	if !strings.Contains(said, "failed retrieving file") && !strings.Contains(said, "404") {
+	// its signature against a keyring that never saw the key.
-		return false
+	installStale
+	// installMirrors: no mirror could be reached at all, and nothing says the database is old.
+	installMirrors
+)
+
+// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit
+// code is the same for every one of these.
+func classifyInstallFailure(said string) installFailure {
+	lower := strings.ToLower(said)
+	gone := strings.Count(lower, "returned error: 404")
+	fetching := strings.Contains(lower, "failed retrieving file")
+	badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") ||
+		strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") ||
+		strings.Contains(lower, "is unknown trust") ||
+		strings.Contains(lower, "could not be looked up remotely")
+	switch {
+	case badSignature:
+		return installStale
+	case fetching && gone > 0:
+		// Every mirror, not one: a single mirror failing is an ordinary transient thing and
+		// retrying is the answer. pacman walks its whole mirror list before giving up, so more
+		// than one 404 among the lines is the index being old rather than one host being wrong.
+		if gone > 1 || !strings.Contains(lower, "could not resolve host") &&
+			!strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") {
+			return installStale
 		}
-	// Every mirror, not one. A single mirror failing is an ordinary transient thing and retrying
+		return installMirrors
-	// is the answer; every one of them saying the file is gone is the index being old.
+	case fetching:
-	return strings.Contains(said, "error") || strings.Count(said, "404") > 1
+		return installMirrors
+	}
+	return installOther
+}
+
+// staleIndex is the yes-or-no form older callers and tests use.
+func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale }
+
+// syncDatabaseAge says how old this machine's package database is, in words a person acts on:
+// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed
+// install so a ten-week-old database is told apart from a mirror outage by reading one line.
+//
+// A variable so a test can say what the machine's database looks like without having one.
+var syncDatabaseAge = func() string {
+	entries, err := filepath.Glob("/var/lib/pacman/sync/*.db")
+	if err != nil || len(entries) == 0 {
+		return "of unknown age (no sync database found under /var/lib/pacman/sync)"
+	}
+	var newest time.Time
+	for _, e := range entries {
+		info, err := os.Stat(e)
+		if err == nil && info.ModTime().After(newest) {
+			newest = info.ModTime()
+		}
+	}
+	if newest.IsZero() {
+		return "of unknown age"
+	}
+	return describeAge(newest, time.Now())
+}
+
+// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye.
+func describeAge(when, now time.Time) string {
+	days := int(now.Sub(when).Hours() / 24)
+	span := fmt.Sprintf("%d days old", days)
+	switch {
+	case days < 1:
+		span = "less than a day old"
+	case days >= 14:
+		span = fmt.Sprintf("%d weeks old", days/7)
+	}
+	return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span)
 }
 
 // ServiceState reads what systemd says about a unit.

internal/system/arch_install_test.go

@@ -0,0 +1,115 @@
+package system
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+)
+
+// pacman's own words from the control node on 2026-10-02 (novox/hq 04-ISSUES/205): every mirror
+// 404s the versioned file a ten-week-old database names, and the one copy that arrives fails its
+// signature. Errors are pacman's stderr, which the runner folds into the error, not the output.
+const staleStderr = `error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.hetzner.com : The requested URL returned error: 404
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.rackspace.com : The requested URL returned error: 404
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from arch.lucassymons.net : Could not resolve host: arch.lucassymons.net
+warning: fatal error from arch.lucassymons.net, skipping for the remainder of this transaction
+error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirrors.cqu.edu.cn : Connection timed out after 10002 milliseconds
+error: nodejs: signature from "Bert Peters (packager key) <bertptrs@archlinux.org>" is invalid
+error: failed to commit transaction (invalid or corrupted package (PGP signature))`
+
+const staleStdout = `resolving dependencies...
+looking for conflicting packages...
+
+Packages (4) ada-3.4.4-1  c-ares-1.34.8-1  simdjson-1:4.6.4-1  nodejs-26.5.0-1
+
+:: Retrieving packages...
+ nodejs-26.5.0-1-x86_64 downloading...
+checking keyring...
+checking package integrity...
+:: File /var/cache/pacman/pkg/nodejs-26.5.0-1-x86_64.pkg.tar.zst is corrupted (invalid or corrupted package (PGP signature)).
+Errors occurred, no packages were upgraded.`
+
+// A runner that behaves as ExecRunner does on failure: stdout as the output, stderr in the error.
+func pacmanFailing(stdout, stderr string) Runner {
+	return func(_ context.Context, name string, args ...string) (string, error) {
+		return stdout, errors.New(name + " exited 1: " + stderr)
+	}
+}
+
+func TestAStaleDatabaseIsSaidAsOneWithItsAgeAndTheRemedy(t *testing.T) {
+	was := syncDatabaseAge
+	defer func() { syncDatabaseAge = was }()
+	syncDatabaseAge = func() string {
+		return describeAge(time.Date(2026, 7, 24, 16, 56, 0, 0, time.UTC), time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC))
+	}
+
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing(staleStdout, staleStderr), "nodejs")
+	if err == nil {
+		t.Fatal("a failed install must fail")
+	}
+	for _, want := range []string{
+		"the package database on this machine is from 2026-07-24, 10 weeks old",
+		"stale package index",
+		"upgrading the machine — a full upgrade (`pacman -Syu`) by its operator — before the mesh can install nodejs",
+		"partial upgrade",
+		"returned error: 404", // pacman's own words follow
+	} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the error does not say %q:\n%s", want, err)
+		}
+	}
+	if strings.Contains(err.Error(), "mirrors or the network") {
+		t.Errorf("a stale database must not be read as a mirror outage:\n%s", err)
+	}
+}
+
+// The same 404s read from stderr alone — the half this classifier used to be blind to.
+func TestTheClassifierReadsWhatPacmanWroteToStderr(t *testing.T) {
+	if classifyInstallFailure(staleStderr) != installStale {
+		t.Fatal("every mirror 404ing the named file is a stale database")
+	}
+	if classifyInstallFailure(staleStdout) != installStale {
+		t.Fatal("a corrupted-signature line alone is a stale keyring")
+	}
+	if classifyInstallFailure("") != installOther {
+		t.Fatal("nothing said is nothing classified")
+	}
+}
+
+func TestAnUnreachableMirrorWithAFreshDatabaseIsAMirrorProblem(t *testing.T) {
+	was := syncDatabaseAge
+	defer func() { syncDatabaseAge = was }()
+	syncDatabaseAge = func() string { return "from 2026-10-02, less than a day old" }
+	outage := `error: failed retrieving file 'core.db' from mirror.hetzner.com : Could not resolve host: mirror.hetzner.com
+error: failed retrieving file 'core.db' from mirror.rackspace.com : Connection timed out after 10001 milliseconds
+error: failed to synchronize all databases (failed to retrieve some files)`
+	if classifyInstallFailure(outage) != installMirrors {
+		t.Fatal("no mirror answering, no 404, no signature fault: the mirrors, not the machine")
+	}
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", outage), "nodejs")
+	if err == nil || !strings.Contains(err.Error(), "mirrors or the network") || !strings.Contains(err.Error(), "less than a day old") {
+		t.Errorf("a mirror outage is said as one, with the database's age beside it:\n%v", err)
+	}
+}
+
+func TestAFailureThatIsNeitherKeepsPacmansWords(t *testing.T) {
+	err := arch{}.InstallPackage(context.Background(), pacmanFailing("", "error: target not found: nodejsx"), "nodejsx")
+	if err == nil || !strings.Contains(err.Error(), "target not found") || strings.Contains(err.Error(), "package database on this machine") {
+		t.Errorf("an unknown package is pacman's own error, not a stale database:\n%v", err)
+	}
+}
+
+func TestDescribeAge(t *testing.T) {
+	now := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
+	for when, want := range map[time.Time]string{
+		now.Add(-2 * time.Hour):       "less than a day old",
+		now.Add(-5 * 24 * time.Hour):  "5 days old",
+		now.Add(-71 * 24 * time.Hour): "10 weeks old",
+	} {
+		if got := describeAge(when, now); !strings.HasSuffix(got, want) {
+			t.Errorf("%s: got %q, want suffix %q", when, got, want)
+		}
+	}
+}