diff --git a/internal/system/arch.go b/internal/system/arch.go index 09a4955..6c32a27 100644 --- a/internal/system/arch.go +++ b/internal/system/arch.go @@ -3,7 +3,10 @@ package system import ( "context" "fmt" + "os" + "path/filepath" "strings" + "time" "github.com/novox/mesh-host/internal/declaration" ) @@ -56,42 +59,119 @@ func (arch) InstallPackage(ctx context.Context, run Runner, name string) error { // **The package manager's own words, and a name for the case that looks like a bug in the // declaration and is not.** A stale index asks the mirrors for a version they have already // superseded and gets a 404 from every one of them — so the package exists, the declaration is - // correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). + // correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as + // old as the index fails one step later, on the signature of whatever a mirror still had. + // + // **Read from everything pacman said.** Its errors go to stderr, which the runner folds into + // the error rather than the output; this classifier read the output alone and so never saw a + // single "failed retrieving file", and the control node reported a ten-week-old database as + // a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205). // // **It is not fixed by syncing here.** `pacman -Sy ` installs a package built against // libraries this machine does not have: a partial upgrade, which Arch does not support and // which breaks the machine in a way that surfaces much later as something unrelated. The // remedy is a full upgrade, and it is a decision about the whole machine rather than - // something to do silently in the middle of applying one resource. - // - // So this says which of the two it is looking at. A declaration that is wrong and a machine - // that is out of date fail identically otherwise, and they are fixed in completely different - // places. - if staleIndex(out) { + // something to do silently in the middle of applying one resource. Whose decision, and on + // what schedule, is issue 205's question; until it is answered the host says what it sees. + said := strings.TrimSpace(out + "\n" + err.Error()) + switch classifyInstallFailure(said) { + case installStale: return fmt.Errorf( - "%s could not be fetched from any mirror, which is what a stale package index looks "+ - "like: this machine is asking for a version the mirrors have replaced. The "+ - "package and the declaration are probably both fine. It is fixed by upgrading "+ - "the machine, not by this host syncing one package — that would be a partial "+ - "upgrade, which this distribution does not support.\n\n%s", - name, strings.TrimSpace(out)) + "%s could not be fetched from any mirror, which is what a stale package index looks like: "+ + "the package database on this machine is %s and the mirrors no longer serve what it "+ + "names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+ + "operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+ + "host does not sync one package by itself, because on this distribution that is a "+ + "partial upgrade (novox/hq 04-ISSUES/205).\n\n%s", + name, syncDatabaseAge(), name, said) + case installMirrors: + return fmt.Errorf( + "no mirror could be reached to fetch %s, and the package database on this machine is "+ + "%s: this reads as the mirrors or the network, not as this machine being out of "+ + "date — try again when they answer.\n\n%s", + name, syncDatabaseAge(), said) } return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out)) } -// staleIndex reports whether a failed install looks like the machine's view being old rather than -// the package being wrong. -// -// By what the package manager said, because there is nothing else to go on: the exit code is the -// same for both. -func staleIndex(out string) bool { - said := strings.ToLower(out) - if !strings.Contains(said, "failed retrieving file") && !strings.Contains(said, "404") { - return false +// How a failed install is read, from what the package manager said. +type installFailure int + +const ( + installOther installFailure = iota + // installStale: the machine's package database or keyring is older than what the mirrors + // serve — every mirror 404s the file the database names, or a package that did arrive fails + // its signature against a keyring that never saw the key. + installStale + // installMirrors: no mirror could be reached at all, and nothing says the database is old. + installMirrors +) + +// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit +// code is the same for every one of these. +func classifyInstallFailure(said string) installFailure { + lower := strings.ToLower(said) + gone := strings.Count(lower, "returned error: 404") + fetching := strings.Contains(lower, "failed retrieving file") + badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") || + strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") || + strings.Contains(lower, "is unknown trust") || + strings.Contains(lower, "could not be looked up remotely") + switch { + case badSignature: + return installStale + case fetching && gone > 0: + // Every mirror, not one: a single mirror failing is an ordinary transient thing and + // retrying is the answer. pacman walks its whole mirror list before giving up, so more + // than one 404 among the lines is the index being old rather than one host being wrong. + if gone > 1 || !strings.Contains(lower, "could not resolve host") && + !strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") { + return installStale + } + return installMirrors + case fetching: + return installMirrors } - // Every mirror, not one. A single mirror failing is an ordinary transient thing and retrying - // is the answer; every one of them saying the file is gone is the index being old. - return strings.Contains(said, "error") || strings.Count(said, "404") > 1 + return installOther +} + +// staleIndex is the yes-or-no form older callers and tests use. +func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale } + +// syncDatabaseAge says how old this machine's package database is, in words a person acts on: +// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed +// install so a ten-week-old database is told apart from a mirror outage by reading one line. +// +// A variable so a test can say what the machine's database looks like without having one. +var syncDatabaseAge = func() string { + entries, err := filepath.Glob("/var/lib/pacman/sync/*.db") + if err != nil || len(entries) == 0 { + return "of unknown age (no sync database found under /var/lib/pacman/sync)" + } + var newest time.Time + for _, e := range entries { + info, err := os.Stat(e) + if err == nil && info.ModTime().After(newest) { + newest = info.ModTime() + } + } + if newest.IsZero() { + return "of unknown age" + } + return describeAge(newest, time.Now()) +} + +// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye. +func describeAge(when, now time.Time) string { + days := int(now.Sub(when).Hours() / 24) + span := fmt.Sprintf("%d days old", days) + switch { + case days < 1: + span = "less than a day old" + case days >= 14: + span = fmt.Sprintf("%d weeks old", days/7) + } + return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span) } // ServiceState reads what systemd says about a unit. diff --git a/internal/system/arch_install_test.go b/internal/system/arch_install_test.go new file mode 100644 index 0000000..6667879 --- /dev/null +++ b/internal/system/arch_install_test.go @@ -0,0 +1,115 @@ +package system + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +// pacman's own words from the control node on 2026-10-02 (novox/hq 04-ISSUES/205): every mirror +// 404s the versioned file a ten-week-old database names, and the one copy that arrives fails its +// signature. Errors are pacman's stderr, which the runner folds into the error, not the output. +const staleStderr = `error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.hetzner.com : The requested URL returned error: 404 +error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.rackspace.com : The requested URL returned error: 404 +error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from arch.lucassymons.net : Could not resolve host: arch.lucassymons.net +warning: fatal error from arch.lucassymons.net, skipping for the remainder of this transaction +error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirrors.cqu.edu.cn : Connection timed out after 10002 milliseconds +error: nodejs: signature from "Bert Peters (packager key) " is invalid +error: failed to commit transaction (invalid or corrupted package (PGP signature))` + +const staleStdout = `resolving dependencies... +looking for conflicting packages... + +Packages (4) ada-3.4.4-1 c-ares-1.34.8-1 simdjson-1:4.6.4-1 nodejs-26.5.0-1 + +:: Retrieving packages... + nodejs-26.5.0-1-x86_64 downloading... +checking keyring... +checking package integrity... +:: File /var/cache/pacman/pkg/nodejs-26.5.0-1-x86_64.pkg.tar.zst is corrupted (invalid or corrupted package (PGP signature)). +Errors occurred, no packages were upgraded.` + +// A runner that behaves as ExecRunner does on failure: stdout as the output, stderr in the error. +func pacmanFailing(stdout, stderr string) Runner { + return func(_ context.Context, name string, args ...string) (string, error) { + return stdout, errors.New(name + " exited 1: " + stderr) + } +} + +func TestAStaleDatabaseIsSaidAsOneWithItsAgeAndTheRemedy(t *testing.T) { + was := syncDatabaseAge + defer func() { syncDatabaseAge = was }() + syncDatabaseAge = func() string { + return describeAge(time.Date(2026, 7, 24, 16, 56, 0, 0, time.UTC), time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)) + } + + err := arch{}.InstallPackage(context.Background(), pacmanFailing(staleStdout, staleStderr), "nodejs") + if err == nil { + t.Fatal("a failed install must fail") + } + for _, want := range []string{ + "the package database on this machine is from 2026-07-24, 10 weeks old", + "stale package index", + "upgrading the machine — a full upgrade (`pacman -Syu`) by its operator — before the mesh can install nodejs", + "partial upgrade", + "returned error: 404", // pacman's own words follow + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("the error does not say %q:\n%s", want, err) + } + } + if strings.Contains(err.Error(), "mirrors or the network") { + t.Errorf("a stale database must not be read as a mirror outage:\n%s", err) + } +} + +// The same 404s read from stderr alone — the half this classifier used to be blind to. +func TestTheClassifierReadsWhatPacmanWroteToStderr(t *testing.T) { + if classifyInstallFailure(staleStderr) != installStale { + t.Fatal("every mirror 404ing the named file is a stale database") + } + if classifyInstallFailure(staleStdout) != installStale { + t.Fatal("a corrupted-signature line alone is a stale keyring") + } + if classifyInstallFailure("") != installOther { + t.Fatal("nothing said is nothing classified") + } +} + +func TestAnUnreachableMirrorWithAFreshDatabaseIsAMirrorProblem(t *testing.T) { + was := syncDatabaseAge + defer func() { syncDatabaseAge = was }() + syncDatabaseAge = func() string { return "from 2026-10-02, less than a day old" } + outage := `error: failed retrieving file 'core.db' from mirror.hetzner.com : Could not resolve host: mirror.hetzner.com +error: failed retrieving file 'core.db' from mirror.rackspace.com : Connection timed out after 10001 milliseconds +error: failed to synchronize all databases (failed to retrieve some files)` + if classifyInstallFailure(outage) != installMirrors { + t.Fatal("no mirror answering, no 404, no signature fault: the mirrors, not the machine") + } + err := arch{}.InstallPackage(context.Background(), pacmanFailing("", outage), "nodejs") + if err == nil || !strings.Contains(err.Error(), "mirrors or the network") || !strings.Contains(err.Error(), "less than a day old") { + t.Errorf("a mirror outage is said as one, with the database's age beside it:\n%v", err) + } +} + +func TestAFailureThatIsNeitherKeepsPacmansWords(t *testing.T) { + err := arch{}.InstallPackage(context.Background(), pacmanFailing("", "error: target not found: nodejsx"), "nodejsx") + if err == nil || !strings.Contains(err.Error(), "target not found") || strings.Contains(err.Error(), "package database on this machine") { + t.Errorf("an unknown package is pacman's own error, not a stale database:\n%v", err) + } +} + +func TestDescribeAge(t *testing.T) { + now := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC) + for when, want := range map[time.Time]string{ + now.Add(-2 * time.Hour): "less than a day old", + now.Add(-5 * 24 * time.Hour): "5 days old", + now.Add(-71 * 24 * time.Hour): "10 weeks old", + } { + if got := describeAge(when, now); !strings.HasSuffix(got, want) { + t.Errorf("%s: got %q, want suffix %q", when, got, want) + } + } +}