A failed install says whether the package database is stale, how old it is, and what fixes it (hq issue 205) #79
+105
-25
@@ -3,7 +3,10 @@ package system
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
@@ -56,42 +59,119 @@ func (arch) InstallPackage(ctx context.Context, run Runner, name string) error {
|
||||
// **The package manager's own words, and a name for the case that looks like a bug in the
|
||||
// declaration and is not.** A stale index asks the mirrors for a version they have already
|
||||
// superseded and gets a 404 from every one of them — so the package exists, the declaration is
|
||||
// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002).
|
||||
// correct, and the machine's idea of what exists is old (novox/hq 04-ISSUES/002). A keyring as
|
||||
// old as the index fails one step later, on the signature of whatever a mirror still had.
|
||||
//
|
||||
// **Read from everything pacman said.** Its errors go to stderr, which the runner folds into
|
||||
// the error rather than the output; this classifier read the output alone and so never saw a
|
||||
// single "failed retrieving file", and the control node reported a ten-week-old database as
|
||||
// a mirror outage with a wall of 404s (novox/hq 04-ISSUES/205).
|
||||
//
|
||||
// **It is not fixed by syncing here.** `pacman -Sy <pkg>` installs a package built against
|
||||
// libraries this machine does not have: a partial upgrade, which Arch does not support and
|
||||
// which breaks the machine in a way that surfaces much later as something unrelated. The
|
||||
// remedy is a full upgrade, and it is a decision about the whole machine rather than
|
||||
// something to do silently in the middle of applying one resource.
|
||||
//
|
||||
// So this says which of the two it is looking at. A declaration that is wrong and a machine
|
||||
// that is out of date fail identically otherwise, and they are fixed in completely different
|
||||
// places.
|
||||
if staleIndex(out) {
|
||||
// something to do silently in the middle of applying one resource. Whose decision, and on
|
||||
// what schedule, is issue 205's question; until it is answered the host says what it sees.
|
||||
said := strings.TrimSpace(out + "\n" + err.Error())
|
||||
switch classifyInstallFailure(said) {
|
||||
case installStale:
|
||||
return fmt.Errorf(
|
||||
"%s could not be fetched from any mirror, which is what a stale package index looks "+
|
||||
"like: this machine is asking for a version the mirrors have replaced. The "+
|
||||
"package and the declaration are probably both fine. It is fixed by upgrading "+
|
||||
"the machine, not by this host syncing one package — that would be a partial "+
|
||||
"upgrade, which this distribution does not support.\n\n%s",
|
||||
name, strings.TrimSpace(out))
|
||||
"%s could not be fetched from any mirror, which is what a stale package index looks like: "+
|
||||
"the package database on this machine is %s and the mirrors no longer serve what it "+
|
||||
"names. It is fixed by upgrading the machine — a full upgrade (`pacman -Syu`) by its "+
|
||||
"operator — before the mesh can install %s. The package and the declaration are probably both fine; the "+
|
||||
"host does not sync one package by itself, because on this distribution that is a "+
|
||||
"partial upgrade (novox/hq 04-ISSUES/205).\n\n%s",
|
||||
name, syncDatabaseAge(), name, said)
|
||||
case installMirrors:
|
||||
return fmt.Errorf(
|
||||
"no mirror could be reached to fetch %s, and the package database on this machine is "+
|
||||
"%s: this reads as the mirrors or the network, not as this machine being out of "+
|
||||
"date — try again when they answer.\n\n%s",
|
||||
name, syncDatabaseAge(), said)
|
||||
}
|
||||
return fmt.Errorf("%w\n\n%s", err, strings.TrimSpace(out))
|
||||
}
|
||||
|
||||
// staleIndex reports whether a failed install looks like the machine's view being old rather than
|
||||
// the package being wrong.
|
||||
//
|
||||
// By what the package manager said, because there is nothing else to go on: the exit code is the
|
||||
// same for both.
|
||||
func staleIndex(out string) bool {
|
||||
said := strings.ToLower(out)
|
||||
if !strings.Contains(said, "failed retrieving file") && !strings.Contains(said, "404") {
|
||||
return false
|
||||
// How a failed install is read, from what the package manager said.
|
||||
type installFailure int
|
||||
|
||||
const (
|
||||
installOther installFailure = iota
|
||||
// installStale: the machine's package database or keyring is older than what the mirrors
|
||||
// serve — every mirror 404s the file the database names, or a package that did arrive fails
|
||||
// its signature against a keyring that never saw the key.
|
||||
installStale
|
||||
// installMirrors: no mirror could be reached at all, and nothing says the database is old.
|
||||
installMirrors
|
||||
)
|
||||
|
||||
// classifyInstallFailure reads pacman's words, because there is nothing else to go on: the exit
|
||||
// code is the same for every one of these.
|
||||
func classifyInstallFailure(said string) installFailure {
|
||||
lower := strings.ToLower(said)
|
||||
gone := strings.Count(lower, "returned error: 404")
|
||||
fetching := strings.Contains(lower, "failed retrieving file")
|
||||
badSignature := strings.Contains(lower, "invalid or corrupted package (pgp signature)") ||
|
||||
strings.Contains(lower, "signature from") && strings.Contains(lower, "is invalid") ||
|
||||
strings.Contains(lower, "is unknown trust") ||
|
||||
strings.Contains(lower, "could not be looked up remotely")
|
||||
switch {
|
||||
case badSignature:
|
||||
return installStale
|
||||
case fetching && gone > 0:
|
||||
// Every mirror, not one: a single mirror failing is an ordinary transient thing and
|
||||
// retrying is the answer. pacman walks its whole mirror list before giving up, so more
|
||||
// than one 404 among the lines is the index being old rather than one host being wrong.
|
||||
if gone > 1 || !strings.Contains(lower, "could not resolve host") &&
|
||||
!strings.Contains(lower, "connection timed out") && !strings.Contains(lower, "failed to connect") {
|
||||
return installStale
|
||||
}
|
||||
// Every mirror, not one. A single mirror failing is an ordinary transient thing and retrying
|
||||
// is the answer; every one of them saying the file is gone is the index being old.
|
||||
return strings.Contains(said, "error") || strings.Count(said, "404") > 1
|
||||
return installMirrors
|
||||
case fetching:
|
||||
return installMirrors
|
||||
}
|
||||
return installOther
|
||||
}
|
||||
|
||||
// staleIndex is the yes-or-no form older callers and tests use.
|
||||
func staleIndex(out string) bool { return classifyInstallFailure(out) == installStale }
|
||||
|
||||
// syncDatabaseAge says how old this machine's package database is, in words a person acts on:
|
||||
// the newest of pacman's sync databases, dated, and how long ago that was. Said beside a failed
|
||||
// install so a ten-week-old database is told apart from a mirror outage by reading one line.
|
||||
//
|
||||
// A variable so a test can say what the machine's database looks like without having one.
|
||||
var syncDatabaseAge = func() string {
|
||||
entries, err := filepath.Glob("/var/lib/pacman/sync/*.db")
|
||||
if err != nil || len(entries) == 0 {
|
||||
return "of unknown age (no sync database found under /var/lib/pacman/sync)"
|
||||
}
|
||||
var newest time.Time
|
||||
for _, e := range entries {
|
||||
info, err := os.Stat(e)
|
||||
if err == nil && info.ModTime().After(newest) {
|
||||
newest = info.ModTime()
|
||||
}
|
||||
}
|
||||
if newest.IsZero() {
|
||||
return "of unknown age"
|
||||
}
|
||||
return describeAge(newest, time.Now())
|
||||
}
|
||||
|
||||
// describeAge is "from 2026-07-24, 10 weeks old" — the date for the record, the span for the eye.
|
||||
func describeAge(when, now time.Time) string {
|
||||
days := int(now.Sub(when).Hours() / 24)
|
||||
span := fmt.Sprintf("%d days old", days)
|
||||
switch {
|
||||
case days < 1:
|
||||
span = "less than a day old"
|
||||
case days >= 14:
|
||||
span = fmt.Sprintf("%d weeks old", days/7)
|
||||
}
|
||||
return fmt.Sprintf("from %s, %s", when.Format("2006-01-02"), span)
|
||||
}
|
||||
|
||||
// ServiceState reads what systemd says about a unit.
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package system
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// pacman's own words from the control node on 2026-10-02 (novox/hq 04-ISSUES/205): every mirror
|
||||
// 404s the versioned file a ten-week-old database names, and the one copy that arrives fails its
|
||||
// signature. Errors are pacman's stderr, which the runner folds into the error, not the output.
|
||||
const staleStderr = `error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.hetzner.com : The requested URL returned error: 404
|
||||
error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirror.rackspace.com : The requested URL returned error: 404
|
||||
error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from arch.lucassymons.net : Could not resolve host: arch.lucassymons.net
|
||||
warning: fatal error from arch.lucassymons.net, skipping for the remainder of this transaction
|
||||
error: failed retrieving file 'nodejs-26.5.0-1-x86_64.pkg.tar.zst' from mirrors.cqu.edu.cn : Connection timed out after 10002 milliseconds
|
||||
error: nodejs: signature from "Bert Peters (packager key) <bertptrs@archlinux.org>" is invalid
|
||||
error: failed to commit transaction (invalid or corrupted package (PGP signature))`
|
||||
|
||||
const staleStdout = `resolving dependencies...
|
||||
looking for conflicting packages...
|
||||
|
||||
Packages (4) ada-3.4.4-1 c-ares-1.34.8-1 simdjson-1:4.6.4-1 nodejs-26.5.0-1
|
||||
|
||||
:: Retrieving packages...
|
||||
nodejs-26.5.0-1-x86_64 downloading...
|
||||
checking keyring...
|
||||
checking package integrity...
|
||||
:: File /var/cache/pacman/pkg/nodejs-26.5.0-1-x86_64.pkg.tar.zst is corrupted (invalid or corrupted package (PGP signature)).
|
||||
Errors occurred, no packages were upgraded.`
|
||||
|
||||
// A runner that behaves as ExecRunner does on failure: stdout as the output, stderr in the error.
|
||||
func pacmanFailing(stdout, stderr string) Runner {
|
||||
return func(_ context.Context, name string, args ...string) (string, error) {
|
||||
return stdout, errors.New(name + " exited 1: " + stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStaleDatabaseIsSaidAsOneWithItsAgeAndTheRemedy(t *testing.T) {
|
||||
was := syncDatabaseAge
|
||||
defer func() { syncDatabaseAge = was }()
|
||||
syncDatabaseAge = func() string {
|
||||
return describeAge(time.Date(2026, 7, 24, 16, 56, 0, 0, time.UTC), time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC))
|
||||
}
|
||||
|
||||
err := arch{}.InstallPackage(context.Background(), pacmanFailing(staleStdout, staleStderr), "nodejs")
|
||||
if err == nil {
|
||||
t.Fatal("a failed install must fail")
|
||||
}
|
||||
for _, want := range []string{
|
||||
"the package database on this machine is from 2026-07-24, 10 weeks old",
|
||||
"stale package index",
|
||||
"upgrading the machine — a full upgrade (`pacman -Syu`) by its operator — before the mesh can install nodejs",
|
||||
"partial upgrade",
|
||||
"returned error: 404", // pacman's own words follow
|
||||
} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the error does not say %q:\n%s", want, err)
|
||||
}
|
||||
}
|
||||
if strings.Contains(err.Error(), "mirrors or the network") {
|
||||
t.Errorf("a stale database must not be read as a mirror outage:\n%s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The same 404s read from stderr alone — the half this classifier used to be blind to.
|
||||
func TestTheClassifierReadsWhatPacmanWroteToStderr(t *testing.T) {
|
||||
if classifyInstallFailure(staleStderr) != installStale {
|
||||
t.Fatal("every mirror 404ing the named file is a stale database")
|
||||
}
|
||||
if classifyInstallFailure(staleStdout) != installStale {
|
||||
t.Fatal("a corrupted-signature line alone is a stale keyring")
|
||||
}
|
||||
if classifyInstallFailure("") != installOther {
|
||||
t.Fatal("nothing said is nothing classified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreachableMirrorWithAFreshDatabaseIsAMirrorProblem(t *testing.T) {
|
||||
was := syncDatabaseAge
|
||||
defer func() { syncDatabaseAge = was }()
|
||||
syncDatabaseAge = func() string { return "from 2026-10-02, less than a day old" }
|
||||
outage := `error: failed retrieving file 'core.db' from mirror.hetzner.com : Could not resolve host: mirror.hetzner.com
|
||||
error: failed retrieving file 'core.db' from mirror.rackspace.com : Connection timed out after 10001 milliseconds
|
||||
error: failed to synchronize all databases (failed to retrieve some files)`
|
||||
if classifyInstallFailure(outage) != installMirrors {
|
||||
t.Fatal("no mirror answering, no 404, no signature fault: the mirrors, not the machine")
|
||||
}
|
||||
err := arch{}.InstallPackage(context.Background(), pacmanFailing("", outage), "nodejs")
|
||||
if err == nil || !strings.Contains(err.Error(), "mirrors or the network") || !strings.Contains(err.Error(), "less than a day old") {
|
||||
t.Errorf("a mirror outage is said as one, with the database's age beside it:\n%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailureThatIsNeitherKeepsPacmansWords(t *testing.T) {
|
||||
err := arch{}.InstallPackage(context.Background(), pacmanFailing("", "error: target not found: nodejsx"), "nodejsx")
|
||||
if err == nil || !strings.Contains(err.Error(), "target not found") || strings.Contains(err.Error(), "package database on this machine") {
|
||||
t.Errorf("an unknown package is pacman's own error, not a stale database:\n%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDescribeAge(t *testing.T) {
|
||||
now := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
|
||||
for when, want := range map[time.Time]string{
|
||||
now.Add(-2 * time.Hour): "less than a day old",
|
||||
now.Add(-5 * 24 * time.Hour): "5 days old",
|
||||
now.Add(-71 * 24 * time.Hour): "10 weeks old",
|
||||
} {
|
||||
if got := describeAge(when, now); !strings.HasSuffix(got, want) {
|
||||
t.Errorf("%s: got %q, want suffix %q", when, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user