ADR 0059's recovery path: the pieces that run when the host will not start. internal/upgrade -- two facts, neither of them the host judging its health. Whether the executable this process started from has been replaced on disk, and which version last completed a reconcile. The first design was wrong and the tests caught it, not review. It asked /proc/self/exe whether it was marked deleted. That is Linux procfs behaviour rather than a fact about files, and it catches only unlink -- a binary swapped by rename onto the same path reads as untouched, which is exactly what a package manager does. Now the identity is captured at start and compared later: no procfs, and neither case missed. known-good is one bare line. The reader is a shell script on a machine where the host is failing to start, so it must not need a parser to be present and working. Written only after a clean apply, which is the whole claim -- not health, because a disconnected node is ordinary and a failing resource is the machine's problem rather than the binary's. packaging/ -- the unit, the rollback unit, and the rollback script. The script shares no code with the host and calls none of it: a binary that cannot start cannot be its own recovery. POSIX sh, nothing that has to be installed. The unit carries Restart=always with a comment saying why on-failure would break every upgrade. Both are tested and both sets of tests were confirmed to bite. Injecting five faults broke exactly the intended tests -- except one, and chasing why it did not found a placebo assertion I had written: `check "exits zero" ... "0" "0"` compares a literal to itself and can never fail. Replaced with the real exit code, after which the injection bites. Also caught: an injection that produced a build failure rather than a test failure, which my grep read as "no failure". Re-run so it compiled, and the test did bite. The script test runs in `make check`, so it is a gate rather than something that was run once. Verified against the real binary: known-good is written beside the store after a clean apply and is NOT written after a failed one.
139 lines
4.8 KiB
Go
139 lines
4.8 KiB
Go
// Package upgrade is how the host survives replacing itself.
|
|
//
|
|
// novox/hq ADR 0057 and ADR 0059. Two facts, and neither is the host judging its own health:
|
|
//
|
|
// - whether the executable this process started from has been replaced on disk, which is how
|
|
// it knows to stand aside for a new one;
|
|
// - which version last got as far as a completed reconcile, which is what a rollback outside
|
|
// this binary reads when this binary will not start.
|
|
//
|
|
// The second is written for a reader that is not the host. A binary that cannot start cannot be
|
|
// its own recovery, so what it leaves behind has to be plain enough for a shell script.
|
|
package upgrade
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
)
|
|
|
|
// KnownGoodName is the file a rollback script reads. Next to the store, because it is node
|
|
// state of exactly the same kind.
|
|
const KnownGoodName = "known-good"
|
|
|
|
// Self is the executable this process started from, remembered.
|
|
//
|
|
// Identity is taken once, at start, and compared later. The obvious alternative — asking
|
|
// /proc/self/exe whether it is marked deleted — was tried and is worse in two ways: it is Linux
|
|
// procfs behaviour rather than a fact about files, and it catches only *unlink*, so a binary
|
|
// swapped by rename onto the same path reads as untouched. Remembering what we started from
|
|
// needs no special filesystem and misses neither case.
|
|
type Self struct {
|
|
path string
|
|
info os.FileInfo
|
|
}
|
|
|
|
// Current captures the running executable's identity.
|
|
//
|
|
// path is what os.Executable() returned; a test passes one it can manipulate, because the
|
|
// boundary being tested is the filesystem and a fake would assert that the fake behaves as
|
|
// expected (novox/hq ADR 0034).
|
|
func Current(path string) (Self, error) {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return Self{}, fmt.Errorf(
|
|
"cannot stat %s, so this host cannot tell whether it is later replaced: %w", path, err)
|
|
}
|
|
return Self{path: path, info: info}, nil
|
|
}
|
|
|
|
// Path is where the executable was when this process started.
|
|
func (s Self) Path() string { return s.path }
|
|
|
|
// Replaced reports whether a different file is at that path now, or none.
|
|
//
|
|
// Never a silent false: a host that cannot read its own image says so rather than assuming it is
|
|
// current, which is the shape of every fault this repository catalogues.
|
|
func (s Self) Replaced() (bool, error) {
|
|
if s.info == nil {
|
|
return false, errors.New("this host never captured its own identity, so it cannot tell " +
|
|
"whether it has been replaced")
|
|
}
|
|
|
|
now, err := os.Stat(s.path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
// Removed rather than upgraded. Still not what is running, and saying "unchanged"
|
|
// would leave the host claiming a version that is no longer installed.
|
|
return true, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
return !os.SameFile(s.info, now), nil
|
|
}
|
|
|
|
// KnownGoodPath is where the marker lives, given where the store lives.
|
|
func KnownGoodPath(statePath string) string {
|
|
return filepath.Join(filepath.Dir(statePath), KnownGoodName)
|
|
}
|
|
|
|
// RecordKnownGood marks a version as one that started and completed a reconcile.
|
|
//
|
|
// Written atomically and as one bare line. The reader is a shell script running on a machine
|
|
// where the host is failing to start, so the format is the least it can be: no JSON, no
|
|
// escaping, nothing that needs a parser to be present and working.
|
|
func RecordKnownGood(path, version string) error {
|
|
if strings.TrimSpace(version) == "" {
|
|
return errors.New("refusing to record an empty version as known-good: a rollback " +
|
|
"reading it would install nothing and report success")
|
|
}
|
|
if strings.ContainsAny(version, "\n\r") {
|
|
return fmt.Errorf("refusing to record %q as known-good: it must be one line", version)
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return err
|
|
}
|
|
|
|
tmp, err := os.CreateTemp(filepath.Dir(path), ".known-good-*")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.Remove(tmp.Name())
|
|
|
|
if _, err := fmt.Fprintln(tmp, version); err != nil {
|
|
tmp.Close()
|
|
return err
|
|
}
|
|
if err := tmp.Sync(); err != nil {
|
|
tmp.Close()
|
|
return err
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp.Name(), path)
|
|
}
|
|
|
|
// ReadKnownGood returns the recorded version, or "" if there has never been one.
|
|
//
|
|
// Absence is not an error. A machine whose host has never completed a reconcile has no version
|
|
// to go back to, and that is a real state rather than a fault: the node was never working, so
|
|
// the failure belongs to the installation and not to an upgrade. A rollback that guessed here
|
|
// would become a second fault.
|
|
func ReadKnownGood(path string) (string, error) {
|
|
raw, err := os.ReadFile(path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return "", nil
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return strings.TrimSpace(string(raw)), nil
|
|
}
|