mesh-host/internal/upgrade/upgrade.go

// Package upgrade is how the host survives replacing itself.
//
// novox/hq ADR 0005. Two facts, and neither is the host judging its own health:
//
//   - whether the executable this process started from has been replaced on disk, which is how
//     it knows to stand aside for a new one;
//   - which version last got as far as a completed reconcile, which is what a rollback outside
//     this binary reads when this binary will not start.
//
// The second is written for a reader that is not the host. A binary that cannot start cannot be
// its own recovery, so what it leaves behind has to be plain enough for a shell script.
package upgrade

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// Files the launcher reads and this binary writes. Next to the store, because they are node
// state of exactly the same kind.
const (
	KnownGoodName = "known-good"
	AttemptsName  = "start-attempts"
)

// Self is the executable this process started from, remembered.
//
// Identity is taken once, at start, and compared later. The obvious alternative — asking
// /proc/self/exe whether it is marked deleted — was tried and is worse in two ways: it is Linux
// procfs behaviour rather than a fact about files, and it catches only *unlink*, so a binary
// swapped by rename onto the same path reads as untouched. Remembering what we started from
// needs no special filesystem and misses neither case.
type Self struct {
	path string
	info os.FileInfo
}

// Current captures the running executable's identity.
//
// path is what os.Executable() returned; a test passes one it can manipulate, because the
// boundary being tested is the filesystem and a fake would assert that the fake behaves as
// expected (novox/hq ADR 0017).
func Current(path string) (Self, error) {
	info, err := os.Stat(path)
	if err != nil {
		return Self{}, fmt.Errorf(
			"cannot stat %s, so this host cannot tell whether it is later replaced: %w", path, err)
	}
	return Self{path: path, info: info}, nil
}

// Path is where the executable was when this process started.
func (s Self) Path() string { return s.path }

// Replaced reports whether a different file is at that path now, or none.
//
// Never a silent false: a host that cannot read its own image says so rather than assuming it is
// current, which is the shape of every fault this repository catalogues.
func (s Self) Replaced() (bool, error) {
	if s.info == nil {
		return false, errors.New("this host never captured its own identity, so it cannot tell " +
			"whether it has been replaced")
	}

	now, err := os.Stat(s.path)
	if errors.Is(err, os.ErrNotExist) {
		// Removed rather than upgraded. Still not what is running, and saying "unchanged"
		// would leave the host claiming a version that is no longer installed.
		return true, nil
	}
	if err != nil {
		return false, err
	}

	return !os.SameFile(s.info, now), nil
}

// KnownGoodPath is where the marker lives, given where the store lives.
func KnownGoodPath(statePath string) string {
	return filepath.Join(filepath.Dir(statePath), KnownGoodName)
}

// AttemptsPath is where the launcher counts starts that have not yet worked.
func AttemptsPath(statePath string) string {
	return filepath.Join(filepath.Dir(statePath), AttemptsName)
}

// ClearAttempts tells the launcher this start worked.
//
// Written at the same moment as known-good and for the same reason: a completed reconcile is
// the evidence, and it is the only evidence either of them has. Without this the counter only
// ever climbs, so a node that has been up for months rolls itself back on its third ordinary
// restart — a healthy machine undone by its own recovery.
func ClearAttempts(path string) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return err
	}
	return os.WriteFile(path, []byte("0\n"), 0o644)
}

// RecordKnownGood marks a version as one that started and completed a reconcile.
//
// Written atomically and as one bare line. The reader is a shell script running on a machine
// where the host is failing to start, so the format is the least it can be: no JSON, no
// escaping, nothing that needs a parser to be present and working.
func RecordKnownGood(path, version string) error {
	if strings.TrimSpace(version) == "" {
		return errors.New("refusing to record an empty version as known-good: a rollback " +
			"reading it would install nothing and report success")
	}
	if strings.ContainsAny(version, "\n\r") {
		return fmt.Errorf("refusing to record %q as known-good: it must be one line", version)
	}
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return err
	}

	tmp, err := os.CreateTemp(filepath.Dir(path), ".known-good-*")
	if err != nil {
		return err
	}
	defer os.Remove(tmp.Name())

	if _, err := fmt.Fprintln(tmp, version); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return err
	}
	if err := tmp.Close(); err != nil {
		return err
	}
	if err := os.Chmod(tmp.Name(), 0o644); err != nil {
		return err
	}
	return os.Rename(tmp.Name(), path)
}

// ReadKnownGood returns the recorded version, or "" if there has never been one.
//
// Absence is not an error. A machine whose host has never completed a reconcile has no version
// to go back to, and that is a real state rather than a fault: the node was never working, so
// the failure belongs to the installation and not to an upgrade. A rollback that guessed here
// would become a second fault.
func ReadKnownGood(path string) (string, error) {
	raw, err := os.ReadFile(path)
	if errors.Is(err, os.ErrNotExist) {
		return "", nil
	}
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(raw)), nil
}

// Where delivered versions live, and what the binary inside one is called.
//
// **A directory named for its version, never a link and never a write over what is running**
// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a
// running executable, so the path a delivery writes must not be the path being executed; and a
// rollback needs the previous version still present, which a single path cannot offer.
//
// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in
// the path, which is why nothing has to be told what is running.
const (
	// DefaultLibexec is where the host's own files live. Fixed rather than derived from where the
	// running executable sits: the first host to understand any of this was copied to a machine by
	// hand, and one that looked for its successor beside itself would never find a delivered version
	// — which is every machine in this mesh on the day this ships.
	DefaultLibexec  = "/usr/lib/nox-mesh-host"
	VersionsDirName = "versions"
	BinaryName      = "nox-mesh-host"
	// PinnedName is the version a rollback chose, which the launcher runs instead of the newest.
	// Without it the launcher would start the newest again and the rollback would flap.
	PinnedName = "rollback-pinned"
)

// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec
// means the default, and the environment overrides it so a test needs no root.
func VersionsDir(libexec string) string {
	if libexec == "" {
		libexec = os.Getenv("MESH_HOST_LIBEXEC")
	}
	if libexec == "" {
		libexec = DefaultLibexec
	}
	return filepath.Join(libexec, VersionsDirName)
}

// PinnedPath is where a rollback records the version it chose.
func PinnedPath(statePath string) string {
	return filepath.Join(filepath.Dir(statePath), PinnedName)
}

// Delivered is one version present on the machine.
type Delivered struct {
	// Version is the directory's name, which is the version.
	Version string
	// Binary is the executable inside it.
	Binary string
	// At is when it arrived, which is how "newest" is decided.
	At time.Time
}

// Versions are the versions delivered to this machine, newest first.
//
// **Newest by when it arrived, not by its name.** A version string comes from what the source was
// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either.
// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the
// filesystem keeps and the delivery sets.
//
// A directory with no executable in it is not a version. A delivery that was interrupted leaves one,
// and running the newest would then mean running nothing.
func Versions(dir string) ([]Delivered, error) {
	entries, err := os.ReadDir(dir)
	if errors.Is(err, os.ErrNotExist) {
		return nil, nil
	}
	if err != nil {
		return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err)
	}

	var out []Delivered
	for _, entry := range entries {
		if !entry.IsDir() {
			continue
		}
		binary := filepath.Join(dir, entry.Name(), BinaryName)
		info, err := os.Stat(binary)
		if err != nil || info.IsDir() {
			continue
		}
		at := info.ModTime()
		if d, err := entry.Info(); err == nil && d.ModTime().After(at) {
			at = d.ModTime()
		}
		out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at})
	}

	// Newest first, and by name when two arrived in the same instant so the answer is never
	// arbitrary — a test that passes half the time is worse than one that fails.
	sort.Slice(out, func(a, b int) bool {
		if out[a].At.Equal(out[b].At) {
			return out[a].Version > out[b].Version
		}
		return out[a].At.After(out[b].At)
	})
	return out, nil
}

// Successor is the version this machine should be running instead of the given one, if any.
//
// Empty when the running version is the newest, which is the ordinary answer. The host asks this
// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the
// host exists to prevent (novox/hq ADR 0141).
func Successor(dir, running string) (Delivered, bool, error) {
	delivered, err := Versions(dir)
	if err != nil {
		return Delivered{}, false, err
	}
	if len(delivered) == 0 {
		return Delivered{}, false, nil
	}
	newest := delivered[0]
	// A machine whose running version is not among the delivered ones is the machine every mesh has
	// one of: the host was put there by hand before any of this existed. Treating that as "stand
	// aside" is correct — what was delivered is what the mesh asked for.
	if newest.Version == running {
		return Delivered{}, false, nil
	}
	return newest, true, nil
}

// Retire removes delivered versions older than the running one's predecessor.
//
// The running version and the one before it are kept, and nothing else: the predecessor is exactly
// what a rollback starts, and every version before that is weight with no reader. Called after a
// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker
// signal would delete the thing a failing host is about to need.
//
// Never the running version, whatever it is asked. A host that deleted its own image would survive
// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a
// process.
func Retire(dir, running string) ([]string, error) {
	delivered, err := Versions(dir)
	if err != nil {
		return nil, err
	}

	keep := map[string]bool{running: true}
	for i, d := range delivered {
		if d.Version != running {
			continue
		}
		// Its predecessor is the next one down the list, which is the next oldest.
		if i+1 < len(delivered) {
			keep[delivered[i+1].Version] = true
		}
		break
	}
	// A running version that was never delivered has no predecessor among these, so the newest
	// delivered one is what a rollback would reach for. Keep it.
	if len(keep) == 1 && len(delivered) > 0 {
		keep[delivered[0].Version] = true
	}

	var removed []string
	for _, d := range delivered {
		if keep[d.Version] {
			continue
		}
		if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil {
			return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err)
		}
		removed = append(removed, d.Version)
	}
	sort.Strings(removed)
	return removed, nil
}