// Package upgrade is how the host survives replacing itself. // // novox/hq ADR 0005. Two facts, and neither is the host judging its own health: // // - whether the executable this process started from has been replaced on disk, which is how // it knows to stand aside for a new one; // - which version last got as far as a completed reconcile, which is what a rollback outside // this binary reads when this binary will not start. // // The second is written for a reader that is not the host. A binary that cannot start cannot be // its own recovery, so what it leaves behind has to be plain enough for a shell script. package upgrade import ( "errors" "fmt" "os" "path/filepath" "sort" "strings" "time" ) // Files the launcher reads and this binary writes. Next to the store, because they are node // state of exactly the same kind. const ( KnownGoodName = "known-good" AttemptsName = "start-attempts" ) // Self is the executable this process started from, remembered. // // Identity is taken once, at start, and compared later. The obvious alternative — asking // /proc/self/exe whether it is marked deleted — was tried and is worse in two ways: it is Linux // procfs behaviour rather than a fact about files, and it catches only *unlink*, so a binary // swapped by rename onto the same path reads as untouched. Remembering what we started from // needs no special filesystem and misses neither case. type Self struct { path string info os.FileInfo } // Current captures the running executable's identity. // // path is what os.Executable() returned; a test passes one it can manipulate, because the // boundary being tested is the filesystem and a fake would assert that the fake behaves as // expected (novox/hq ADR 0017). func Current(path string) (Self, error) { info, err := os.Stat(path) if err != nil { return Self{}, fmt.Errorf( "cannot stat %s, so this host cannot tell whether it is later replaced: %w", path, err) } return Self{path: path, info: info}, nil } // Path is where the executable was when this process started. func (s Self) Path() string { return s.path } // Replaced reports whether a different file is at that path now, or none. // // Never a silent false: a host that cannot read its own image says so rather than assuming it is // current, which is the shape of every fault this repository catalogues. func (s Self) Replaced() (bool, error) { if s.info == nil { return false, errors.New("this host never captured its own identity, so it cannot tell " + "whether it has been replaced") } now, err := os.Stat(s.path) if errors.Is(err, os.ErrNotExist) { // Removed rather than upgraded. Still not what is running, and saying "unchanged" // would leave the host claiming a version that is no longer installed. return true, nil } if err != nil { return false, err } return !os.SameFile(s.info, now), nil } // KnownGoodPath is where the marker lives, given where the store lives. func KnownGoodPath(statePath string) string { return filepath.Join(filepath.Dir(statePath), KnownGoodName) } // AttemptsPath is where the launcher counts starts that have not yet worked. func AttemptsPath(statePath string) string { return filepath.Join(filepath.Dir(statePath), AttemptsName) } // ClearAttempts tells the launcher this start worked. // // Written at the same moment as known-good and for the same reason: a completed reconcile is // the evidence, and it is the only evidence either of them has. Without this the counter only // ever climbs, so a node that has been up for months rolls itself back on its third ordinary // restart — a healthy machine undone by its own recovery. func ClearAttempts(path string) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } return os.WriteFile(path, []byte("0\n"), 0o644) } // RecordKnownGood marks a version as one that started and completed a reconcile. // // Written atomically and as one bare line. The reader is a shell script running on a machine // where the host is failing to start, so the format is the least it can be: no JSON, no // escaping, nothing that needs a parser to be present and working. func RecordKnownGood(path, version string) error { if strings.TrimSpace(version) == "" { return errors.New("refusing to record an empty version as known-good: a rollback " + "reading it would install nothing and report success") } if strings.ContainsAny(version, "\n\r") { return fmt.Errorf("refusing to record %q as known-good: it must be one line", version) } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return err } tmp, err := os.CreateTemp(filepath.Dir(path), ".known-good-*") if err != nil { return err } defer os.Remove(tmp.Name()) if _, err := fmt.Fprintln(tmp, version); err != nil { tmp.Close() return err } if err := tmp.Sync(); err != nil { tmp.Close() return err } if err := tmp.Close(); err != nil { return err } if err := os.Chmod(tmp.Name(), 0o644); err != nil { return err } return os.Rename(tmp.Name(), path) } // ReadKnownGood returns the recorded version, or "" if there has never been one. // // Absence is not an error. A machine whose host has never completed a reconcile has no version // to go back to, and that is a real state rather than a fault: the node was never working, so // the failure belongs to the installation and not to an upgrade. A rollback that guessed here // would become a second fault. func ReadKnownGood(path string) (string, error) { raw, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return "", nil } if err != nil { return "", err } return strings.TrimSpace(string(raw)), nil } // Where delivered versions live, and what the binary inside one is called. // // **A directory named for its version, never a link and never a write over what is running** // (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a // running executable, so the path a delivery writes must not be the path being executed; and a // rollback needs the previous version still present, which a single path cannot offer. // // The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in // the path, which is why nothing has to be told what is running. const ( // DefaultLibexec is where the host's own files live. Fixed rather than derived from where the // running executable sits: the first host to understand any of this was copied to a machine by // hand, and one that looked for its successor beside itself would never find a delivered version // — which is every machine in this mesh on the day this ships. DefaultLibexec = "/usr/lib/nox-mesh-host" VersionsDirName = "versions" BinaryName = "nox-mesh-host" // PinnedName is the version a rollback chose, which the launcher runs instead of the newest. // Without it the launcher would start the newest again and the rollback would flap. PinnedName = "rollback-pinned" ) // VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec // means the default, and the environment overrides it so a test needs no root. func VersionsDir(libexec string) string { if libexec == "" { libexec = os.Getenv("MESH_HOST_LIBEXEC") } if libexec == "" { libexec = DefaultLibexec } return filepath.Join(libexec, VersionsDirName) } // PinnedPath is where a rollback records the version it chose. func PinnedPath(statePath string) string { return filepath.Join(filepath.Dir(statePath), PinnedName) } // Delivered is one version present on the machine. type Delivered struct { // Version is the directory's name, which is the version. Version string // Binary is the executable inside it. Binary string // At is when it arrived, which is how "newest" is decided. At time.Time } // Versions are the versions delivered to this machine, newest first. // // **Newest by when it arrived, not by its name.** A version string comes from what the source was // tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either. // Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the // filesystem keeps and the delivery sets. // // A directory with no executable in it is not a version. A delivery that was interrupted leaves one, // and running the newest would then mean running nothing. func Versions(dir string) ([]Delivered, error) { entries, err := os.ReadDir(dir) if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err) } var out []Delivered for _, entry := range entries { if !entry.IsDir() { continue } binary := filepath.Join(dir, entry.Name(), BinaryName) info, err := os.Stat(binary) if err != nil || info.IsDir() { continue } at := info.ModTime() if d, err := entry.Info(); err == nil && d.ModTime().After(at) { at = d.ModTime() } out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at}) } // Newest first, and by name when two arrived in the same instant so the answer is never // arbitrary — a test that passes half the time is worse than one that fails. sort.Slice(out, func(a, b int) bool { if out[a].At.Equal(out[b].At) { return out[a].Version > out[b].Version } return out[a].At.After(out[b].At) }) return out, nil } // Successor is the version this machine should be running instead of the given one, if any. // // Empty when the running version is the newest, which is the ordinary answer. The host asks this // between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the // host exists to prevent (novox/hq ADR 0141). func Successor(dir, running string) (Delivered, bool, error) { delivered, err := Versions(dir) if err != nil { return Delivered{}, false, err } if len(delivered) == 0 { return Delivered{}, false, nil } newest := delivered[0] // A machine whose running version is not among the delivered ones is the machine every mesh has // one of: the host was put there by hand before any of this existed. Treating that as "stand // aside" is correct — what was delivered is what the mesh asked for. if newest.Version == running { return Delivered{}, false, nil } return newest, true, nil } // Retire removes delivered versions older than the running one's predecessor. // // The running version and the one before it are kept, and nothing else: the predecessor is exactly // what a rollback starts, and every version before that is weight with no reader. Called after a // reconcile completes, which is the same evidence known-good is written on — retiring on any weaker // signal would delete the thing a failing host is about to need. // // Never the running version, whatever it is asked. A host that deleted its own image would survive // until it stopped and then be unstartable, and the launcher's rollback reads a version, not a // process. func Retire(dir, running string) ([]string, error) { delivered, err := Versions(dir) if err != nil { return nil, err } keep := map[string]bool{running: true} for i, d := range delivered { if d.Version != running { continue } // Its predecessor is the next one down the list, which is the next oldest. if i+1 < len(delivered) { keep[delivered[i+1].Version] = true } break } // A running version that was never delivered has no predecessor among these, so the newest // delivered one is what a rollback would reach for. Keep it. if len(keep) == 1 && len(delivered) > 0 { keep[delivered[0].Version] = true } var removed []string for _, d := range delivered { if keep[d.Version] { continue } if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil { return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err) } removed = append(removed, d.Version) } sort.Strings(removed) return removed, nil }