The host delivers its own successor, and versions live side by side
The supervision was already right: a clean exit means the host stood aside, and the launcher's next turn runs what is on disk. Two things made it dead code — nothing told the running host a successor was waiting, and the rollback resolved its known-good version through pacman, which no machine here uses and which two of three operating systems do not have. Keeping a version rather than a path was the clue. Versions now live in directories named for them: - the launcher picks the newest delivered one every time round the loop, or the one a rollback pinned, or the host placed by hand when nothing is delivered; - the running host stands aside between reconciles, never inside one, by exiting cleanly — and returns nil so the launcher does not count it as a crash; - a completed reconcile retires what is older than the predecessor, keeping the predecessor because that is what a rollback starts, and never the running one; - rollback pins the predecessor instead of reinstalling a package: no package manager, no cache anyone may clean, same script on every operating system; - the report says which host version produced it, so 'behind' is answerable. Newest is when it arrived, never how the name sorts: '1.10' orders before '1.9', and ordering by name would start an older host and call it an upgrade. novox/hq ADR 0141. The delivery half — a module carrying the next host — follows; until then nothing delivers a version and every machine takes the fallback, which is what it does today.
This commit is contained in:
@@ -110,6 +110,13 @@ type Report struct {
|
||||
// and the mesh's up in its place, and where the found configuration's original was kept.
|
||||
Tunnel *CarriedTunnel `json:"tunnel,omitempty"`
|
||||
|
||||
// Host is the version of the host that produced this report (novox/hq ADR 0141).
|
||||
//
|
||||
// Without it nothing can say a machine is behind, so "every machine current with its source"
|
||||
// could not include the host — the one component the mesh did not deliver. It is a fact the
|
||||
// machine states about itself, like the firewall it found and the links that face outside.
|
||||
Host string `json:"host,omitempty"`
|
||||
|
||||
// Outward is the links on this machine that face outside it — the ones carrying a default
|
||||
// route (novox/hq ADR 0140). Every node reports it, adopted or converged, because a converged
|
||||
// node's filter is written around it.
|
||||
|
||||
@@ -16,7 +16,9 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Files the launcher reads and this binary writes. Next to the store, because they are node
|
||||
@@ -157,3 +159,168 @@ func ReadKnownGood(path string) (string, error) {
|
||||
}
|
||||
return strings.TrimSpace(string(raw)), nil
|
||||
}
|
||||
|
||||
// Where delivered versions live, and what the binary inside one is called.
|
||||
//
|
||||
// **A directory named for its version, never a link and never a write over what is running**
|
||||
// (novox/hq ADR 0141). Two facts follow from that one choice: the kernel refuses to truncate a
|
||||
// running executable, so the path a delivery writes must not be the path being executed; and a
|
||||
// rollback needs the previous version still present, which a single path cannot offer.
|
||||
//
|
||||
// The mesh creates no links (novox/hq ADR 0012), so nothing points at "current". The version is in
|
||||
// the path, which is why nothing has to be told what is running.
|
||||
const (
|
||||
// DefaultLibexec is where the host's own files live. Fixed rather than derived from where the
|
||||
// running executable sits: the first host to understand any of this was copied to a machine by
|
||||
// hand, and one that looked for its successor beside itself would never find a delivered version
|
||||
// — which is every machine in this mesh on the day this ships.
|
||||
DefaultLibexec = "/usr/lib/nox-mesh-host"
|
||||
VersionsDirName = "versions"
|
||||
BinaryName = "nox-mesh-host"
|
||||
// PinnedName is the version a rollback chose, which the launcher runs instead of the newest.
|
||||
// Without it the launcher would start the newest again and the rollback would flap.
|
||||
PinnedName = "rollback-pinned"
|
||||
)
|
||||
|
||||
// VersionsDir is where delivered versions live, given where the host's libexec is. An empty libexec
|
||||
// means the default, and the environment overrides it so a test needs no root.
|
||||
func VersionsDir(libexec string) string {
|
||||
if libexec == "" {
|
||||
libexec = os.Getenv("MESH_HOST_LIBEXEC")
|
||||
}
|
||||
if libexec == "" {
|
||||
libexec = DefaultLibexec
|
||||
}
|
||||
return filepath.Join(libexec, VersionsDirName)
|
||||
}
|
||||
|
||||
// PinnedPath is where a rollback records the version it chose.
|
||||
func PinnedPath(statePath string) string {
|
||||
return filepath.Join(filepath.Dir(statePath), PinnedName)
|
||||
}
|
||||
|
||||
// Delivered is one version present on the machine.
|
||||
type Delivered struct {
|
||||
// Version is the directory's name, which is the version.
|
||||
Version string
|
||||
// Binary is the executable inside it.
|
||||
Binary string
|
||||
// At is when it arrived, which is how "newest" is decided.
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// Versions are the versions delivered to this machine, newest first.
|
||||
//
|
||||
// **Newest by when it arrived, not by its name.** A version string comes from what the source was
|
||||
// tagged or described as, and those do not sort: "1.10" before "1.9", a commit hash before either.
|
||||
// Ordering by name would run an older host and call it an upgrade. When it arrived is a fact the
|
||||
// filesystem keeps and the delivery sets.
|
||||
//
|
||||
// A directory with no executable in it is not a version. A delivery that was interrupted leaves one,
|
||||
// and running the newest would then mean running nothing.
|
||||
func Versions(dir string) ([]Delivered, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read the delivered versions at %s: %w", dir, err)
|
||||
}
|
||||
|
||||
var out []Delivered
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
binary := filepath.Join(dir, entry.Name(), BinaryName)
|
||||
info, err := os.Stat(binary)
|
||||
if err != nil || info.IsDir() {
|
||||
continue
|
||||
}
|
||||
at := info.ModTime()
|
||||
if d, err := entry.Info(); err == nil && d.ModTime().After(at) {
|
||||
at = d.ModTime()
|
||||
}
|
||||
out = append(out, Delivered{Version: entry.Name(), Binary: binary, At: at})
|
||||
}
|
||||
|
||||
// Newest first, and by name when two arrived in the same instant so the answer is never
|
||||
// arbitrary — a test that passes half the time is worse than one that fails.
|
||||
sort.Slice(out, func(a, b int) bool {
|
||||
if out[a].At.Equal(out[b].At) {
|
||||
return out[a].Version > out[b].Version
|
||||
}
|
||||
return out[a].At.After(out[b].At)
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Successor is the version this machine should be running instead of the given one, if any.
|
||||
//
|
||||
// Empty when the running version is the newest, which is the ordinary answer. The host asks this
|
||||
// between reconciles and nowhere else: standing aside mid-apply is the half-configured machine the
|
||||
// host exists to prevent (novox/hq ADR 0141).
|
||||
func Successor(dir, running string) (Delivered, bool, error) {
|
||||
delivered, err := Versions(dir)
|
||||
if err != nil {
|
||||
return Delivered{}, false, err
|
||||
}
|
||||
if len(delivered) == 0 {
|
||||
return Delivered{}, false, nil
|
||||
}
|
||||
newest := delivered[0]
|
||||
// A machine whose running version is not among the delivered ones is the machine every mesh has
|
||||
// one of: the host was put there by hand before any of this existed. Treating that as "stand
|
||||
// aside" is correct — what was delivered is what the mesh asked for.
|
||||
if newest.Version == running {
|
||||
return Delivered{}, false, nil
|
||||
}
|
||||
return newest, true, nil
|
||||
}
|
||||
|
||||
// Retire removes delivered versions older than the running one's predecessor.
|
||||
//
|
||||
// The running version and the one before it are kept, and nothing else: the predecessor is exactly
|
||||
// what a rollback starts, and every version before that is weight with no reader. Called after a
|
||||
// reconcile completes, which is the same evidence known-good is written on — retiring on any weaker
|
||||
// signal would delete the thing a failing host is about to need.
|
||||
//
|
||||
// Never the running version, whatever it is asked. A host that deleted its own image would survive
|
||||
// until it stopped and then be unstartable, and the launcher's rollback reads a version, not a
|
||||
// process.
|
||||
func Retire(dir, running string) ([]string, error) {
|
||||
delivered, err := Versions(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keep := map[string]bool{running: true}
|
||||
for i, d := range delivered {
|
||||
if d.Version != running {
|
||||
continue
|
||||
}
|
||||
// Its predecessor is the next one down the list, which is the next oldest.
|
||||
if i+1 < len(delivered) {
|
||||
keep[delivered[i+1].Version] = true
|
||||
}
|
||||
break
|
||||
}
|
||||
// A running version that was never delivered has no predecessor among these, so the newest
|
||||
// delivered one is what a rollback would reach for. Keep it.
|
||||
if len(keep) == 1 && len(delivered) > 0 {
|
||||
keep[delivered[0].Version] = true
|
||||
}
|
||||
|
||||
var removed []string
|
||||
for _, d := range delivered {
|
||||
if keep[d.Version] {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(dir, d.Version)); err != nil {
|
||||
return removed, fmt.Errorf("cannot retire the host version %s: %w", d.Version, err)
|
||||
}
|
||||
removed = append(removed, d.Version)
|
||||
}
|
||||
sort.Strings(removed)
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package upgrade
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// deliver writes a version as a delivery would: a directory named for it with the binary inside.
|
||||
// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock.
|
||||
func deliver(t *testing.T, dir, version string, at time.Time) string {
|
||||
t.Helper()
|
||||
into := filepath.Join(dir, version)
|
||||
if err := os.MkdirAll(into, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
binary := filepath.Join(into, BinaryName)
|
||||
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(binary, at, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(into, at, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return binary
|
||||
}
|
||||
|
||||
// **Newest is when it arrived, not how its name sorts.**
|
||||
//
|
||||
// A version string is whatever the source was tagged or described as, and those do not sort: "1.10"
|
||||
// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older
|
||||
// host and call that an upgrade.
|
||||
func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := time.Now().Add(-time.Hour)
|
||||
deliver(t, dir, "1.10", base) // sorts LAST by name, arrived first
|
||||
deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last
|
||||
|
||||
got, err := Versions(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 || got[0].Version != "1.9" {
|
||||
t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest"
|
||||
// would then mean running nothing, so it is not a version.
|
||||
func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deliver(t, dir, "good", time.Now().Add(-time.Hour))
|
||||
|
||||
got, err := Versions(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Version != "good" {
|
||||
t.Fatalf("versions are %+v, want only the one with a binary", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions
|
||||
// directory at all, and that must read as "no successor" rather than as an error that stops a
|
||||
// reconcile.
|
||||
func TestNoVersionsDirectoryIsNotAnError(t *testing.T) {
|
||||
got, err := Versions(filepath.Join(t.TempDir(), "absent"))
|
||||
if err != nil {
|
||||
t.Fatalf("an absent versions directory should not be an error: %v", err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("versions are %+v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := time.Now().Add(-time.Hour)
|
||||
deliver(t, dir, "one", base)
|
||||
deliver(t, dir, "two", base.Add(time.Minute))
|
||||
|
||||
next, yes, err := Successor(dir, "one")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !yes || next.Version != "two" {
|
||||
t.Fatalf("successor is %+v (%v), want two", next, yes)
|
||||
}
|
||||
|
||||
if _, yes, err := Successor(dir, "two"); err != nil || yes {
|
||||
t.Fatalf("the newest version is its own successor (%v, %v)", yes, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A host put there by hand, before any of this existed, is not among the delivered versions. What the
|
||||
// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a
|
||||
// machine would be ignored for ever, which is every machine in this mesh today.
|
||||
func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
deliver(t, dir, "delivered", time.Now().Add(-time.Hour))
|
||||
|
||||
next, yes, err := Successor(dir, "copied-by-hand")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !yes || next.Version != "delivered" {
|
||||
t.Fatalf("successor is %+v (%v), want the delivered one", next, yes)
|
||||
}
|
||||
}
|
||||
|
||||
// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what
|
||||
// a rollback starts; everything older has no reader.
|
||||
func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := time.Now().Add(-4 * time.Hour)
|
||||
for i, v := range []string{"one", "two", "three", "four"} {
|
||||
deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour))
|
||||
}
|
||||
|
||||
removed, err := Retire(dir, "four")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sort.Strings(removed)
|
||||
if !reflect.DeepEqual(removed, []string{"one", "two"}) {
|
||||
t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed)
|
||||
}
|
||||
for _, kept := range []string{"three", "four"} {
|
||||
if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil {
|
||||
t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Never the running version, whatever it is asked.** A host that deleted its own image would run
|
||||
// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a
|
||||
// process.
|
||||
func TestRetireNeverRemovesTheRunningVersion(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := time.Now().Add(-2 * time.Hour)
|
||||
deliver(t, dir, "older", base)
|
||||
deliver(t, dir, "newer", base.Add(time.Hour))
|
||||
|
||||
// Asked while running the OLDER one, which is what a machine looks like between a delivery and
|
||||
// the moment it stands aside.
|
||||
if _, err := Retire(dir, "older"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil {
|
||||
t.Fatalf("the running version was retired: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine running a hand-placed host keeps the newest delivered version, because that is what a
|
||||
// rollback would reach for. Retiring it would leave the machine with no way back at all.
|
||||
func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := time.Now().Add(-3 * time.Hour)
|
||||
deliver(t, dir, "old", base)
|
||||
deliver(t, dir, "new", base.Add(time.Hour))
|
||||
|
||||
removed, err := Retire(dir, "copied-by-hand")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(removed, []string{"old"}) {
|
||||
t.Fatalf("retired %v, want only old — new is the rollback target", removed)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil {
|
||||
t.Fatalf("the only delivered version was retired: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user