The supervision was already right: a clean exit means the host stood aside, and the launcher's next turn runs what is on disk. Two things made it dead code — nothing told the running host a successor was waiting, and the rollback resolved its known-good version through pacman, which no machine here uses and which two of three operating systems do not have. Keeping a version rather than a path was the clue. Versions now live in directories named for them: - the launcher picks the newest delivered one every time round the loop, or the one a rollback pinned, or the host placed by hand when nothing is delivered; - the running host stands aside between reconciles, never inside one, by exiting cleanly — and returns nil so the launcher does not count it as a crash; - a completed reconcile retires what is older than the predecessor, keeping the predecessor because that is what a rollback starts, and never the running one; - rollback pins the predecessor instead of reinstalling a package: no package manager, no cache anyone may clean, same script on every operating system; - the report says which host version produced it, so 'behind' is answerable. Newest is when it arrived, never how the name sorts: '1.10' orders before '1.9', and ordering by name would start an older host and call it an upgrade. novox/hq ADR 0141. The delivery half — a module carrying the next host — follows; until then nothing delivers a version and every machine takes the fallback, which is what it does today.
181 lines
6.0 KiB
Go
181 lines
6.0 KiB
Go
package upgrade
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// deliver writes a version as a delivery would: a directory named for it with the binary inside.
|
|
// at fixes when it arrived, because "newest" is when it arrived and a test must not race the clock.
|
|
func deliver(t *testing.T, dir, version string, at time.Time) string {
|
|
t.Helper()
|
|
into := filepath.Join(dir, version)
|
|
if err := os.MkdirAll(into, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
binary := filepath.Join(into, BinaryName)
|
|
if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Chtimes(binary, at, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Chtimes(into, at, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return binary
|
|
}
|
|
|
|
// **Newest is when it arrived, not how its name sorts.**
|
|
//
|
|
// A version string is whatever the source was tagged or described as, and those do not sort: "1.10"
|
|
// orders before "1.9", and a commit hash orders before either. Ordering by name would start an older
|
|
// host and call that an upgrade.
|
|
func TestNewestIsWhenItArrivedAndNotHowItSorts(t *testing.T) {
|
|
dir := t.TempDir()
|
|
base := time.Now().Add(-time.Hour)
|
|
deliver(t, dir, "1.10", base) // sorts LAST by name, arrived first
|
|
deliver(t, dir, "1.9", base.Add(time.Minute)) // sorts first by name, arrived last
|
|
|
|
got, err := Versions(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 2 || got[0].Version != "1.9" {
|
|
t.Fatalf("newest is %+v, want the one that arrived last (1.9)", got)
|
|
}
|
|
}
|
|
|
|
// A delivery that was interrupted leaves a directory with no executable in it. Running "the newest"
|
|
// would then mean running nothing, so it is not a version.
|
|
func TestADirectoryWithNoBinaryIsNotAVersion(t *testing.T) {
|
|
dir := t.TempDir()
|
|
if err := os.MkdirAll(filepath.Join(dir, "half-delivered"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
deliver(t, dir, "good", time.Now().Add(-time.Hour))
|
|
|
|
got, err := Versions(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 1 || got[0].Version != "good" {
|
|
t.Fatalf("versions are %+v, want only the one with a binary", got)
|
|
}
|
|
}
|
|
|
|
// Nothing delivered is not a fault. A machine whose host was placed by hand has no versions
|
|
// directory at all, and that must read as "no successor" rather than as an error that stops a
|
|
// reconcile.
|
|
func TestNoVersionsDirectoryIsNotAnError(t *testing.T) {
|
|
got, err := Versions(filepath.Join(t.TempDir(), "absent"))
|
|
if err != nil {
|
|
t.Fatalf("an absent versions directory should not be an error: %v", err)
|
|
}
|
|
if len(got) != 0 {
|
|
t.Fatalf("versions are %+v, want none", got)
|
|
}
|
|
}
|
|
|
|
func TestTheNewestVersionIsTheSuccessorAndTheRunningOneIsNot(t *testing.T) {
|
|
dir := t.TempDir()
|
|
base := time.Now().Add(-time.Hour)
|
|
deliver(t, dir, "one", base)
|
|
deliver(t, dir, "two", base.Add(time.Minute))
|
|
|
|
next, yes, err := Successor(dir, "one")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !yes || next.Version != "two" {
|
|
t.Fatalf("successor is %+v (%v), want two", next, yes)
|
|
}
|
|
|
|
if _, yes, err := Successor(dir, "two"); err != nil || yes {
|
|
t.Fatalf("the newest version is its own successor (%v, %v)", yes, err)
|
|
}
|
|
}
|
|
|
|
// A host put there by hand, before any of this existed, is not among the delivered versions. What the
|
|
// mesh delivered is what it asked for, so that is a successor — otherwise the first delivery to such a
|
|
// machine would be ignored for ever, which is every machine in this mesh today.
|
|
func TestAHostThatWasNeverDeliveredHasASuccessor(t *testing.T) {
|
|
dir := t.TempDir()
|
|
deliver(t, dir, "delivered", time.Now().Add(-time.Hour))
|
|
|
|
next, yes, err := Successor(dir, "copied-by-hand")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !yes || next.Version != "delivered" {
|
|
t.Fatalf("successor is %+v (%v), want the delivered one", next, yes)
|
|
}
|
|
}
|
|
|
|
// The running version and its predecessor are kept, and nothing else. The predecessor is exactly what
|
|
// a rollback starts; everything older has no reader.
|
|
func TestRetireKeepsTheRunningVersionAndItsPredecessor(t *testing.T) {
|
|
dir := t.TempDir()
|
|
base := time.Now().Add(-4 * time.Hour)
|
|
for i, v := range []string{"one", "two", "three", "four"} {
|
|
deliver(t, dir, v, base.Add(time.Duration(i)*time.Hour))
|
|
}
|
|
|
|
removed, err := Retire(dir, "four")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sort.Strings(removed)
|
|
if !reflect.DeepEqual(removed, []string{"one", "two"}) {
|
|
t.Fatalf("retired %v, want one and two — three is the predecessor a rollback needs", removed)
|
|
}
|
|
for _, kept := range []string{"three", "four"} {
|
|
if _, err := os.Stat(filepath.Join(dir, kept, BinaryName)); err != nil {
|
|
t.Fatalf("%s was retired and a rollback now has nowhere to go: %v", kept, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **Never the running version, whatever it is asked.** A host that deleted its own image would run
|
|
// until it stopped and then be unstartable, and the launcher's rollback reads a version rather than a
|
|
// process.
|
|
func TestRetireNeverRemovesTheRunningVersion(t *testing.T) {
|
|
dir := t.TempDir()
|
|
base := time.Now().Add(-2 * time.Hour)
|
|
deliver(t, dir, "older", base)
|
|
deliver(t, dir, "newer", base.Add(time.Hour))
|
|
|
|
// Asked while running the OLDER one, which is what a machine looks like between a delivery and
|
|
// the moment it stands aside.
|
|
if _, err := Retire(dir, "older"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "older", BinaryName)); err != nil {
|
|
t.Fatalf("the running version was retired: %v", err)
|
|
}
|
|
}
|
|
|
|
// A machine running a hand-placed host keeps the newest delivered version, because that is what a
|
|
// rollback would reach for. Retiring it would leave the machine with no way back at all.
|
|
func TestRetireKeepsTheNewestWhenTheRunningVersionWasNeverDelivered(t *testing.T) {
|
|
dir := t.TempDir()
|
|
base := time.Now().Add(-3 * time.Hour)
|
|
deliver(t, dir, "old", base)
|
|
deliver(t, dir, "new", base.Add(time.Hour))
|
|
|
|
removed, err := Retire(dir, "copied-by-hand")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(removed, []string{"old"}) {
|
|
t.Fatalf("retired %v, want only old — new is the rollback target", removed)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "new", BinaryName)); err != nil {
|
|
t.Fatalf("the only delivered version was retired: %v", err)
|
|
}
|
|
}
|