jochen
0c405b70cc
The launcher trusted a counter only a by-hand reconcile ever cleared and a known-good nothing in the daemon wrote, so no machine could roll its host back; the controller and the node tools were replaced in place with nothing kept. - The launcher runs a delivered host that is not known-good on trial: one that crashes, stops for nothing, or does not report within ten minutes goes back to known-good, once per version, recorded in rolled-back. The host proves itself when the mesh takes a report under its own build, says every standing verdict on its reports, never stands aside for a rolled-back version, and restarts its service once when its launcher was replaced on disk. - The engine keeps the controller's and the node tools' previous build beside the new one and judges the new one: the lease taken by the controller it started (read-only direct get of mesh-controller_lease/holder), or this machine's runtime answering $SRV.PING.node-tools.<node>, within sixty seconds of time it could ask. Not healthy: the previous restored, once, said. Proved: the previous deleted. A build declared not-reversible is never rolled back. - Retire never removes a version newer than the running one.
347 lines
14 KiB
Go
347 lines
14 KiB
Go
package witness
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/link"
|
|
)
|
|
|
|
// Each rollback path, induced against a real directory: a controller that never takes the lease, a
|
|
// runtime that never answers PING, a build declared not reversible; and a healthy update, which
|
|
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).
|
|
|
|
const (
|
|
buildA = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
buildB = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
|
buildC = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
|
|
)
|
|
|
|
// place does what the applier does with a declared build: ask Place, unpack when told, commit.
|
|
func place(t *testing.T, root, name, by, build, recorded, notReversible string, at time.Time) Placing {
|
|
t.Helper()
|
|
p, err := Place(root, name, by, build, recorded, notReversible, at)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p.Unpack {
|
|
dir := filepath.Join(root, name)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "build"), []byte(build), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := p.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func running(t *testing.T, root, name string) string {
|
|
t.Helper()
|
|
raw, err := os.ReadFile(filepath.Join(root, name, "build"))
|
|
if err != nil {
|
|
return "nothing"
|
|
}
|
|
return string(raw)
|
|
}
|
|
|
|
func kept(root, name string) bool {
|
|
_, err := os.Stat(previousDir(root, name))
|
|
return err == nil
|
|
}
|
|
|
|
// asker answers as the build it is told is running would.
|
|
type asker struct {
|
|
mu sync.Mutex
|
|
lease func() ([]byte, bool, error)
|
|
ping func() error
|
|
asked int
|
|
}
|
|
|
|
func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
a.asked++
|
|
return a.lease()
|
|
}
|
|
|
|
func (a *asker) Ping(context.Context, string, string) error {
|
|
a.mu.Lock()
|
|
defer a.mu.Unlock()
|
|
a.asked++
|
|
return a.ping()
|
|
}
|
|
|
|
type machine struct {
|
|
commands []string
|
|
failRestart bool
|
|
}
|
|
|
|
func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
|
|
m.commands = append(m.commands, name+" "+strings.Join(args, " "))
|
|
if m.failRestart && len(args) > 0 && args[0] == "restart" {
|
|
return "", fmt.Errorf("exit status 1")
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
|
|
return &Watcher{Root: root, Node: "anchor", Host: "anchor",
|
|
Asker: func() link.Asker {
|
|
if a == nil {
|
|
return nil
|
|
}
|
|
return a
|
|
},
|
|
Run: m.run, Now: func() time.Time { return *clock },
|
|
Concluded: func(v link.Rollback) { *verdicts = append(*verdicts, v) },
|
|
Every: Every,
|
|
}
|
|
}
|
|
|
|
// look runs the watcher n times, the clock moving Every each time.
|
|
func look(w *Watcher, clock *time.Time, n int) {
|
|
for i := 0; i < n; i++ {
|
|
*clock = clock.Add(Every)
|
|
w.Look(context.Background())
|
|
}
|
|
}
|
|
|
|
func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) {
|
|
return func() ([]byte, bool, error) {
|
|
return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`,
|
|
instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil
|
|
}
|
|
}
|
|
|
|
// A controller that starts and never takes the lease: restored to the build before, once, said once.
|
|
func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
|
|
root := t.TempDir()
|
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
|
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
|
if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") {
|
|
t.Fatal("the new controller is not running with the build before it kept beside it")
|
|
}
|
|
|
|
// The old instance's lease, taken an hour ago, is all the bus ever shows.
|
|
a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)}
|
|
m := &machine{}
|
|
var verdicts []link.Rollback
|
|
w := watcher(root, a, m, &clock, &verdicts)
|
|
|
|
look(w, &clock, int(ControllerWithin/Every)-1)
|
|
if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB {
|
|
t.Fatalf("rolled back before its bound: %+v", verdicts)
|
|
}
|
|
look(w, &clock, 1)
|
|
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
|
|
verdicts[0].Component != link.ComponentController {
|
|
t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts)
|
|
}
|
|
if running(t, root, "mesh-controller") != buildA {
|
|
t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller"))
|
|
}
|
|
if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" {
|
|
t.Fatalf("the machine was asked %v", m.commands)
|
|
}
|
|
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") {
|
|
t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
|
|
}
|
|
|
|
// Once: the restored build is not judged again, and nothing more is said or done.
|
|
look(w, &clock, 30)
|
|
if len(verdicts) != 1 || len(m.commands) != 2 {
|
|
t.Fatalf("judged again after a rollback: %d verdicts, %v", len(verdicts), m.commands)
|
|
}
|
|
// Standing, so every report says it — until the mesh asks for another build.
|
|
if s := Standing(root); len(s) != 1 || s[0].From != buildB {
|
|
t.Fatalf("what every report says is %+v", s)
|
|
}
|
|
// The mesh still declares B: refused, A kept running, the verdict still standing.
|
|
p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock)
|
|
if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") {
|
|
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller"))
|
|
}
|
|
if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
|
|
t.Fatal("refusing the rolled-back build changed what stands or started a trial")
|
|
}
|
|
// A newer build is a new trial, and ends what was concluded once it is proved.
|
|
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
|
|
if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 {
|
|
t.Fatal("a newer build after a rollback was not placed on trial")
|
|
}
|
|
if err := Proved(root, "mesh-controller"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(Standing(root)) != 0 || kept(root, "mesh-controller") {
|
|
t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
|
|
}
|
|
}
|
|
|
|
// A runtime that never answers PING: restored, once.
|
|
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
|
|
root := t.TempDir()
|
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
|
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
|
|
|
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
|
|
m := &machine{}
|
|
var verdicts []link.Rollback
|
|
w := watcher(root, a, m, &clock, &verdicts)
|
|
look(w, &clock, int(NodeToolsWithin/Every)+20)
|
|
|
|
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools {
|
|
t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts)
|
|
}
|
|
if running(t, root, "node-tools") != buildA || kept(root, "node-tools") {
|
|
t.Fatal("the runtime running is not the build before, or a copy of it is still kept")
|
|
}
|
|
}
|
|
|
|
// A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
|
|
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
|
|
root := t.TempDir()
|
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
// The controller placed by a host from before any witness: a record, and no state.
|
|
if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
|
started := clock
|
|
|
|
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
|
|
m := &machine{}
|
|
var verdicts []link.Rollback
|
|
w := watcher(root, a, m, &clock, &verdicts)
|
|
look(w, &clock, 5)
|
|
if !kept(root, "mesh-controller") {
|
|
t.Fatal("the build before was deleted while the new one was still on trial")
|
|
}
|
|
// A third build while the second is on trial: the one kept is still the one seen healthy.
|
|
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
|
|
if s, _ := Load(root, "mesh-controller"); s.Previous != buildA {
|
|
t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
|
|
}
|
|
started = clock
|
|
a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock)
|
|
look(w, &clock, 2)
|
|
if len(verdicts) != 0 || len(m.commands) != 0 {
|
|
t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
|
|
}
|
|
if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC {
|
|
t.Fatal("the build before was not deleted once the new one was proved")
|
|
}
|
|
}
|
|
|
|
// A build declared not reversible is never rolled back: the build before never starts against what it
|
|
// changed, and the verdict is urgent and stands.
|
|
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
|
root := t.TempDir()
|
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
|
place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock)
|
|
|
|
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
|
|
m := &machine{}
|
|
var verdicts []link.Rollback
|
|
w := watcher(root, a, m, &clock, &verdicts)
|
|
look(w, &clock, int(ControllerWithin/Every)+20)
|
|
|
|
if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
|
|
t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
|
|
}
|
|
if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
|
|
t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
|
|
}
|
|
if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
|
|
t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
|
|
running(t, root, "mesh-controller"))
|
|
}
|
|
if len(Standing(root)) != 1 {
|
|
t.Fatal("the not-reversible verdict does not stand")
|
|
}
|
|
}
|
|
|
|
// A witness that cannot ask counts nothing against the build, and says so at GiveUp.
|
|
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
|
|
root := t.TempDir()
|
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
|
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
|
|
|
a := &asker{lease: func() ([]byte, bool, error) {
|
|
return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk)
|
|
}}
|
|
m := &machine{}
|
|
var verdicts []link.Rollback
|
|
w := watcher(root, a, m, &clock, &verdicts)
|
|
look(w, &clock, int(GiveUp/Every)-1)
|
|
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
|
|
t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
|
|
}
|
|
look(w, &clock, 1)
|
|
if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
|
|
t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
|
|
}
|
|
if !kept(root, "mesh-controller") {
|
|
t.Fatal("the build before was deleted though the new one was never seen healthy")
|
|
}
|
|
|
|
// And a host with no link at all is the same: nothing counted.
|
|
root2 := t.TempDir()
|
|
place(t, root2, "node-tools", ByPing, buildA, "", "", clock)
|
|
place(t, root2, "node-tools", ByPing, buildB, buildA, "", clock)
|
|
var none []link.Rollback
|
|
w2 := watcher(root2, nil, m, &clock, &none)
|
|
look(w2, &clock, int(NodeToolsWithin/Every)*3)
|
|
if len(none) != 0 || running(t, root2, "node-tools") != buildB {
|
|
t.Fatal("a build was rolled back while this host had no link to ask with")
|
|
}
|
|
}
|
|
|
|
// A restoration whose build will not start is said as such — not as a rollback that worked.
|
|
func TestARestorationThatDoesNotStartIsSaid(t *testing.T) {
|
|
root := t.TempDir()
|
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
|
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
|
v, err := Restore(context.Background(), root, "node-tools", "it never answered", (&machine{failRestart: true}).run, clock)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v.Outcome != link.RestoreFailed || running(t, root, "node-tools") != buildA {
|
|
t.Fatalf("the verdict is %+v, running %s", v, running(t, root, "node-tools"))
|
|
}
|
|
}
|
|
|
|
// What the engine keeps survives the engine: a host that stands aside mid-trial resumes it.
|
|
func TestATrialSurvivesTheHostRestarting(t *testing.T) {
|
|
root := t.TempDir()
|
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
|
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
|
a := &asker{ping: func() error { return link.ErrNoAnswer }}
|
|
var verdicts []link.Rollback
|
|
half := int(NodeToolsWithin/Every) / 2
|
|
look(watcher(root, a, &machine{}, &clock, &verdicts), &clock, half)
|
|
// A new watcher, as a successor host would start.
|
|
look(watcher(root, a, &machine{}, &clock, &verdicts), &clock, int(NodeToolsWithin/Every)-half)
|
|
if len(verdicts) != 1 {
|
|
t.Fatalf("the bound was not carried across the host restarting: %+v", verdicts)
|
|
}
|
|
}
|