package witness import ( "context" "fmt" "os" "path/filepath" "strings" "sync" "testing" "time" "github.com/novox/mesh-host/internal/link" ) // Each rollback path, induced against a real directory: a controller that never takes the lease, a // runtime that never answers PING, a build declared not reversible; and a healthy update, which // deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 ยง8). const ( buildA = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" buildB = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" buildC = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" ) // place does what the applier does with a declared build: ask Place, unpack when told, commit. func place(t *testing.T, root, name, by, build, recorded, notReversible string, at time.Time) Placing { t.Helper() p, err := Place(root, name, by, build, recorded, notReversible, at) if err != nil { t.Fatal(err) } if p.Unpack { dir := filepath.Join(root, name) if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "build"), []byte(build), 0o644); err != nil { t.Fatal(err) } } if err := p.Commit(); err != nil { t.Fatal(err) } return p } func running(t *testing.T, root, name string) string { t.Helper() raw, err := os.ReadFile(filepath.Join(root, name, "build")) if err != nil { return "nothing" } return string(raw) } func kept(root, name string) bool { _, err := os.Stat(previousDir(root, name)) return err == nil } // asker answers as the build it is told is running would. type asker struct { mu sync.Mutex lease func() ([]byte, bool, error) ping func() error asked int } func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) { a.mu.Lock() defer a.mu.Unlock() a.asked++ return a.lease() } func (a *asker) Ping(context.Context, string, string) error { a.mu.Lock() defer a.mu.Unlock() a.asked++ return a.ping() } type machine struct { commands []string failRestart bool } func (m *machine) run(_ context.Context, name string, args ...string) (string, error) { m.commands = append(m.commands, name+" "+strings.Join(args, " ")) if m.failRestart && len(args) > 0 && args[0] == "restart" { return "", fmt.Errorf("exit status 1") } return "", nil } func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher { return &Watcher{Root: root, Node: "anchor", Host: "anchor", Asker: func() link.Asker { if a == nil { return nil } return a }, Run: m.run, Now: func() time.Time { return *clock }, Concluded: func(v link.Rollback) { *verdicts = append(*verdicts, v) }, Every: Every, } } // look runs the watcher n times, the clock moving Every each time. func look(w *Watcher, clock *time.Time, n int) { for i := 0; i < n; i++ { *clock = clock.Add(Every) w.Look(context.Background()) } } func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) { return func() ([]byte, bool, error) { return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`, instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil } } // A controller that starts and never takes the lease: restored to the build before, once, said once. func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") { t.Fatal("the new controller is not running with the build before it kept beside it") } // The old instance's lease, taken an hour ago, is all the bus ever shows. a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, int(ControllerWithin/Every)-1) if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB { t.Fatalf("rolled back before its bound: %+v", verdicts) } look(w, &clock, 1) if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA || verdicts[0].Component != link.ComponentController { t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts) } if running(t, root, "mesh-controller") != buildA { t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller")) } if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" { t.Fatalf("the machine was asked %v", m.commands) } if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") { t.Fatalf("the verdict does not say why: %s", verdicts[0].Why) } // Once: the restored build is not judged again, and nothing more is said or done. look(w, &clock, 30) if len(verdicts) != 1 || len(m.commands) != 2 { t.Fatalf("judged again after a rollback: %d verdicts, %v", len(verdicts), m.commands) } // Standing, so every report says it โ€” until the mesh asks for another build. if s := Standing(root); len(s) != 1 || s[0].From != buildB { t.Fatalf("what every report says is %+v", s) } // The mesh still declares B: refused, A kept running, the verdict still standing. p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock) if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") { t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller")) } if len(Standing(root)) != 1 || len(Trials(root)) != 0 { t.Fatal("refusing the rolled-back build changed what stands or started a trial") } // A newer build is a new trial, and ends what was concluded once it is proved. place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 { t.Fatal("a newer build after a rollback was not placed on trial") } if err := Proved(root, "mesh-controller"); err != nil { t.Fatal(err) } if len(Standing(root)) != 0 || kept(root, "mesh-controller") { t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept") } } // A runtime that never answers PING: restored, once. func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) place(t, root, "node-tools", ByPing, buildA, "", "", clock) place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, int(NodeToolsWithin/Every)+20) if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools { t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts) } if running(t, root, "node-tools") != buildA || kept(root, "node-tools") { t.Fatal("the runtime running is not the build before, or a copy of it is still kept") } } // A healthy update: nothing deleted before it is proved; the build before it deleted once it is. func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) // The controller placed by a host from before any witness: a record, and no state. if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil { t.Fatal(err) } place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) started := clock a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, 5) if !kept(root, "mesh-controller") { t.Fatal("the build before was deleted while the new one was still on trial") } // A third build while the second is on trial: the one kept is still the one seen healthy. place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) if s, _ := Load(root, "mesh-controller"); s.Previous != buildA { t.Fatalf("the build kept to go back to is %s, want A โ€” the last one seen healthy", s.Previous) } started = clock a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock) look(w, &clock, 2) if len(verdicts) != 0 || len(m.commands) != 0 { t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands) } if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC { t.Fatal("the build before was not deleted once the new one was proved") } } // A build declared not reversible is never rolled back: the build before never starts against what it // changed, and the verdict is urgent and stands. func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock) a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, int(ControllerWithin/Every)+20) if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" { t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts) } if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") { t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why) } if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands, running(t, root, "mesh-controller")) } if len(Standing(root)) != 1 { t.Fatal("the not-reversible verdict does not stand") } } // A witness that cannot ask counts nothing against the build, and says so at GiveUp. func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) a := &asker{lease: func() ([]byte, bool, error) { return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk) }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, int(GiveUp/Every)-1) if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands) } look(w, &clock, 1) if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 { t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts) } if !kept(root, "mesh-controller") { t.Fatal("the build before was deleted though the new one was never seen healthy") } // And a host with no link at all is the same: nothing counted. root2 := t.TempDir() place(t, root2, "node-tools", ByPing, buildA, "", "", clock) place(t, root2, "node-tools", ByPing, buildB, buildA, "", clock) var none []link.Rollback w2 := watcher(root2, nil, m, &clock, &none) look(w2, &clock, int(NodeToolsWithin/Every)*3) if len(none) != 0 || running(t, root2, "node-tools") != buildB { t.Fatal("a build was rolled back while this host had no link to ask with") } } // A restoration whose build will not start is said as such โ€” not as a rollback that worked. func TestARestorationThatDoesNotStartIsSaid(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) place(t, root, "node-tools", ByPing, buildA, "", "", clock) place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) v, err := Restore(context.Background(), root, "node-tools", "it never answered", (&machine{failRestart: true}).run, clock) if err != nil { t.Fatal(err) } if v.Outcome != link.RestoreFailed || running(t, root, "node-tools") != buildA { t.Fatalf("the verdict is %+v, running %s", v, running(t, root, "node-tools")) } } // What the engine keeps survives the engine: a host that stands aside mid-trial resumes it. func TestATrialSurvivesTheHostRestarting(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) place(t, root, "node-tools", ByPing, buildA, "", "", clock) place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) a := &asker{ping: func() error { return link.ErrNoAnswer }} var verdicts []link.Rollback half := int(NodeToolsWithin/Every) / 2 look(watcher(root, a, &machine{}, &clock, &verdicts), &clock, half) // A new watcher, as a successor host would start. look(watcher(root, a, &machine{}, &clock, &verdicts), &clock, int(NodeToolsWithin/Every)-half) if len(verdicts) != 1 { t.Fatalf("the bound was not carried across the host restarting: %+v", verdicts) } }