mesh-host/internal/witness/watch_test.go

package witness

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/link"
)

// Each rollback path, induced against a real directory: a runtime that never answers PING, a build
// declared not reversible; and a healthy update, which
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).

const (
	buildA = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	buildB = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
	buildC = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
)

// place does what the applier does with a declared build: ask Place, unpack when told, commit.
func place(t *testing.T, root, name, by, build, recorded, notReversible string, at time.Time) Placing {
	t.Helper()
	p, err := Place(root, name, by, build, recorded, notReversible, at)
	if err != nil {
		t.Fatal(err)
	}
	if p.Unpack {
		dir := filepath.Join(root, name)
		if err := os.MkdirAll(dir, 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(dir, "build"), []byte(build), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	if err := p.Commit(); err != nil {
		t.Fatal(err)
	}
	return p
}

func running(t *testing.T, root, name string) string {
	t.Helper()
	raw, err := os.ReadFile(filepath.Join(root, name, "build"))
	if err != nil {
		return "nothing"
	}
	return string(raw)
}

func kept(root, name string) bool {
	_, err := os.Stat(previousDir(root, name))
	return err == nil
}

// asker answers as the build it is told is running would.
type asker struct {
	mu    sync.Mutex
	ping  func() error
	asked int
}

func (a *asker) Ping(context.Context, string, string) error {
	a.mu.Lock()
	defer a.mu.Unlock()
	a.asked++
	return a.ping()
}

type machine struct {
	commands    []string
	failRestart bool
}

func (m *machine) run(_ context.Context, name string, args ...string) (string, error) {
	m.commands = append(m.commands, name+" "+strings.Join(args, " "))
	if m.failRestart && len(args) > 0 && args[0] == "restart" {
		return "", fmt.Errorf("exit status 1")
	}
	return "", nil
}

func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
	return &Watcher{Root: root, Node: "anchor",
		Asker: func() link.Asker {
			if a == nil {
				return nil
			}
			return a
		},
		Run: m.run, Now: func() time.Time { return *clock },
		Concluded: func(v link.Rollback) { *verdicts = append(*verdicts, v) },
		Every:     Every,
	}
}

// look runs the watcher n times, the clock moving Every each time.
func look(w *Watcher, clock *time.Time, n int) {
	for i := 0; i < n; i++ {
		*clock = clock.Add(Every)
		w.Look(context.Background())
	}
}

// A runtime that starts and never answers PING: restored to the build before, once, said once.
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
	root := t.TempDir()
	clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
	place(t, root, "node-tools", ByPing, buildA, "", "", clock)
	place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
	if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") {
		t.Fatal("the new runtime is not running with the build before it kept beside it")
	}

	a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
	m := &machine{}
	var verdicts []link.Rollback
	w := watcher(root, a, m, &clock, &verdicts)

	look(w, &clock, int(NodeToolsWithin/Every)-1)
	if len(verdicts) != 0 || running(t, root, "node-tools") != buildB {
		t.Fatalf("rolled back before its bound: %+v", verdicts)
	}
	look(w, &clock, 1)
	if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
		verdicts[0].Component != link.ComponentNodeTools {
		t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts)
	}
	if running(t, root, "node-tools") != buildA {
		t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools"))
	}
	if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" {
		t.Fatalf("the machine was asked %v", m.commands)
	}
	if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") {
		t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
	}

	// Once: the restored build is not judged again, and nothing more is said or done.
	look(w, &clock, 30)
	if len(verdicts) != 1 || len(m.commands) != 2 {
		t.Fatalf("judged again after a rollback: %d verdicts, %v", len(verdicts), m.commands)
	}
	// Standing, so every report says it — until the mesh asks for another build.
	if s := Standing(root); len(s) != 1 || s[0].From != buildB {
		t.Fatalf("what every report says is %+v", s)
	}
	// The mesh still declares B: refused, A kept running, the verdict still standing.
	p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock)
	if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") {
		t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools"))
	}
	if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
		t.Fatal("refusing the rolled-back build changed what stands or started a trial")
	}
	// A newer build is a new trial, and ends what was concluded once it is proved.
	place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
	if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 {
		t.Fatal("a newer build after a rollback was not placed on trial")
	}
	if err := Proved(root, "node-tools"); err != nil {
		t.Fatal(err)
	}
	if len(Standing(root)) != 0 || kept(root, "node-tools") {
		t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
	}
}

// A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
	root := t.TempDir()
	clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
	// The runtime placed by a host from before any witness: a record, and no state.
	if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil {
		t.Fatal(err)
	}
	place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)

	a := &asker{ping: func() error { return link.ErrNoAnswer }}
	m := &machine{}
	var verdicts []link.Rollback
	w := watcher(root, a, m, &clock, &verdicts)
	look(w, &clock, 5)
	if !kept(root, "node-tools") {
		t.Fatal("the build before was deleted while the new one was still on trial")
	}
	// A third build while the second is on trial: the one kept is still the one seen healthy.
	place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
	if s, _ := Load(root, "node-tools"); s.Previous != buildA {
		t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
	}
	a.ping = func() error { return nil }
	look(w, &clock, 2)
	if len(verdicts) != 0 || len(m.commands) != 0 {
		t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
	}
	if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC {
		t.Fatal("the build before was not deleted once the new one was proved")
	}
}

// A build declared not reversible is never rolled back: the build before never starts against what it
// changed, and the verdict is urgent and stands.
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
	root := t.TempDir()
	clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
	place(t, root, "node-tools", ByPing, buildA, "", "", clock)
	place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock)

	a := &asker{ping: func() error { return link.ErrNoAnswer }}
	m := &machine{}
	var verdicts []link.Rollback
	w := watcher(root, a, m, &clock, &verdicts)
	look(w, &clock, int(NodeToolsWithin/Every)+20)

	if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
		t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
	}
	if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
		t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
	}
	if len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
		t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
			running(t, root, "node-tools"))
	}
	if len(Standing(root)) != 1 {
		t.Fatal("the not-reversible verdict does not stand")
	}
}

// A witness that cannot ask counts nothing against the build, and says so at GiveUp.
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
	root := t.TempDir()
	clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
	place(t, root, "node-tools", ByPing, buildA, "", "", clock)
	place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)

	a := &asker{ping: func() error {
		return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk)
	}}
	m := &machine{}
	var verdicts []link.Rollback
	w := watcher(root, a, m, &clock, &verdicts)
	look(w, &clock, int(GiveUp/Every)-1)
	if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
		t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
	}
	look(w, &clock, 1)
	if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
		t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
	}
	if !kept(root, "node-tools") {
		t.Fatal("the build before was deleted though the new one was never seen healthy")
	}

	// And a host with no link at all is the same: nothing counted.
	root2 := t.TempDir()
	place(t, root2, "node-tools", ByPing, buildA, "", "", clock)
	place(t, root2, "node-tools", ByPing, buildB, buildA, "", clock)
	var none []link.Rollback
	w2 := watcher(root2, nil, m, &clock, &none)
	look(w2, &clock, int(NodeToolsWithin/Every)*3)
	if len(none) != 0 || running(t, root2, "node-tools") != buildB {
		t.Fatal("a build was rolled back while this host had no link to ask with")
	}
}

// A restoration whose build will not start is said as such — not as a rollback that worked.
func TestARestorationThatDoesNotStartIsSaid(t *testing.T) {
	root := t.TempDir()
	clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
	place(t, root, "node-tools", ByPing, buildA, "", "", clock)
	place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
	v, err := Restore(context.Background(), root, "node-tools", "it never answered", (&machine{failRestart: true}).run, clock)
	if err != nil {
		t.Fatal(err)
	}
	if v.Outcome != link.RestoreFailed || running(t, root, "node-tools") != buildA {
		t.Fatalf("the verdict is %+v, running %s", v, running(t, root, "node-tools"))
	}
}

// What the engine keeps survives the engine: a host that stands aside mid-trial resumes it.
func TestATrialSurvivesTheHostRestarting(t *testing.T) {
	root := t.TempDir()
	clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
	place(t, root, "node-tools", ByPing, buildA, "", "", clock)
	place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
	a := &asker{ping: func() error { return link.ErrNoAnswer }}
	var verdicts []link.Rollback
	half := int(NodeToolsWithin/Every) / 2
	look(watcher(root, a, &machine{}, &clock, &verdicts), &clock, half)
	// A new watcher, as a successor host would start.
	look(watcher(root, a, &machine{}, &clock, &verdicts), &clock, int(NodeToolsWithin/Every)-half)
	if len(verdicts) != 1 {
		t.Fatalf("the bound was not carried across the host restarting: %+v", verdicts)
	}
}