mesh-catalog/modules/pacman/cmd/pacman-tools/transaction_test.go

package main

// An upgrade through an ask, and a keeper started again following its run (novox/hq ADR 0508 §1, §8;
// acceptance 1841/AC1 and 1841/AC9).

import (
	"encoding/json"
	"errors"
	"strings"
	"sync"
	"testing"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
)

// fakeRoot is the root helper: it records the transactions it was asked to start.
type fakeRoot struct {
	started []string
	fail    error
	journal []string
	mapped  []Mapped
	mapsErr error
	// mirrors is each mirror list written, whole (ADR 0585 §3), under mu: the keeper's loop writes it.
	mu      sync.Mutex
	mirrors [][]string
}

func (f *fakeRoot) Mirrorlist(servers []string) error {
	if f.fail != nil {
		return f.fail
	}
	f.mu.Lock()
	defer f.mu.Unlock()
	f.mirrors = append(f.mirrors, append([]string(nil), servers...))
	return nil
}

func (f *fakeRoot) written() [][]string {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([][]string(nil), f.mirrors...)
}

func (f *fakeRoot) Transaction(kind, run string, ignore []string) error {
	if f.fail != nil {
		return f.fail
	}
	f.started = append(f.started, kind+" "+run+" ignore="+strings.Join(ignore, ","))
	return nil
}

func (f *fakeRoot) Rollback(run string, plan []RollbackItem) error {
	if f.fail != nil {
		return f.fail
	}
	var names []string
	for _, i := range plan {
		names = append(names, i.Name+":"+i.To+":"+i.Source)
	}
	f.started = append(f.started, RunRollback+" "+run+" plan="+strings.Join(names, ","))
	return nil
}

func (f *fakeRoot) Journal(unit string, n int) ([]string, error) { return f.journal, nil }

func (f *fakeRoot) Maps() ([]Mapped, error) { return f.mapped, f.mapsErr }

// keeperBed is one node's keeper over states in memory and a machine of fakes: a unit's state, the lock and
// pacman's log are set by the test as the transaction goes.
type keeperBed struct {
	u                                       *Upgrades
	root                                    *fakeRoot
	events                                  *eventLog
	installed, updates, runs, records, asks *memBucket
	rounds                                  *memBucket
	now                                     time.Time
	unitState                               string // systemctl show's answer for the run's unit
	locked                                  bool
	log                                     string
}

const upgradeLog = "[2026-10-04T12:00:01+0000] [PACMAN] Running 'pacman -Syu --noconfirm --ignore linux,linux-headers'\n" +
	"[2026-10-04T12:00:02+0000] [PACMAN] starting full system upgrade\n" +
	"[2026-10-04T12:00:09+0000] [ALPM] transaction started\n" +
	"[2026-10-04T12:00:10+0000] [ALPM] upgraded bash (5.3.15-1 -> 5.3.20-1)\n" +
	"[2026-10-04T12:00:10+0000] [ALPM] installed made-up-library (1.0-1)\n" +
	"[2026-10-04T12:00:11+0000] [ALPM] transaction completed\n"

func newKeeperBed(t *testing.T) *keeperBed {
	k := &keeperBed{root: &fakeRoot{}, events: &eventLog{}, installed: newBucket(), updates: newBucket(), runs: newBucket(),
		records: newBucket(), asks: newBucket(), rounds: newBucket(), now: time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC), unitState: "LoadState=not-found\n"}
	saved := fetch
	fetch = func(string) ([]byte, error) { return nil, errors.New("offline") }
	t.Cleanup(func() { fetch = saved })
	if _, err := k.installed.Put("node-a", nodeAPackages); err != nil {
		t.Fatal(err)
	}
	if _, err := k.updates.Put("node-a", UpdatesValue{Reachable: true, Packages: []VersionChange{
		{Name: "bash", From: "5.3.15-1", To: "5.3.20-1"},
		{Name: "linux", From: "7.2.8.arch1-2", To: "7.3.1.arch1-1"},
		{Name: "linux-headers", From: "7.2.8.arch1-2", To: "7.3.1.arch1-1"},
	}}); err != nil {
		t.Fatal(err)
	}
	sources := capturedZFSSource(t)
	files := func(paths ...string) Ran { return Ran{Stdout: strings.Join(paths, "\n") + "\n"} }
	m := machine(fake(func(c call) Ran {
		line := c.String()
		switch {
		case line == "systemctl list-units --state=failed --plain --no-legend --no-pager":
			return Ran{}
		case line == "dkms status":
			return Ran{Stdout: zfsStatus}
		case line == "pacman -Qq":
			return Ran{Stdout: "bash\nlinux\n"}
		case line == "pacman -Qlq -- linux":
			return files("/usr/lib/modules/7.2.8-arch1-2/vmlinuz")
		case line == "pacman -Qlq -- linux-headers":
			return files("/usr/lib/modules/7.2.8-arch1-2/build/Makefile")
		case strings.HasPrefix(line, "systemctl show "+unitPrefix):
			return Ran{Stdout: k.unitState}
		}
		return Ran{Status: 99, Stderr: "unexpected command: " + line}
	}, nil), 1000)
	m.Now = func() time.Time { return k.now }
	m.ReadFile = func(path string) ([]byte, error) {
		switch {
		case path == OSRelease:
			return []byte("7.2.8-arch1-2\n"), nil
		case path == DBLock && k.locked:
			return []byte{}, nil
		case path == PacmanLog:
			return []byte(k.log), nil
		}
		if s, ok := sources[path]; ok {
			return []byte(s), nil
		}
		return nil, errNoFile
	}
	k.u = k.keeper(m)
	return k
}

// keeper is a keeper for node-a over the bed's states: a second call is the keeper started again.
func (k *keeperBed) keeper(m *Machine) *Upgrades {
	return &Upgrades{Node: "node-a", Machine: m, Root: k.root, Events: k.events, Installed: k.installed, Updates: k.updates,
		Runs: k.runs, Records: k.records, Rounds: k.rounds, Asks: k.asks}
}

// ask writes an ask for node and hands its change to the keeper, as the watch of `asks` would.
func (k *keeperBed) ask(t *testing.T, node, act string) (string, error) {
	t.Helper()
	a := Ask{ID: RunID(k.now) + "-abc123", Node: node, Act: act, At: k.now}
	key := NodeKey(node, a.ID)
	rev, err := k.asks.Create(key, a)
	if err != nil {
		t.Fatal(err)
	}
	raw, _ := json.Marshal(a)
	return key, k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: key, Op: "put", Value: raw, Revision: rev})
}

func (k *keeperBed) run(t *testing.T) Run {
	t.Helper()
	keys, _ := k.runs.Keys()
	if len(keys) != 1 {
		t.Fatalf("runs: %v", keys)
	}
	var r Run
	k.runs.value(keys[0], &r)
	return r
}

func steps(r Run) string {
	var s []string
	for _, st := range r.Steps {
		s = append(s, st.Step)
	}
	return strings.Join(s, " ")
}

func TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) {
	k := newKeeperBed(t)
	key, err := k.ask(t, "node-a", ActUpgrade)
	if err != nil {
		t.Fatal(err)
	}
	r := k.run(t)
	if r.Run != "20261004T120000Z" || r.From != FromAsk || r.Ask != "20261004T120000Z-abc123" || r.Kind != RunUpgrade {
		t.Errorf("the run: %+v", r)
	}
	if steps(r) != "recorded checked-before started" {
		t.Fatalf("steps before the transaction ends: %s", steps(r))
	}
	if h := r.Steps[1].Held; len(h) != 1 || h[0].Package != "linux" {
		t.Errorf("the gate's step names the held kernel: %+v", h)
	}
	if s := r.Steps[2]; s.Unit != "mesh-pacman-upgrade-20261004t120000z" || strings.Join(s.Ignored, ",") != "linux,linux-headers" {
		t.Errorf("started: %+v", s)
	}
	if strings.Join(k.root.started, ";") != "upgrade 20261004T120000Z ignore=linux,linux-headers" {
		t.Errorf("the root helper was asked: %v", k.root.started)
	}
	if _, there := k.asks.values[key]; there {
		t.Error("the ask is deleted once the run names it")
	}
	var rec Record
	if !k.records.value("node-a.20261004T120000Z", &rec) {
		t.Error("no record was taken")
	}

	// The transaction ends: the lock is released and the keeper follows the run.
	k.unitState, k.log, k.now = "LoadState=not-found\n", upgradeLog, k.now.Add(15*time.Second)
	if err := k.u.Follow(); err != nil {
		t.Fatal(err)
	}
	r = k.run(t)
	if steps(r) != "recorded checked-before started finished checked-after" {
		t.Fatalf("steps: %s", steps(r))
	}
	end := r.Steps[3]
	if end.Exit == nil || *end.Exit != 0 || end.Changes == nil || len(end.Changes.Changed) != 1 || end.Changes.Changed[0].Name != "bash" ||
		len(end.Changes.Added) != 1 {
		t.Errorf("finished: %+v %+v", end, end.Changes)
	}
	if c := r.Steps[4].Check; c == nil || !c.Passed {
		t.Errorf("the check: %+v", c)
	}
	if got := strings.Join(k.events.names(), " "); got != "upgrade.recorded upgrade.kernel-held upgrade.started upgrade.finished upgrade.checked" {
		t.Errorf("events: %s", got)
	}
	if w := k.runs.writesTo("node-a."); len(w) != 5 || w[0] != "create node-a.20261004T120000Z" {
		t.Errorf("each step written once: %v", w)
	}
	if err := k.u.Follow(); err != nil || len(k.runs.writesTo("node-a.")) != 5 {
		t.Errorf("an ended run is not written again: %v", err)
	}
}

func TestAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked(t *testing.T) {
	k := newKeeperBed(t)
	if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
		t.Fatal(err)
	}
	k.unitState = "LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\n"
	k.root.journal = []string{"checking for file conflicts...", "error: failed to commit transaction (conflicting files)", "Errors occurred, no packages were upgraded."}
	if err := k.u.Follow(); err != nil {
		t.Fatal(err)
	}
	r := k.run(t)
	end := r.Steps[3]
	if steps(r) != "recorded checked-before started failed checked-after" || *end.Exit != 1 ||
		end.Error != "error: failed to commit transaction (conflicting files)" || r.Steps[4].Check.Passed {
		t.Errorf("%s %+v %+v", steps(r), end, r.Steps[4].Check)
	}
	if got := k.events.names(); got[len(got)-2] != EventFailed || got[len(got)-1] != EventChecked {
		t.Errorf("events: %v", got)
	}
}

func TestATransactionThatCannotStartFailsAndIsChecked(t *testing.T) {
	k := newKeeperBed(t)
	k.root.fail = errors.New("the root helper's socket is not there")
	if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
		t.Fatal(err)
	}
	r := k.run(t)
	if steps(r) != "recorded checked-before failed checked-after" || !strings.Contains(r.Steps[2].Error, "socket is not there") {
		t.Errorf("%s %+v", steps(r), r.Steps[2])
	}
}

func TestAnAskIsTakenByItsNodesKeeperAlone(t *testing.T) {
	k := newKeeperBed(t)
	key, err := k.ask(t, "node-b", ActUpgrade)
	if err != nil {
		t.Fatal(err)
	}
	if keys, _ := k.runs.Keys(); len(keys) != 0 || len(k.root.started) != 0 {
		t.Errorf("node-a's keeper took node-b's ask: %v %v", keys, k.root.started)
	}
	if _, there := k.asks.values[key]; !there {
		t.Error("another node's ask is left for its keeper")
	}
	if err := k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: "node-a.x", Op: "delete"}); err != nil {
		t.Errorf("a delete is no ask: %v", err)
	}
}

func TestAnAskARunAlreadyNamesIsDeletedAndNotRunAgain(t *testing.T) {
	k := newKeeperBed(t)
	done := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk, Ask: RunID(k.now) + "-abc123",
		Steps: []Step{{Step: StepRefused, At: k.now, Why: "x"}}}
	if _, err := k.runs.Create(NodeKey("node-a", done.Run), done); err != nil {
		t.Fatal(err)
	}
	key, err := k.ask(t, "node-a", ActUpgrade)
	if err != nil {
		t.Fatal(err)
	}
	if keys, _ := k.runs.Keys(); len(keys) != 1 || len(k.root.started) != 0 {
		t.Errorf("taken twice: %v", keys)
	}
	if _, there := k.asks.values[key]; there {
		t.Error("the ask a run names is deleted")
	}
}

func TestARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn(t *testing.T) {
	k := newKeeperBed(t)
	k.locked = true
	if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
		t.Fatal(err)
	}
	r := k.run(t)
	if steps(r) != "refused" || !strings.Contains(r.Steps[0].Why, DBLock) || len(k.root.started) != 0 {
		t.Errorf("locked: %s %+v", steps(r), r.Steps)
	}
	if e := k.events.sent; len(e) != 1 || e[0].Name != EventFailed || e[0].Body.(UpgradeFailed).Step != StepRefused {
		t.Errorf("a refused run is said: %+v", e)
	}

	k = newKeeperBed(t)
	going := Run{Node: "node-a", Run: "20261004T110000Z", Kind: RunUpgrade, From: FromRound, Steps: []Step{{Step: StepStarted, At: k.now}}}
	if _, err := k.runs.Create(NodeKey("node-a", going.Run), going); err != nil {
		t.Fatal(err)
	}
	if _, err := k.u.StartUpgrade(FromAsk, "", "a1"); err != nil {
		t.Fatal(err)
	}
	var r2 Run
	k.runs.value("node-a.20261004T120000Z", &r2)
	if steps(r2) != "refused" || !strings.Contains(r2.Steps[0].Why, "20261004T110000Z is still going") {
		t.Errorf("another run going: %+v", r2)
	}
}

func TestAKeeperStartedAgainFollowsItsRunToItsEnd(t *testing.T) {
	k := newKeeperBed(t)
	if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
		t.Fatal(err)
	}
	// The keeper goes away while the transaction runs; a new one starts and finds it still running.
	again := k.keeper(k.u.Machine)
	k.unitState = "LoadState=loaded\nActiveState=active\nResult=success\nExecMainStatus=0\n"
	before := len(k.runs.writes)
	if err := again.Follow(); err != nil || len(k.runs.writes) != before {
		t.Fatalf("a running transaction is left running: %v, %d writes", err, len(k.runs.writes)-before)
	}
	k.unitState, k.log = "LoadState=not-found\n", upgradeLog
	if err := again.Follow(); err != nil {
		t.Fatal(err)
	}
	if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" {
		t.Errorf("followed to its end: %s", steps(r))
	}
}

func TestAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps(t *testing.T) {
	// Stopped after the gate, before the transaction started: nothing started, so the run is refused.
	k := newKeeperBed(t)
	stopped := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk,
		Steps: []Step{{Step: StepRecorded, At: k.now}, {Step: StepCheckedBefore, At: k.now}}}
	if _, err := k.runs.Create(NodeKey("node-a", stopped.Run), stopped); err != nil {
		t.Fatal(err)
	}
	if err := k.u.Follow(); err != nil {
		t.Fatal(err)
	}
	if r := k.run(t); steps(r) != "recorded checked-before refused" || !strings.Contains(r.Steps[2].Why, "nothing was started") {
		t.Errorf("%s %+v", steps(r), r.Steps)
	}

	// Stopped after the transaction started and before it was written: its log says it ran.
	k = newKeeperBed(t)
	if _, err := k.runs.Create(NodeKey("node-a", stopped.Run), stopped); err != nil {
		t.Fatal(err)
	}
	k.log = upgradeLog
	if err := k.u.Follow(); err != nil {
		t.Fatal(err)
	}
	if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" || r.Steps[2].Unit != "mesh-pacman-upgrade-20261004t115900z" {
		t.Errorf("%s %+v", steps(r), r.Steps)
	}

	// Stopped after the transaction ended and before it was checked.
	k = newKeeperBed(t)
	zero := 0
	unchecked := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk,
		Steps: []Step{{Step: StepRecorded, At: k.now}, {Step: StepCheckedBefore, At: k.now}, {Step: StepStarted, At: k.now}, {Step: StepFinished, At: k.now, Exit: &zero}}}
	if _, err := k.runs.Create(NodeKey("node-a", unchecked.Run), unchecked); err != nil {
		t.Fatal(err)
	}
	if err := k.u.Follow(); err != nil {
		t.Fatal(err)
	}
	if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" {
		t.Errorf("%s", steps(r))
	}
}

func TestPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing(t *testing.T) {
	asks := newBucket()
	savedState, savedNode, savedFetch := upgradeState, thisNode, fetch
	upgradeState = func(name string) Bucket {
		if name != StateAsks {
			t.Fatalf("wrote %s", name)
		}
		return asks
	}
	thisNode = func() string { return "node-a" }
	fetch = func(string) ([]byte, error) { return nil, errors.New("offline") }
	defer func() { upgradeState, thisNode, fetch = savedState, savedNode, savedFetch }()
	var calls []call
	m := machine(fake(func(c call) Ran { return Ran{} }, &calls), 1000)
	for _, tool := range tools(m) {
		if tool.Name != "pacman_upgrade" {
			continue
		}
		out, err := tool.Run(map[string]any{})
		if err != nil {
			t.Fatal(err)
		}
		a := out.(map[string]any)["asked"].(Ask)
		if a.Node != "node-a" || a.Act != ActUpgrade || !strings.HasPrefix(a.ID, "20261004T120000Z-") {
			t.Errorf("ask: %+v", a)
		}
		if keys, _ := asks.Keys(); len(keys) != 1 || keys[0] != NodeKey("node-a", a.ID) {
			t.Errorf("asks: %v", keys)
		}
		for _, c := range calls {
			if c.name == "sudo" || c.name == "systemd-run" {
				t.Errorf("the tool acted itself: %s", c)
			}
		}
		if _, err := tool.Run(map[string]any{"node": "node-b"}); err != nil {
			t.Fatal(err)
		}
		if keys, _ := asks.Keys(); len(keys) != 2 || !strings.HasPrefix(keys[1], "node-b.") {
			t.Errorf("another node's ask: %v", keys)
		}
		return
	}
	t.Fatal("no pacman_upgrade")
}