Files
mesh-catalog/modules/pacman/cmd/pacman-tools/transaction_test.go
T
jschoubben 9afbd35759 NP1: the NFS server and package manager modules' tests fold into one test per file, within 50 as the check counts
The check counts every test the integration level runs, untagged ones in: the NFS server
module held 51 and the package manager module 121, over the budget of 50. Each folded file's
tests run in order as rows of one test that keeps the first one's name and logs each case
before it runs; the root helper's test sockets are numbered per call, as one test now holds
several listeners, and the case that skips without pacman-conf runs last. Counts now 48 and 42.

Builds: design 47 H2 §819; H-A-349 DQ-14
2026-11-22 01:18:06 +01:00

470 lines
17 KiB
Go

package main
// An upgrade through an ask, and a keeper started again following its run (novox/hq ADR 0508 §1, §8;
// acceptance 1841/AC1 and 1841/AC9).
import (
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// fakeRoot is the root helper: it records the transactions it was asked to start.
type fakeRoot struct {
started []string
fail error
journal []string
mapped []Mapped
mapsErr error
// mirrors is each mirror list written, whole (ADR 0585 §3), under mu: the keeper's loop writes it.
mu sync.Mutex
mirrors [][]string
}
func (f *fakeRoot) Mirrorlist(servers []string) error {
if f.fail != nil {
return f.fail
}
f.mu.Lock()
defer f.mu.Unlock()
f.mirrors = append(f.mirrors, append([]string(nil), servers...))
return nil
}
func (f *fakeRoot) written() [][]string {
f.mu.Lock()
defer f.mu.Unlock()
return append([][]string(nil), f.mirrors...)
}
func (f *fakeRoot) Transaction(kind, run string, ignore []string) error {
if f.fail != nil {
return f.fail
}
f.started = append(f.started, kind+" "+run+" ignore="+strings.Join(ignore, ","))
return nil
}
func (f *fakeRoot) Rollback(run string, plan []RollbackItem) error {
if f.fail != nil {
return f.fail
}
var names []string
for _, i := range plan {
names = append(names, i.Name+":"+i.To+":"+i.Source)
}
f.started = append(f.started, RunRollback+" "+run+" plan="+strings.Join(names, ","))
return nil
}
func (f *fakeRoot) Journal(unit string, n int) ([]string, error) { return f.journal, nil }
func (f *fakeRoot) Maps() ([]Mapped, error) { return f.mapped, f.mapsErr }
// keeperBed is one node's keeper over states in memory and a machine of fakes: a unit's state, the lock and
// pacman's log are set by the test as the transaction goes.
type keeperBed struct {
u *Upgrades
root *fakeRoot
events *eventLog
installed, updates, runs, records, asks *memBucket
rounds *memBucket
now time.Time
unitState string // systemctl show's answer for the run's unit
locked bool
log string
}
const upgradeLog = "[2026-10-04T12:00:01+0000] [PACMAN] Running 'pacman -Syu --noconfirm --ignore linux,linux-headers'\n" +
"[2026-10-04T12:00:02+0000] [PACMAN] starting full system upgrade\n" +
"[2026-10-04T12:00:09+0000] [ALPM] transaction started\n" +
"[2026-10-04T12:00:10+0000] [ALPM] upgraded bash (5.3.15-1 -> 5.3.20-1)\n" +
"[2026-10-04T12:00:10+0000] [ALPM] installed made-up-library (1.0-1)\n" +
"[2026-10-04T12:00:11+0000] [ALPM] transaction completed\n"
func newKeeperBed(t *testing.T) *keeperBed {
k := &keeperBed{root: &fakeRoot{}, events: &eventLog{}, installed: newBucket(), updates: newBucket(), runs: newBucket(),
records: newBucket(), asks: newBucket(), rounds: newBucket(), now: time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC), unitState: "LoadState=not-found\n"}
saved := fetch
fetch = func(string) ([]byte, error) { return nil, errors.New("offline") }
t.Cleanup(func() { fetch = saved })
if _, err := k.installed.Put("node-a", nodeAPackages); err != nil {
t.Fatal(err)
}
if _, err := k.updates.Put("node-a", UpdatesValue{Reachable: true, Packages: []VersionChange{
{Name: "bash", From: "5.3.15-1", To: "5.3.20-1"},
{Name: "linux", From: "7.2.8.arch1-2", To: "7.3.1.arch1-1"},
{Name: "linux-headers", From: "7.2.8.arch1-2", To: "7.3.1.arch1-1"},
}}); err != nil {
t.Fatal(err)
}
sources := capturedZFSSource(t)
files := func(paths ...string) Ran { return Ran{Stdout: strings.Join(paths, "\n") + "\n"} }
m := machine(fake(func(c call) Ran {
line := c.String()
switch {
case line == "systemctl list-units --state=failed --plain --no-legend --no-pager":
return Ran{}
case line == "dkms status":
return Ran{Stdout: zfsStatus}
case line == "pacman -Qq":
return Ran{Stdout: "bash\nlinux\n"}
case line == "pacman -Qlq -- linux":
return files("/usr/lib/modules/7.2.8-arch1-2/vmlinuz")
case line == "pacman -Qlq -- linux-headers":
return files("/usr/lib/modules/7.2.8-arch1-2/build/Makefile")
case strings.HasPrefix(line, "systemctl show "+unitPrefix):
return Ran{Stdout: k.unitState}
}
return Ran{Status: 99, Stderr: "unexpected command: " + line}
}, nil), 1000)
m.Now = func() time.Time { return k.now }
m.ReadFile = func(path string) ([]byte, error) {
switch {
case path == OSRelease:
return []byte("7.2.8-arch1-2\n"), nil
case path == DBLock && k.locked:
return []byte{}, nil
case path == PacmanLog:
return []byte(k.log), nil
}
if s, ok := sources[path]; ok {
return []byte(s), nil
}
return nil, errNoFile
}
k.u = k.keeper(m)
return k
}
// keeper is a keeper for node-a over the bed's states: a second call is the keeper started again.
func (k *keeperBed) keeper(m *Machine) *Upgrades {
return &Upgrades{Node: "node-a", Machine: m, Root: k.root, Events: k.events, Installed: k.installed, Updates: k.updates,
Runs: k.runs, Records: k.records, Rounds: k.rounds, Asks: k.asks}
}
// ask writes an ask for node and hands its change to the keeper, as the watch of `asks` would.
func (k *keeperBed) ask(t *testing.T, node, act string) (string, error) {
t.Helper()
a := Ask{ID: RunID(k.now) + "-abc123", Node: node, Act: act, At: k.now}
key := NodeKey(node, a.ID)
rev, err := k.asks.Create(key, a)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(a)
return key, k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: key, Op: "put", Value: raw, Revision: rev})
}
func (k *keeperBed) run(t *testing.T) Run {
t.Helper()
keys, _ := k.runs.Keys()
if len(keys) != 1 {
t.Fatalf("runs: %v", keys)
}
var r Run
k.runs.value(keys[0], &r)
return r
}
func steps(r Run) string {
var s []string
for _, st := range r.Steps {
s = append(s, st.Step)
}
return strings.Join(s, " ")
}
func testAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) {
k := newKeeperBed(t)
key, err := k.ask(t, "node-a", ActUpgrade)
if err != nil {
t.Fatal(err)
}
r := k.run(t)
if r.Run != "20261004T120000Z" || r.From != FromAsk || r.Ask != "20261004T120000Z-abc123" || r.Kind != RunUpgrade {
t.Errorf("the run: %+v", r)
}
if steps(r) != "recorded checked-before started" {
t.Fatalf("steps before the transaction ends: %s", steps(r))
}
if h := r.Steps[1].Held; len(h) != 1 || h[0].Package != "linux" {
t.Errorf("the gate's step names the held kernel: %+v", h)
}
if s := r.Steps[2]; s.Unit != "mesh-pacman-upgrade-20261004t120000z" || strings.Join(s.Ignored, ",") != "linux,linux-headers" {
t.Errorf("started: %+v", s)
}
if strings.Join(k.root.started, ";") != "upgrade 20261004T120000Z ignore=linux,linux-headers" {
t.Errorf("the root helper was asked: %v", k.root.started)
}
if _, there := k.asks.values[key]; there {
t.Error("the ask is deleted once the run names it")
}
var rec Record
if !k.records.value("node-a.20261004T120000Z", &rec) {
t.Error("no record was taken")
}
// The transaction ends: the lock is released and the keeper follows the run.
k.unitState, k.log, k.now = "LoadState=not-found\n", upgradeLog, k.now.Add(15*time.Second)
if err := k.u.Follow(); err != nil {
t.Fatal(err)
}
r = k.run(t)
if steps(r) != "recorded checked-before started finished checked-after" {
t.Fatalf("steps: %s", steps(r))
}
end := r.Steps[3]
if end.Exit == nil || *end.Exit != 0 || end.Changes == nil || len(end.Changes.Changed) != 1 || end.Changes.Changed[0].Name != "bash" ||
len(end.Changes.Added) != 1 {
t.Errorf("finished: %+v %+v", end, end.Changes)
}
if c := r.Steps[4].Check; c == nil || !c.Passed {
t.Errorf("the check: %+v", c)
}
if got := strings.Join(k.events.names(), " "); got != "upgrade.recorded upgrade.kernel-held upgrade.started upgrade.finished upgrade.checked" {
t.Errorf("events: %s", got)
}
if w := k.runs.writesTo("node-a."); len(w) != 5 || w[0] != "create node-a.20261004T120000Z" {
t.Errorf("each step written once: %v", w)
}
if err := k.u.Follow(); err != nil || len(k.runs.writesTo("node-a.")) != 5 {
t.Errorf("an ended run is not written again: %v", err)
}
}
func testAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked(t *testing.T) {
k := newKeeperBed(t)
if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
t.Fatal(err)
}
k.unitState = "LoadState=loaded\nActiveState=failed\nResult=exit-code\nExecMainStatus=1\n"
k.root.journal = []string{"checking for file conflicts...", "error: failed to commit transaction (conflicting files)", "Errors occurred, no packages were upgraded."}
if err := k.u.Follow(); err != nil {
t.Fatal(err)
}
r := k.run(t)
end := r.Steps[3]
if steps(r) != "recorded checked-before started failed checked-after" || *end.Exit != 1 ||
end.Error != "error: failed to commit transaction (conflicting files)" || r.Steps[4].Check.Passed {
t.Errorf("%s %+v %+v", steps(r), end, r.Steps[4].Check)
}
if got := k.events.names(); got[len(got)-2] != EventFailed || got[len(got)-1] != EventChecked {
t.Errorf("events: %v", got)
}
}
func testATransactionThatCannotStartFailsAndIsChecked(t *testing.T) {
k := newKeeperBed(t)
k.root.fail = errors.New("the root helper's socket is not there")
if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
t.Fatal(err)
}
r := k.run(t)
if steps(r) != "recorded checked-before failed checked-after" || !strings.Contains(r.Steps[2].Error, "socket is not there") {
t.Errorf("%s %+v", steps(r), r.Steps[2])
}
}
func testAnAskIsTakenByItsNodesKeeperAlone(t *testing.T) {
k := newKeeperBed(t)
key, err := k.ask(t, "node-b", ActUpgrade)
if err != nil {
t.Fatal(err)
}
if keys, _ := k.runs.Keys(); len(keys) != 0 || len(k.root.started) != 0 {
t.Errorf("node-a's keeper took node-b's ask: %v %v", keys, k.root.started)
}
if _, there := k.asks.values[key]; !there {
t.Error("another node's ask is left for its keeper")
}
if err := k.u.TakeAsk(stdio.StateChange{State: StateAsks, Key: "node-a.x", Op: "delete"}); err != nil {
t.Errorf("a delete is no ask: %v", err)
}
}
func testAnAskARunAlreadyNamesIsDeletedAndNotRunAgain(t *testing.T) {
k := newKeeperBed(t)
done := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk, Ask: RunID(k.now) + "-abc123",
Steps: []Step{{Step: StepRefused, At: k.now, Why: "x"}}}
if _, err := k.runs.Create(NodeKey("node-a", done.Run), done); err != nil {
t.Fatal(err)
}
key, err := k.ask(t, "node-a", ActUpgrade)
if err != nil {
t.Fatal(err)
}
if keys, _ := k.runs.Keys(); len(keys) != 1 || len(k.root.started) != 0 {
t.Errorf("taken twice: %v", keys)
}
if _, there := k.asks.values[key]; there {
t.Error("the ask a run names is deleted")
}
}
func testARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn(t *testing.T) {
k := newKeeperBed(t)
k.locked = true
if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
t.Fatal(err)
}
r := k.run(t)
if steps(r) != "refused" || !strings.Contains(r.Steps[0].Why, DBLock) || len(k.root.started) != 0 {
t.Errorf("locked: %s %+v", steps(r), r.Steps)
}
if e := k.events.sent; len(e) != 1 || e[0].Name != EventFailed || e[0].Body.(UpgradeFailed).Step != StepRefused {
t.Errorf("a refused run is said: %+v", e)
}
k = newKeeperBed(t)
going := Run{Node: "node-a", Run: "20261004T110000Z", Kind: RunUpgrade, From: FromRound, Steps: []Step{{Step: StepStarted, At: k.now}}}
if _, err := k.runs.Create(NodeKey("node-a", going.Run), going); err != nil {
t.Fatal(err)
}
if _, err := k.u.StartUpgrade(FromAsk, "", "a1"); err != nil {
t.Fatal(err)
}
var r2 Run
k.runs.value("node-a.20261004T120000Z", &r2)
if steps(r2) != "refused" || !strings.Contains(r2.Steps[0].Why, "20261004T110000Z is still going") {
t.Errorf("another run going: %+v", r2)
}
}
func testAKeeperStartedAgainFollowsItsRunToItsEnd(t *testing.T) {
k := newKeeperBed(t)
if _, err := k.ask(t, "node-a", ActUpgrade); err != nil {
t.Fatal(err)
}
// The keeper goes away while the transaction runs; a new one starts and finds it still running.
again := k.keeper(k.u.Machine)
k.unitState = "LoadState=loaded\nActiveState=active\nResult=success\nExecMainStatus=0\n"
before := len(k.runs.writes)
if err := again.Follow(); err != nil || len(k.runs.writes) != before {
t.Fatalf("a running transaction is left running: %v, %d writes", err, len(k.runs.writes)-before)
}
k.unitState, k.log = "LoadState=not-found\n", upgradeLog
if err := again.Follow(); err != nil {
t.Fatal(err)
}
if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" {
t.Errorf("followed to its end: %s", steps(r))
}
}
func testAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps(t *testing.T) {
// Stopped after the gate, before the transaction started: nothing started, so the run is refused.
k := newKeeperBed(t)
stopped := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk,
Steps: []Step{{Step: StepRecorded, At: k.now}, {Step: StepCheckedBefore, At: k.now}}}
if _, err := k.runs.Create(NodeKey("node-a", stopped.Run), stopped); err != nil {
t.Fatal(err)
}
if err := k.u.Follow(); err != nil {
t.Fatal(err)
}
if r := k.run(t); steps(r) != "recorded checked-before refused" || !strings.Contains(r.Steps[2].Why, "nothing was started") {
t.Errorf("%s %+v", steps(r), r.Steps)
}
// Stopped after the transaction started and before it was written: its log says it ran.
k = newKeeperBed(t)
if _, err := k.runs.Create(NodeKey("node-a", stopped.Run), stopped); err != nil {
t.Fatal(err)
}
k.log = upgradeLog
if err := k.u.Follow(); err != nil {
t.Fatal(err)
}
if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" || r.Steps[2].Unit != "mesh-pacman-upgrade-20261004t115900z" {
t.Errorf("%s %+v", steps(r), r.Steps)
}
// Stopped after the transaction ended and before it was checked.
k = newKeeperBed(t)
zero := 0
unchecked := Run{Node: "node-a", Run: "20261004T115900Z", Kind: RunUpgrade, From: FromAsk,
Steps: []Step{{Step: StepRecorded, At: k.now}, {Step: StepCheckedBefore, At: k.now}, {Step: StepStarted, At: k.now}, {Step: StepFinished, At: k.now, Exit: &zero}}}
if _, err := k.runs.Create(NodeKey("node-a", unchecked.Run), unchecked); err != nil {
t.Fatal(err)
}
if err := k.u.Follow(); err != nil {
t.Fatal(err)
}
if r := k.run(t); steps(r) != "recorded checked-before started finished checked-after" {
t.Errorf("%s", steps(r))
}
}
func testPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing(t *testing.T) {
asks := newBucket()
savedState, savedNode, savedFetch := upgradeState, thisNode, fetch
upgradeState = func(name string) Bucket {
if name != StateAsks {
t.Fatalf("wrote %s", name)
}
return asks
}
thisNode = func() string { return "node-a" }
fetch = func(string) ([]byte, error) { return nil, errors.New("offline") }
defer func() { upgradeState, thisNode, fetch = savedState, savedNode, savedFetch }()
var calls []call
m := machine(fake(func(c call) Ran { return Ran{} }, &calls), 1000)
for _, tool := range tools(m) {
if tool.Name != "pacman_upgrade" {
continue
}
out, err := tool.Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
a := out.(map[string]any)["asked"].(Ask)
if a.Node != "node-a" || a.Act != ActUpgrade || !strings.HasPrefix(a.ID, "20261004T120000Z-") {
t.Errorf("ask: %+v", a)
}
if keys, _ := asks.Keys(); len(keys) != 1 || keys[0] != NodeKey("node-a", a.ID) {
t.Errorf("asks: %v", keys)
}
for _, c := range calls {
if c.name == "sudo" || c.name == "systemd-run" {
t.Errorf("the tool acted itself: %s", c)
}
}
if _, err := tool.Run(map[string]any{"node": "node-b"}); err != nil {
t.Fatal(err)
}
if keys, _ := asks.Keys(); len(keys) != 2 || !strings.HasPrefix(keys[1], "node-b.") {
t.Errorf("another node's ask: %v", keys)
}
return
}
t.Fatal("no pacman_upgrade")
}
// TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep runs this file's cases in order as one test, each a row: the check counts every top-level test against the
// module's budget of 50 (design 47 H2 §819). A failing case is named by the line logged before it.
func TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep(t *testing.T) {
for _, c := range []struct {
name string
run func(*testing.T)
}{
{"TestAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep", testAnAskedUpgradeIsARunWrittenAndEmittedAtEachStep},
{"TestAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked", testAFailedTransactionIsWrittenWithItsExitAndLastErrorAndChecked},
{"TestATransactionThatCannotStartFailsAndIsChecked", testATransactionThatCannotStartFailsAndIsChecked},
{"TestAnAskIsTakenByItsNodesKeeperAlone", testAnAskIsTakenByItsNodesKeeperAlone},
{"TestAnAskARunAlreadyNamesIsDeletedAndNotRunAgain", testAnAskARunAlreadyNamesIsDeletedAndNotRunAgain},
{"TestARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn", testARunDoesNotStartWhileTheDatabaseIsLockedOrAnotherRunGoesOn},
{"TestAKeeperStartedAgainFollowsItsRunToItsEnd", testAKeeperStartedAgainFollowsItsRunToItsEnd},
{"TestAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps", testAKeeperStartedAgainTakesUpARunStoppedBetweenItsSteps},
{"TestPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing", testPacmanUpgradeWritesAnAskForTheNodeAndStartsNothing},
} {
t.Logf("case %s", c.name)
c.run(t)
}
}