mesh-catalog/modules/pacman/cmd/pacman-tools/keeper_test.go

package main

// The keeper's start and its lock watch (novox/hq ADR 0507 §2, §3; acceptance 1840/AC1-AC3), over a fake state,
// a fake machine, a fake clock and a fake watch.

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
)

// fakeStore is one state in memory, recording every write.
type fakeStore struct {
	mu       sync.Mutex
	name     string
	values   map[string]json.RawMessage
	revs     map[string]uint64
	at       map[string]time.Time
	rev      uint64
	writes   []string // "put <key>", "delete <key>"
	watchers []fakeWatch
	now      func() time.Time
}

type fakeWatch struct {
	key    string
	handle func(stdio.StateChange) error
}

func newFakeStore(name string) *fakeStore {
	return &fakeStore{name: name, values: map[string]json.RawMessage{}, revs: map[string]uint64{}, at: map[string]time.Time{}}
}

func (s *fakeStore) Get(key string) (*stdio.StateEntry, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	v, ok := s.values[key]
	if !ok {
		return nil, nil
	}
	return &stdio.StateEntry{Key: key, Value: v, Revision: s.revs[key]}, nil
}

func (s *fakeStore) write(key string, value any, op string) (uint64, error) {
	data, err := json.Marshal(value)
	if err != nil {
		return 0, err
	}
	s.mu.Lock()
	s.rev++
	s.values[key], s.revs[key], s.at[key] = data, s.rev, s.clock()
	s.writes = append(s.writes, op+" "+key)
	rev, ws := s.rev, append([]fakeWatch(nil), s.watchers...)
	s.mu.Unlock()
	for _, w := range ws {
		if matchKey(w.key, key) {
			w.handle(stdio.StateChange{State: s.name, Key: key, Op: "put", Value: data, Revision: rev})
		}
	}
	return rev, nil
}

// clock is when a write lands: the store's own time, which a test sets.
func (s *fakeStore) clock() time.Time {
	if s.now != nil {
		return s.now()
	}
	return t0
}

func (s *fakeStore) Put(key string, value any) (uint64, error) { return s.write(key, value, "put") }

func (s *fakeStore) Create(key string, value any) (uint64, error) {
	if e, _ := s.Get(key); e != nil {
		return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
	}
	return s.write(key, value, "create")
}

func (s *fakeStore) Update(key string, value any, revision uint64) (uint64, error) {
	if e, _ := s.Get(key); e == nil || e.Revision != revision {
		return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
	}
	return s.write(key, value, "update")
}

func (s *fakeStore) Delete(key string) error {
	s.mu.Lock()
	delete(s.values, key)
	delete(s.revs, key)
	s.rev++
	s.writes = append(s.writes, "delete "+key)
	rev, ws := s.rev, append([]fakeWatch(nil), s.watchers...)
	s.mu.Unlock()
	for _, w := range ws {
		if matchKey(w.key, key) {
			w.handle(stdio.StateChange{State: s.name, Key: key, Op: "delete", Revision: rev})
		}
	}
	return nil
}

func (s *fakeStore) Keys() ([]string, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	keys := []string{}
	for k := range s.values {
		keys = append(keys, k)
	}
	sortStrings(keys)
	return keys, nil
}

func (s *fakeStore) Watch(key string, handle func(stdio.StateChange) error) error {
	keys, _ := s.Keys()
	for _, k := range keys {
		if !matchKey(key, k) {
			continue
		}
		e, _ := s.Get(k)
		if err := handle(stdio.StateChange{State: s.name, Key: k, Op: "put", Value: e.Value, Revision: e.Revision, Current: true}); err != nil {
			return err
		}
	}
	s.mu.Lock()
	s.watchers = append(s.watchers, fakeWatch{key, handle})
	s.mu.Unlock()
	return nil
}

// Writes is what was written, in order, and forgets it.
func (s *fakeStore) Writes() []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	w := s.writes
	s.writes = nil
	return w
}

// matchKey narrows as the bus does: `*` one name, `**` the rest, empty every key.
func matchKey(pattern, key string) bool {
	if pattern == "" {
		return true
	}
	p, k := strings.Split(pattern, "."), strings.Split(key, ".")
	for i := range p {
		if p[i] == "**" {
			return len(k) > i
		}
		if i >= len(k) || (p[i] != "*" && p[i] != k[i]) {
			return false
		}
	}
	return len(p) == len(k)
}

func sortStrings(s []string) {
	for i := 1; i < len(s); i++ {
		for j := i; j > 0 && s[j] < s[j-1]; j-- {
			s[j], s[j-1] = s[j-1], s[j]
		}
	}
}

// fakeStates is the module's states in memory.
type fakeStates struct {
	mu     sync.Mutex
	states map[string]*fakeStore
}

func (f *fakeStates) State(name string) Store { return f.store(name) }

func (f *fakeStates) store(name string) *fakeStore {
	f.mu.Lock()
	defer f.mu.Unlock()
	if f.states == nil {
		f.states = map[string]*fakeStore{}
	}
	if f.states[name] == nil {
		f.states[name] = newFakeStore(name)
	}
	return f.states[name]
}

// emitted is the events a keeper emitted.
type emitted struct {
	mu     sync.Mutex
	events []emittedEvent
}

type emittedEvent struct {
	name string
	body any
	id   string
}

func (e *emitted) Emit(name string, body any, id string) error {
	e.mu.Lock()
	defer e.mu.Unlock()
	e.events = append(e.events, emittedEvent{name, body, id})
	return nil
}

func (e *emitted) Take() []emittedEvent {
	e.mu.Lock()
	defer e.mu.Unlock()
	out := e.events
	e.events = nil
	return out
}

// fakeClock moves only when the test moves it; After fires once the clock passes it.
type fakeClock struct {
	mu     sync.Mutex
	now    time.Time
	timers []fakeTimer
}

type fakeTimer struct {
	at time.Time
	c  chan time.Time
}

func (c *fakeClock) Now() time.Time {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.now
}

func (c *fakeClock) After(d time.Duration) <-chan time.Time {
	c.mu.Lock()
	defer c.mu.Unlock()
	t := fakeTimer{c.now.Add(d), make(chan time.Time, 1)}
	if d <= 0 {
		t.c <- c.now
		return t.c
	}
	c.timers = append(c.timers, t)
	return t.c
}

func (c *fakeClock) Advance(d time.Duration) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.now = c.now.Add(d)
	kept := c.timers[:0]
	for _, t := range c.timers {
		if !t.at.After(c.now) {
			t.c <- c.now
			continue
		}
		kept = append(kept, t)
	}
	c.timers = kept
}

// fakeWatcher hands the test's signals to the keeper.
type fakeWatcher struct{ c chan Signal }

func (w fakeWatcher) Signals() <-chan Signal { return w.c }
func (w fakeWatcher) Close() error           { return nil }

// pacmanDB is a node's package database as pacman answers it: the test changes it between reads.
type pacmanDB struct {
	mu       sync.Mutex
	q, qe    string
	sl       string
	checkup  Ran
	commands []string
}

func (d *pacmanDB) set(q, qe string) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.q, d.qe = q, qe
}

func (d *pacmanDB) setUpdates(r Ran) {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.checkup = r
}

func (d *pacmanDB) runner() Runner {
	return func(_ context.Context, name string, args ...string) Ran {
		d.mu.Lock()
		defer d.mu.Unlock()
		line := strings.TrimSpace(name + " " + strings.Join(args, " "))
		d.commands = append(d.commands, line)
		switch line {
		case "pacman -Q":
			return Ran{Stdout: d.q}
		case "pacman -Qeq":
			return Ran{Stdout: d.qe}
		case "pacman -Sl":
			return Ran{Stdout: d.sl}
		case "checkupdates":
			return d.checkup
		}
		return Ran{Status: 99, Stderr: "unexpected command: " + line}
	}
}

var syncDBs = "core glibc 2.42-1 [installed]\ncore linux 6.17.1-1 [installed]\nextra zsh 5.9-2 [installed: 5.9-1]\n" +
	"extra zsh-completions 0.35-1\nextra tmux 3.5-1\nextra-testing zsh 5.9-3\n"

var t0 = time.Date(2026, 10, 24, 4, 0, 0, 0, time.UTC)

func newTestKeeper(db *pacmanDB) (*Keeper, *fakeStates, *emitted, *fakeClock, fakeWatcher) {
	states, ev, clock := &fakeStates{}, &emitted{}, &fakeClock{now: t0}
	w := fakeWatcher{make(chan Signal)}
	m := machine(db.runner(), 1000)
	k := &Keeper{Node: "node-a", Machine: m, State: states.State, Emit: ev.Emit, Clock: clock, Watcher: w,
		Logf: func(string, ...any) {}}
	return k, states, ev, clock, w
}

func installedOf(t *testing.T, s *fakeStore, key string) InstalledValue {
	t.Helper()
	e, _ := s.Get(key)
	if e == nil {
		t.Fatalf("no %s", key)
	}
	var v InstalledValue
	if err := json.Unmarshal(e.Value, &v); err != nil {
		t.Fatal(err)
	}
	return v
}

// 1840/AC1: at the start every package is written with its version, its reason and its repository or foreign,
// and written again only when it differs from the key.
func TestAtItsStartTheKeeperWritesEveryPackageOnlyWhenItDiffersFromTheKey(t *testing.T) {
	db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
	db.set("glibc 2.42-1\nlinux 6.17.1-1\nyay 12.0-1\nzsh 5.9-1\n", "linux\nyay\nzsh\n")
	k, states, ev, _, _ := newTestKeeper(db)
	k.DBChanged = func() time.Time { return t0.Add(-time.Hour) }
	k.refreshInstalled(k.DBChanged())
	inst := states.store(StateInstalled)
	if w := inst.Writes(); strings.Join(w, ",") != "put node-a" {
		t.Fatalf("writes: %v", w)
	}
	v := installedOf(t, inst, "node-a")
	want := []InstalledPackage{
		{"glibc", "2.42-1", false, "core"}, {"linux", "6.17.1-1", true, "core"},
		{"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", true, "extra"},
	}
	if !samePackages(v.Packages, want) || !v.At.Equal(t0.Add(-time.Hour)) {
		t.Fatalf("installed.node-a: %+v", v)
	}
	if e := ev.Take(); len(e) != 1 || len(e[0].body.(PackageChanged).Added) != 4 {
		t.Fatalf("the first write adds every package: %+v", e)
	}
	// a second start over the same database writes nothing and says nothing, whatever the time.
	k.refreshInstalled(t0)
	if w := inst.Writes(); len(w) != 0 {
		t.Errorf("the same packages were written again: %v", w)
	}
	if e := ev.Take(); len(e) != 0 {
		t.Errorf("the same packages were said again: %+v", e)
	}
}

// 1840/AC2: a transaction that adds, removes and changes packages is one package.changed naming the node and
// exactly those; one that changes nothing writes and emits nothing; a lock held again waits for its release.
func TestAReleasedLockSaysExactlyWhatTheTransactionChangedOnce(t *testing.T) {
	db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
	db.set("glibc 2.42-1\nlinux 6.17.1-1\nzsh 5.9-1\n", "linux\nzsh\n")
	k, states, ev, clock, _ := newTestKeeper(db)
	k.refreshInstalled(t0)
	ev.Take()
	states.store(StateInstalled).Writes()

	held := false
	k.Locked = func() bool { return held }
	db.set("glibc 2.42-1\nlinux 6.17.2-1\ntmux 3.5-1\n", "linux\ntmux\n")
	held = true
	k.signal(LockReleased)
	if w := states.store(StateInstalled).Writes(); len(w) != 0 {
		t.Fatalf("read while the lock was held again: %v", w)
	}
	held = false
	clock.Advance(time.Minute)
	k.signal(LockReleased)
	if w := states.store(StateInstalled).Writes(); strings.Join(w, ",") != "put node-a" {
		t.Fatalf("writes: %v", w)
	}
	e := ev.Take()
	if len(e) != 1 || e[0].name != EventPackageChanged {
		t.Fatalf("events: %+v", e)
	}
	got := e[0].body.(PackageChanged)
	if got.Node != "node-a" || !got.At.Equal(t0.Add(time.Minute)) ||
		fmt.Sprint(got.Added) != "[{tmux 3.5-1}]" || fmt.Sprint(got.Removed) != "[{zsh 5.9-1}]" ||
		fmt.Sprint(got.Changed) != "[{linux 6.17.1-1 6.17.2-1}]" {
		t.Fatalf("package.changed: %+v", got)
	}
	if !strings.HasPrefix(e[0].id, "installed.node-a.") {
		t.Errorf("the event's id is the key's revision, so a second emit is dropped: %q", e[0].id)
	}
	// a transaction that changed nothing (a database sync alone).
	k.signal(LockReleased)
	if w := states.store(StateInstalled).Writes(); len(w) != 0 {
		t.Errorf("nothing changed and the key was written: %v", w)
	}
	if e := ev.Take(); len(e) != 0 {
		t.Errorf("nothing changed and it was said: %+v", e)
	}
	// a change of reason alone rewrites the key and is no package.changed.
	db.set("glibc 2.42-1\nlinux 6.17.2-1\ntmux 3.5-1\n", "glibc\nlinux\ntmux\n")
	k.signal(LockReleased)
	if w := states.store(StateInstalled).Writes(); len(w) != 1 {
		t.Errorf("a changed reason is a changed key: %v", w)
	}
	if e := ev.Take(); len(e) != 0 {
		t.Errorf("a changed reason is no change of a version: %+v", e)
	}
}

// bigDB is a package database whose value is over the bus's cap.
func bigDB(n int) string {
	var b strings.Builder
	for i := 0; i < n; i++ {
		fmt.Fprintf(&b, "package-with-a-long-name-%05d 1.0.0-1\n", i)
	}
	return b.String()
}

// 1840/AC3: a value over the bus's cap is not written: the key keeps its last value and `<node>.not-current`
// holds the size and since when, written once; the next write that fits deletes it.
func TestAValueOverTheCapKeepsTheLastAndMarksTheNodeNotCurrent(t *testing.T) {
	db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
	db.set("glibc 2.42-1\n", "")
	k, states, ev, clock, _ := newTestKeeper(db)
	k.refreshInstalled(t0)
	inst := states.store(StateInstalled)
	inst.Writes()
	ev.Take()

	db.set(bigDB(5000), "")
	clock.Advance(time.Minute)
	k.signal(LockReleased)
	if w := inst.Writes(); strings.Join(w, ",") != "put node-a.not-current" {
		t.Fatalf("writes: %v", w)
	}
	if v := installedOf(t, inst, "node-a"); len(v.Packages) != 1 {
		t.Fatalf("the last value is kept: %d packages", len(v.Packages))
	}
	if e := ev.Take(); len(e) != 0 {
		t.Errorf("nothing was written and it was said: %+v", e)
	}
	e, _ := inst.Get("node-a.not-current")
	var nc NotCurrent
	json.Unmarshal(e.Value, &nc)
	if nc.Bytes <= ValueCap || !nc.Since.Equal(t0.Add(time.Minute)) {
		t.Fatalf("not-current: %+v", nc)
	}
	// still over: the mark is written once.
	db.set(bigDB(5001), "")
	clock.Advance(time.Minute)
	k.signal(LockReleased)
	if w := inst.Writes(); len(w) != 0 {
		t.Errorf("the mark was written again: %v", w)
	}
	// back under the cap: written, and the mark deleted.
	db.set("glibc 2.42-1\nzsh 5.9-1\n", "")
	k.signal(LockReleased)
	if w := inst.Writes(); strings.Join(w, ",") != "put node-a,delete node-a.not-current" {
		t.Errorf("writes: %v", w)
	}
}

// The mark goes too when the node's packages come back to what the key holds: the key is current again.
func TestTheMarkGoesWhenThePackagesComeBackToTheKey(t *testing.T) {
	db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
	db.set("glibc 2.42-1\n", "")
	k, states, _, _, _ := newTestKeeper(db)
	k.refreshInstalled(t0)
	db.set(bigDB(5000), "")
	k.signal(LockReleased)
	inst := states.store(StateInstalled)
	inst.Writes()
	db.set("glibc 2.42-1\n", "")
	k.signal(LockReleased)
	if w := inst.Writes(); strings.Join(w, ",") != "delete node-a.not-current" {
		t.Errorf("writes: %v", w)
	}
}

// The loop: a released lock is read, a hook runs after it with what it changed, a watched state's change reaches
// its hook, and a wake comes at the time it asked.
func TestTheLoopRunsTheHooksOnItsOneGoroutine(t *testing.T) {
	db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
	db.set("glibc 2.42-1\n", "")
	k, states, _, clock, w := newTestKeeper(db)
	var mu sync.Mutex
	var seen []string
	say := func(s string) { mu.Lock(); seen = append(seen, s); mu.Unlock() }
	k.Hooks = Hooks{
		Start: []func(*Keeper) error{func(*Keeper) error { say("start"); return nil }},
		AfterTransaction: []func(*Keeper, Transaction) error{func(_ *Keeper, tr Transaction) error {
			say(fmt.Sprintf("transaction changed=%v", tr.Changed != nil))
			return nil
		}},
		Changes: []StateHook{{State: StateAsks, Key: "node-a.*", Handle: func(_ *Keeper, c stdio.StateChange) error {
			say(fmt.Sprintf("%s %s current=%v", c.Op, c.Key, c.Current))
			return nil
		}}},
		Wakes: []Wake{func(_ *Keeper, now time.Time) time.Time {
			if now.Equal(t0) {
				return t0.Add(time.Hour)
			}
			say("wake " + now.Sub(t0).String())
			return time.Time{}
		}},
	}
	states.store(StateAsks).Put("node-a.ask-1", Ask{ID: "ask-1"})
	states.store(StateAsks).Put("node-b.ask-2", Ask{ID: "ask-2"})
	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan error, 1)
	go func() { done <- k.Run(ctx) }()
	waitFor(t, &mu, &seen, "start")
	db.set("glibc 2.42-1\nzsh 5.9-1\n", "")
	w.c <- LockReleased
	waitFor(t, &mu, &seen, "transaction changed=true")
	states.store(StateAsks).Put("node-a.ask-3", Ask{ID: "ask-3"})
	waitFor(t, &mu, &seen, "put node-a.ask-3 current=false")
	clock.Advance(time.Hour)
	waitFor(t, &mu, &seen, "wake 1h0m0s")
	cancel()
	if err := <-done; err != context.Canceled {
		t.Errorf("Run ended with %v", err)
	}
	mu.Lock()
	defer mu.Unlock()
	if strings.Join(seen, "|") != "put node-a.ask-1 current=true|start|transaction changed=true|put node-a.ask-3 current=false|wake 1h0m0s" &&
		strings.Join(seen, "|") != "start|put node-a.ask-1 current=true|transaction changed=true|put node-a.ask-3 current=false|wake 1h0m0s" {
		t.Errorf("seen: %v", seen)
	}
}

func waitFor(t *testing.T, mu *sync.Mutex, seen *[]string, want string) {
	t.Helper()
	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
		mu.Lock()
		for _, s := range *seen {
			if s == want {
				mu.Unlock()
				return
			}
		}
		mu.Unlock()
	}
	t.Fatalf("never saw %q", want)
}

// The watch ending ends the keeper, loudly.
func TestTheKeeperStopsWhenTheWatchEnds(t *testing.T) {
	db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
	k, _, _, _, w := newTestKeeper(db)
	close(w.c)
	if err := k.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "watch ended") {
		t.Errorf("Run: %v", err)
	}
}

// The machine's watch: the lock deleted and the mirror list written or moved in are each a signal; anything else
// in those directories is none.
func TestTheWatchSeesTheLockReleasedAndTheMirrorListRewritten(t *testing.T) {
	lockDir, mirrorDir := t.TempDir(), t.TempDir()
	w, err := WatchMachine(lockDir, LockName, mirrorDir, MirrorName)
	if err != nil {
		t.Fatal(err)
	}
	next := func() Signal {
		select {
		case s := <-w.Signals():
			return s
		case <-time.After(200 * time.Millisecond):
			return 0
		}
	}
	lock := filepath.Join(lockDir, LockName)
	os.WriteFile(lock, nil, 0o644)
	os.WriteFile(filepath.Join(lockDir, "other"), nil, 0o644)
	os.Remove(filepath.Join(lockDir, "other"))
	if s := next(); s != 0 {
		t.Fatalf("a lock taken, and another file, are no signal: %v", s)
	}
	os.Remove(lock)
	if s := next(); s != LockReleased {
		t.Fatalf("the lock deleted: %v", s)
	}
	os.WriteFile(filepath.Join(mirrorDir, MirrorName), []byte("Server = x\n"), 0o644)
	if s := next(); s != MirrorsRewritten {
		t.Fatalf("the list written: %v", s)
	}
	tmp := filepath.Join(mirrorDir, "mirrorlist.new")
	os.WriteFile(tmp, []byte("Server = y\n"), 0o644)
	if s := next(); s != 0 {
		t.Fatalf("another file written: %v", s)
	}
	os.Rename(tmp, filepath.Join(mirrorDir, MirrorName))
	if s := next(); s != MirrorsRewritten {
		t.Fatalf("the list moved in: %v", s)
	}
	w.Close()
	for deadline := time.After(2 * time.Second); ; {
		select {
		case _, open := <-w.Signals():
			if !open {
				return
			}
		case <-deadline:
			t.Fatal("closing the watch did not end its signals")
		}
	}
}