mesh-catalog/modules/pacman/cmd/pacman-tools/round_integration_test.go

//go:build integration

package main

// The round over a real bus (novox/hq ADR 0508 §5, §6; acceptance 1841/AC15): the keepers of two made-up nodes,
// each whole (its own bus account, its states, its watches) against a JetStream server of the test's own, with
// fake machines and a fake root helper each. The first node of the order opens the week's round and runs; the
// second starts only once the first's after-upgrade check passed; a failed check stops the round there.
// Compiled by the merge gate's `go vet -tags integration`; run by the merge check, never by hand.

import (
	"context"
	"encoding/json"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/process"
	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"
)

// countingRoot is a node's root helper: it counts the transactions it was asked to start.
type countingRoot struct {
	mu      sync.Mutex
	started []string
	mirrors []string
}

func (r *countingRoot) Transaction(kind, run string, ignore []string) error {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.started = append(r.started, kind+" "+run)
	return nil
}
func (r *countingRoot) Rollback(string, []RollbackItem) error { return nil }
func (r *countingRoot) Journal(string, int) ([]string, error) { return nil, nil }
func (r *countingRoot) Maps() ([]Mapped, error)               { return []Mapped{}, nil }

// Mirrorlist keeps the mirror list written last (ADR 0585 §3).
func (r *countingRoot) Mirrorlist(servers []string) error {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.mirrors = append([]string(nil), servers...)
	return nil
}

func (r *countingRoot) mirrorlist() []string {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.mirrors
}
func (r *countingRoot) count() int {
	r.mu.Lock()
	defer r.mu.Unlock()
	return len(r.started)
}

// roundNode is one made-up node's keeper on the test's bus.
type roundNode struct {
	root    *countingRoot
	signals chan Signal
}

// startRoundNode starts node's keeper; its machine's clock begins at the Saturday's opening, its transaction
// unit is gone (ended well) whenever asked, and its after-upgrade check passes unless passes says otherwise.
func startRoundNode(t *testing.T, ctx context.Context, url, node string, order []string, passes bool) *roundNode {
	t.Helper()
	db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
	db.set("glibc 2.42-1\nzsh 5.9-1\n", "zsh\n")
	base := db.runner()
	run := func(c context.Context, name string, args ...string) Ran {
		line := strings.TrimSpace(name + " " + strings.Join(args, " "))
		switch {
		case strings.HasPrefix(line, "systemctl list-units --state=failed"), line == "dkms status":
			return Ran{}
		case strings.HasPrefix(line, "systemctl show "+unitPrefix):
			return Ran{Stdout: "LoadState=not-found\n"}
		}
		return base(c, name, args...)
	}
	m := machine(run, 1000)
	began := time.Now()
	m.Now = func() time.Time { return t0.Add(time.Since(began)) }
	m.ReadFile = func(path string) ([]byte, error) {
		switch path {
		case OSRelease:
			return []byte("7.2.8-arch1-2\n"), nil
		case ReflectorConf:
			return []byte("--sort rate\n"), nil
		}
		return nil, errNoFile
	}
	n := &roundNode{root: &countingRoot{}, signals: make(chan Signal)}
	k := &Keeper{Config: KeeperConfig{UpgradeOrder: order, RecordDays: 180, RoundStartMinutes: 120}, Machine: m, Clock: realClock{},
		Watcher: fakeWatcher{n.signals}, Logf: func(f string, a ...any) { t.Logf(node+": "+f, a...) }, MakeHooks: keeperHooks,
		Root: n.root, Check: func(Run, *Record) Check { return Check{Passed: passes} },
		// The mesh's mirrors, ranked by the node that opens the round (ADR 0585 §2): reflector stood in for.
		Rank: func(_ []byte, round string) (MirrorList, error) {
			return MirrorList{Servers: roundMirrors, RankedAt: t0, RankedBy: node, Round: round}, nil
		}}
	account := process.Account{URL: url, Node: node, Module: "pacman", User: node + ".pacman.keeper", Password: "pw"}
	go func() { _ = runKeeper(ctx, account, k) }()
	return n
}

// roundMirrors is the mirror list the round's opener ranks.
var roundMirrors = []string{"https://nl.mirror.example/archlinux/$repo/os/$arch", "https://de.mirror.example/$repo/os/$arch"}

func roundBus(t *testing.T) (string, map[string]jetstream.KeyValue) {
	t.Helper()
	url := newTestBus(t)
	nc, err := nats.Connect(url)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(nc.Close)
	js, err := jetstream.New(nc)
	if err != nil {
		t.Fatal(err)
	}
	ctx := context.Background()
	buckets := map[string]jetstream.KeyValue{}
	for _, name := range States {
		kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "pacman_" + name, History: 8})
		if err != nil {
			t.Fatal(err)
		}
		buckets[name] = kv
	}
	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "pacman-events", Subjects: []string{"mesh.mod.pacman.event.>"}}); err != nil {
		t.Fatal(err)
	}
	return url, buckets
}

func eventually(t *testing.T, what string, within time.Duration, ok func() bool) {
	t.Helper()
	for deadline := time.Now().Add(within); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
		if ok() {
			return
		}
	}
	t.Fatalf("never: %s", what)
}

func roundNodeOnBus(kv jetstream.KeyValue, round, node string) RoundNode {
	var rn RoundNode
	if e, err := kv.Get(context.Background(), RoundNodeKey(round, node)); err == nil {
		_ = json.Unmarshal(e.Value(), &rn)
	}
	return rn
}

func TestTheRoundGoesNodeByNodeOnARealBus(t *testing.T) {
	url, buckets := roundBus(t)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	order := []string{"node-a", "node-b"}
	a := startRoundNode(t, ctx, url, "node-a", order, true)
	b := startRoundNode(t, ctx, url, "node-b", order, true)
	round := RoundID(t0)

	eventually(t, "node-a opened the round and started its run", 20*time.Second, func() bool {
		return a.root.count() == 1 && roundNodeOnBus(buckets[StateRounds], round, "node-a").Run != ""
	})
	time.Sleep(time.Second)
	if b.root.count() != 0 {
		t.Fatalf("node-b started before node-a passed: %v", b.root.started)
	}
	// node-a's transaction ends: its lock is released, its run is finished and checked.
	a.signals <- LockReleased
	eventually(t, "node-b started once node-a passed", 20*time.Second, func() bool {
		return b.root.count() == 1 && roundNodeOnBus(buckets[StateRounds], round, "node-b").Run != ""
	})
	if a.root.count() != 1 {
		t.Errorf("node-a ran again: %v", a.root.started)
	}
	// ADR 0585 §2, §3: the round opened naming the revision of the list node-a ranked in the write that opened it, and
	// both nodes wrote that list whole to their mirror list file, from the bus.
	e, err := buckets[StateRounds].Get(context.Background(), round)
	var opened Round
	if err == nil {
		_ = json.Unmarshal(e.Value(), &opened)
	}
	l, lerr := buckets[StateMirrors].Get(context.Background(), MirrorsKey)
	if err != nil || lerr != nil || opened.Mirrors == 0 || opened.Mirrors != l.Revision() || !strings.Contains(string(l.Value()), `"round":"`+round+`"`) {
		t.Errorf("the round opened with mirrors %d (%q), the list %v (%v, %v)", opened.Mirrors, opened.MirrorsNote, l, err, lerr)
	}
	for _, n := range []*roundNode{a, b} {
		eventually(t, "each node wrote the mesh's mirror list", 10*time.Second, func() bool {
			return strings.Join(n.root.mirrorlist(), " ") == strings.Join(roundMirrors, " ")
		})
	}
}

func TestAFailedCheckStopsTheRoundOnARealBus(t *testing.T) {
	url, buckets := roundBus(t)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	order := []string{"node-a", "node-b"}
	a := startRoundNode(t, ctx, url, "node-a", order, false)
	b := startRoundNode(t, ctx, url, "node-b", order, true)
	round := RoundID(t0)
	eventually(t, "node-a started its run", 20*time.Second, func() bool { return a.root.count() == 1 })
	a.signals <- LockReleased
	runA := ""
	eventually(t, "node-a's run checked", 20*time.Second, func() bool {
		runA = roundNodeOnBus(buckets[StateRounds], round, "node-a").Run
		if runA == "" {
			return false
		}
		e, err := buckets[StateRuns].Get(context.Background(), NodeKey("node-a", runA))
		return err == nil && strings.Contains(string(e.Value()), `"checked-after"`)
	})
	time.Sleep(2 * time.Second)
	if b.root.count() != 0 || roundNodeOnBus(buckets[StateRounds], round, "node-b").Run != "" {
		t.Fatalf("node-b started after node-a's check failed: %v", b.root.started)
	}
}