181 lines
6.4 KiB
Go
181 lines
6.4 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
// The round over a real bus (novox/hq ADR 0508 §5, §6; acceptance 1841/AC15): the keepers of two made-up nodes,
|
|
// each whole (its own bus account, its states, its watches) against a JetStream server of the test's own, with
|
|
// fake machines and a fake root helper each. The first node of the order opens the week's round and runs; the
|
|
// second starts only once the first's after-upgrade check passed; a failed check stops the round there.
|
|
// Compiled by the merge gate's `go vet -tags integration`; run by the merge check, never by hand.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// countingRoot is a node's root helper: it counts the transactions it was asked to start.
|
|
type countingRoot struct {
|
|
mu sync.Mutex
|
|
started []string
|
|
}
|
|
|
|
func (r *countingRoot) Transaction(kind, run string, ignore []string) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.started = append(r.started, kind+" "+run)
|
|
return nil
|
|
}
|
|
func (r *countingRoot) Rollback(string, []RollbackItem) error { return nil }
|
|
func (r *countingRoot) Journal(string, int) ([]string, error) { return nil, nil }
|
|
func (r *countingRoot) Maps() ([]Mapped, error) { return []Mapped{}, nil }
|
|
func (r *countingRoot) count() int {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return len(r.started)
|
|
}
|
|
|
|
// roundNode is one made-up node's keeper on the test's bus.
|
|
type roundNode struct {
|
|
root *countingRoot
|
|
signals chan Signal
|
|
}
|
|
|
|
// startRoundNode starts node's keeper; its machine's clock begins at the Saturday's opening, its transaction
|
|
// unit is gone (ended well) whenever asked, and its after-upgrade check passes unless passes says otherwise.
|
|
func startRoundNode(t *testing.T, ctx context.Context, url, node string, order []string, passes bool) *roundNode {
|
|
t.Helper()
|
|
db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
|
|
db.set("glibc 2.42-1\nzsh 5.9-1\n", "zsh\n")
|
|
base := db.runner()
|
|
run := func(c context.Context, name string, args ...string) Ran {
|
|
line := strings.TrimSpace(name + " " + strings.Join(args, " "))
|
|
switch {
|
|
case strings.HasPrefix(line, "systemctl list-units --state=failed"), line == "dkms status":
|
|
return Ran{}
|
|
case strings.HasPrefix(line, "systemctl show "+unitPrefix):
|
|
return Ran{Stdout: "LoadState=not-found\n"}
|
|
}
|
|
return base(c, name, args...)
|
|
}
|
|
m := machine(run, 1000)
|
|
began := time.Now()
|
|
m.Now = func() time.Time { return t0.Add(time.Since(began)) }
|
|
m.ReadFile = func(path string) ([]byte, error) {
|
|
if path == OSRelease {
|
|
return []byte("7.2.8-arch1-2\n"), nil
|
|
}
|
|
return nil, errNoFile
|
|
}
|
|
n := &roundNode{root: &countingRoot{}, signals: make(chan Signal)}
|
|
k := &Keeper{Config: KeeperConfig{UpgradeOrder: order, RecordDays: 180, RoundStartMinutes: 120}, Machine: m, Clock: realClock{},
|
|
Watcher: fakeWatcher{n.signals}, Logf: func(f string, a ...any) { t.Logf(node+": "+f, a...) }, MakeHooks: keeperHooks,
|
|
Root: n.root, Check: func(Run, *Record) Check { return Check{Passed: passes} }}
|
|
account := process.Account{URL: url, Node: node, Module: "pacman", User: node + ".pacman.keeper", Password: "pw"}
|
|
go func() { _ = runKeeper(ctx, account, k) }()
|
|
return n
|
|
}
|
|
|
|
func roundBus(t *testing.T) (string, map[string]jetstream.KeyValue) {
|
|
t.Helper()
|
|
url := newTestBus(t)
|
|
nc, err := nats.Connect(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
buckets := map[string]jetstream.KeyValue{}
|
|
for _, name := range States {
|
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "pacman_" + name, History: 8})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
buckets[name] = kv
|
|
}
|
|
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "pacman-events", Subjects: []string{"mesh.mod.pacman.event.>"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return url, buckets
|
|
}
|
|
|
|
func eventually(t *testing.T, what string, within time.Duration, ok func() bool) {
|
|
t.Helper()
|
|
for deadline := time.Now().Add(within); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
|
if ok() {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("never: %s", what)
|
|
}
|
|
|
|
func roundNodeOnBus(kv jetstream.KeyValue, round, node string) RoundNode {
|
|
var rn RoundNode
|
|
if e, err := kv.Get(context.Background(), RoundNodeKey(round, node)); err == nil {
|
|
_ = json.Unmarshal(e.Value(), &rn)
|
|
}
|
|
return rn
|
|
}
|
|
|
|
func TestTheRoundGoesNodeByNodeOnARealBus(t *testing.T) {
|
|
url, buckets := roundBus(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
order := []string{"node-a", "node-b"}
|
|
a := startRoundNode(t, ctx, url, "node-a", order, true)
|
|
b := startRoundNode(t, ctx, url, "node-b", order, true)
|
|
round := RoundID(t0)
|
|
|
|
eventually(t, "node-a opened the round and started its run", 20*time.Second, func() bool {
|
|
return a.root.count() == 1 && roundNodeOnBus(buckets[StateRounds], round, "node-a").Run != ""
|
|
})
|
|
time.Sleep(time.Second)
|
|
if b.root.count() != 0 {
|
|
t.Fatalf("node-b started before node-a passed: %v", b.root.started)
|
|
}
|
|
// node-a's transaction ends: its lock is released, its run is finished and checked.
|
|
a.signals <- LockReleased
|
|
eventually(t, "node-b started once node-a passed", 20*time.Second, func() bool {
|
|
return b.root.count() == 1 && roundNodeOnBus(buckets[StateRounds], round, "node-b").Run != ""
|
|
})
|
|
if a.root.count() != 1 {
|
|
t.Errorf("node-a ran again: %v", a.root.started)
|
|
}
|
|
}
|
|
|
|
func TestAFailedCheckStopsTheRoundOnARealBus(t *testing.T) {
|
|
url, buckets := roundBus(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
order := []string{"node-a", "node-b"}
|
|
a := startRoundNode(t, ctx, url, "node-a", order, false)
|
|
b := startRoundNode(t, ctx, url, "node-b", order, true)
|
|
round := RoundID(t0)
|
|
eventually(t, "node-a started its run", 20*time.Second, func() bool { return a.root.count() == 1 })
|
|
a.signals <- LockReleased
|
|
runA := ""
|
|
eventually(t, "node-a's run checked", 20*time.Second, func() bool {
|
|
runA = roundNodeOnBus(buckets[StateRounds], round, "node-a").Run
|
|
if runA == "" {
|
|
return false
|
|
}
|
|
e, err := buckets[StateRuns].Get(context.Background(), NodeKey("node-a", runA))
|
|
return err == nil && strings.Contains(string(e.Value()), `"checked-after"`)
|
|
})
|
|
time.Sleep(2 * time.Second)
|
|
if b.root.count() != 0 || roundNodeOnBus(buckets[StateRounds], round, "node-b").Run != "" {
|
|
t.Fatalf("node-b started after node-a's check failed: %v", b.root.started)
|
|
}
|
|
}
|