173 lines
6.6 KiB
Go
173 lines
6.6 KiB
Go
package main
|
|
|
|
// The upgrade parts joined to the keeper's loop (keeperHooks): the log read into runs and old records pruned at
|
|
// the start, an ask for this node taken from `asks`, and the records pruned again a day later.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
func TestTheKeeperRunsTheUpgradePartsAtItsStartOnItsAsksAndDaily(t *testing.T) {
|
|
db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
|
|
db.set("glibc 2.42-1\n", "")
|
|
k, states, _, clock, _ := newTestKeeper(db)
|
|
log := capturedLog(t)
|
|
k.Machine.ReadFile = func(path string) ([]byte, error) {
|
|
if path == PacmanLog {
|
|
return []byte(log), nil
|
|
}
|
|
return nil, errNoFile
|
|
}
|
|
k.MakeHooks = keeperHooks
|
|
k.Logf = t.Logf
|
|
k.Config = KeeperConfig{RecordDays: 180}
|
|
records := states.store(StateRecords)
|
|
records.Put(NodeKey("node-a", "20250101T040000Z"), Record{Node: "node-a", Run: "20250101T040000Z"})
|
|
records.Put(NodeKey("node-b", "20250101T040000Z"), Record{Node: "node-b", Run: "20250101T040000Z"})
|
|
done := Run{Node: "node-a", Run: "20261003T040000Z", Kind: RunUpgrade, From: FromAsk, Ask: "ask-1",
|
|
Steps: []Step{{Step: StepRefused, At: t0, Why: "a test"}}}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
ended := make(chan error, 1)
|
|
go func() { ended <- k.Run(ctx) }()
|
|
until := func(what string, ok func() bool) {
|
|
t.Helper()
|
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
|
if ok() {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("never: %s", what)
|
|
}
|
|
until("the log read into runs", func() bool {
|
|
keys, _ := states.store(StateRuns).Keys()
|
|
return len(keys) > 1
|
|
})
|
|
until("the old record pruned at the start", func() bool {
|
|
e, _ := records.Get(NodeKey("node-a", "20250101T040000Z"))
|
|
return e == nil
|
|
})
|
|
if e, _ := records.Get(NodeKey("node-b", "20250101T040000Z")); e == nil {
|
|
t.Error("another node's record was pruned")
|
|
}
|
|
|
|
// a run an ask started, ended; its ask left behind by a keeper that went away.
|
|
states.store(StateRuns).Put(NodeKey("node-a", done.Run), done)
|
|
asks := states.store(StateAsks)
|
|
asks.Put(NodeKey("node-a", "ask-1"), Ask{ID: "ask-1", Node: "node-a", Act: ActUpgrade, At: t0})
|
|
until("the ask a run names deleted", func() bool {
|
|
e, _ := asks.Get(NodeKey("node-a", "ask-1"))
|
|
return e == nil
|
|
})
|
|
|
|
records.Put(NodeKey("node-a", "20250102T040000Z"), Record{Node: "node-a", Run: "20250102T040000Z"})
|
|
time.Sleep(20 * time.Millisecond)
|
|
if e, _ := records.Get(NodeKey("node-a", "20250102T040000Z")); e == nil {
|
|
t.Fatal("pruned before a day passed")
|
|
}
|
|
clock.Advance(UpdatesEvery)
|
|
until("the old record pruned a day later", func() bool {
|
|
e, _ := records.Get(NodeKey("node-a", "20250102T040000Z"))
|
|
return e == nil
|
|
})
|
|
cancel()
|
|
if err := <-ended; err == nil || !strings.Contains(err.Error(), "canceled") {
|
|
t.Errorf("Run: %v", err)
|
|
}
|
|
}
|
|
|
|
// roundKeeper is a keeper of node in a round of the made-up nodes order, its machine's time the keeper's clock.
|
|
func roundKeeper(t *testing.T, node string, order []string) (*Keeper, *fakeStates, *fakeClock) {
|
|
t.Helper()
|
|
db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
|
|
db.set("glibc 2.42-1\n", "")
|
|
k, states, _, clock, _ := newTestKeeper(db)
|
|
k.Node = node
|
|
k.Machine.Now = clock.Now
|
|
k.MakeHooks = keeperHooks
|
|
k.Logf = t.Logf
|
|
k.Config = KeeperConfig{RecordDays: 180, RoundStartMinutes: 120, UpgradeOrder: order}
|
|
k.AskRecord = func(string) (*stdio.StateEntry, error) { return nil, nil }
|
|
return k, states, clock
|
|
}
|
|
|
|
func until(t *testing.T, what string, ok func() bool) {
|
|
t.Helper()
|
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
|
if ok() {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("never: %s", what)
|
|
}
|
|
|
|
func roundNodeOf(s *fakeStore, round, node string) RoundNode {
|
|
var rn RoundNode
|
|
if e, _ := s.Get(RoundNodeKey(round, node)); e != nil {
|
|
_ = json.Unmarshal(e.Value, &rn)
|
|
}
|
|
return rn
|
|
}
|
|
|
|
// 1841/AC6 through the keeper's loop: the first of the order opens the week's round at its start on Saturday
|
|
// 04:00 and starts its run in it; the round's parts get the operator channel's reader and the rounds state.
|
|
func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) {
|
|
k, states, _ := roundKeeper(t, "node-a", []string{"node-a", "node-b"})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = k.Run(ctx) }()
|
|
rounds := states.store(StateRounds)
|
|
round := RoundID(t0)
|
|
until(t, "the round opened", func() bool { e, _ := rounds.Get(round); return e != nil })
|
|
until(t, "node-a's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-a").Run != "" })
|
|
if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" {
|
|
t.Errorf("node-b started in node-a's keeper: %+v", rn)
|
|
}
|
|
}
|
|
|
|
// 1841/AC6 through the keeper's loop: a node waits until the node before it passed its check, and starts on
|
|
// the change of `runs` that says so, with no wait for a timer.
|
|
func TestTheNextNodesKeeperStartsWhenTheRunBeforeItPassed(t *testing.T) {
|
|
k, states, clock := roundKeeper(t, "node-b", []string{"node-a", "node-b"})
|
|
round := RoundID(t0)
|
|
rounds, runs := states.store(StateRounds), states.store(StateRuns)
|
|
runA := RunID(t0)
|
|
rounds.Put(round, Round{Round: round, Opened: t0, OpenedBy: "node-a", Order: []string{"node-a", "node-b"}})
|
|
rounds.Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: runA})
|
|
a := Run{Node: "node-a", Run: runA, Kind: RunUpgrade, From: FromRound, Round: round,
|
|
Steps: []Step{{Step: StepRecorded, At: t0}, {Step: StepStarted, At: t0}}}
|
|
runs.Put(NodeKey("node-a", runA), a)
|
|
clock.Advance(10 * time.Minute)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = k.Run(ctx) }()
|
|
time.Sleep(30 * time.Millisecond)
|
|
if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" {
|
|
t.Fatalf("node-b started before node-a passed: %+v", rn)
|
|
}
|
|
a.Steps = append(a.Steps, Step{Step: StepFinished, At: clock.Now()}, Step{Step: StepCheckedAfter, At: clock.Now(), Check: &Check{Passed: true}})
|
|
runs.Put(NodeKey("node-a", runA), a)
|
|
until(t, "node-b's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-b").Run != "" })
|
|
}
|
|
|
|
// The pass of a node is answered from the operator channel's record: the keeper hands its reader on.
|
|
func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) {
|
|
k, states, _ := roundKeeper(t, "node-a", nil)
|
|
k.State = states.State
|
|
h := keeperHooks(k)
|
|
if k.upgrades == nil || k.upgrades.AskRecord == nil || k.upgrades.Rounds == nil {
|
|
t.Fatalf("the round's parts lack the channel's reader or the rounds state: %+v", k.upgrades)
|
|
}
|
|
if len(h.Wakes) < 2 {
|
|
t.Fatalf("no wake for the round: %d wakes", len(h.Wakes))
|
|
}
|
|
}
|