jschoubben
2b495c4818
Each node's reflector ranked its own list on its own timer, and on 2026-10-31 one node's lagging first mirror left it a Node.js version behind the rest. The list is now the module's state mirrors, key list: the node that opens a week's round ranks it with the module's reflector options, keeps only mirrors whose last sync is no older than the newest found, and the round's key names the list's revision. Every keeper writes /etc/pacman.d/mirrorlist whole from the key through the root helper's new verb mirrorlist, when the key changes and when the file differs, and never writes an empty or unreadable list. reflector.timer is declared stopped and disabled, and reflector.conf saves nothing. pacman_mirrors answers the shared list, its ranker and age and whether this node's file matches; with refresh it asks this node's keeper to rank again, refused while a round runs, naming it. The round's integration test proves the opening write and both nodes' files. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
259 lines
11 KiB
Go
259 lines
11 KiB
Go
package main
|
|
|
|
// The upgrade parts joined to the keeper's loop (keeperHooks): the log read into runs and old records pruned at
|
|
// the start, an ask for this node taken from `asks`, and the records pruned again a day later.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
func TestTheKeeperRunsTheUpgradePartsAtItsStartOnItsAsksAndDaily(t *testing.T) {
|
|
db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
|
|
db.set("glibc 2.42-1\n", "")
|
|
k, states, _, clock, _ := newTestKeeper(db)
|
|
log := capturedLog(t)
|
|
k.Machine.ReadFile = func(path string) ([]byte, error) {
|
|
if path == PacmanLog {
|
|
return []byte(log), nil
|
|
}
|
|
return nil, errNoFile
|
|
}
|
|
k.MakeHooks = keeperHooks
|
|
k.Logf = t.Logf
|
|
k.Config = KeeperConfig{RecordDays: 180}
|
|
records := states.store(StateRecords)
|
|
records.Put(NodeKey("node-a", "20250101T040000Z"), Record{Node: "node-a", Run: "20250101T040000Z"})
|
|
records.Put(NodeKey("node-b", "20250101T040000Z"), Record{Node: "node-b", Run: "20250101T040000Z"})
|
|
done := Run{Node: "node-a", Run: "20261003T040000Z", Kind: RunUpgrade, From: FromAsk, Ask: "ask-1",
|
|
Steps: []Step{{Step: StepRefused, At: t0, Why: "a test"}}}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
ended := make(chan error, 1)
|
|
go func() { ended <- k.Run(ctx) }()
|
|
until := func(what string, ok func() bool) {
|
|
t.Helper()
|
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
|
if ok() {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("never: %s", what)
|
|
}
|
|
until("the log read into runs", func() bool {
|
|
keys, _ := states.store(StateRuns).Keys()
|
|
return len(keys) > 1
|
|
})
|
|
until("the old record pruned at the start", func() bool {
|
|
e, _ := records.Get(NodeKey("node-a", "20250101T040000Z"))
|
|
return e == nil
|
|
})
|
|
if e, _ := records.Get(NodeKey("node-b", "20250101T040000Z")); e == nil {
|
|
t.Error("another node's record was pruned")
|
|
}
|
|
|
|
// a run an ask started, ended; its ask left behind by a keeper that went away.
|
|
states.store(StateRuns).Put(NodeKey("node-a", done.Run), done)
|
|
asks := states.store(StateAsks)
|
|
asks.Put(NodeKey("node-a", "ask-1"), Ask{ID: "ask-1", Node: "node-a", Act: ActUpgrade, At: t0})
|
|
until("the ask a run names deleted", func() bool {
|
|
e, _ := asks.Get(NodeKey("node-a", "ask-1"))
|
|
return e == nil
|
|
})
|
|
|
|
records.Put(NodeKey("node-a", "20250102T040000Z"), Record{Node: "node-a", Run: "20250102T040000Z"})
|
|
time.Sleep(20 * time.Millisecond)
|
|
if e, _ := records.Get(NodeKey("node-a", "20250102T040000Z")); e == nil {
|
|
t.Fatal("pruned before a day passed")
|
|
}
|
|
clock.Advance(UpdatesEvery)
|
|
until("the old record pruned a day later", func() bool {
|
|
e, _ := records.Get(NodeKey("node-a", "20250102T040000Z"))
|
|
return e == nil
|
|
})
|
|
cancel()
|
|
if err := <-ended; err == nil || !strings.Contains(err.Error(), "canceled") {
|
|
t.Errorf("Run: %v", err)
|
|
}
|
|
}
|
|
|
|
// roundKeeper is a keeper of node in a round of the made-up nodes order, its machine's time the keeper's clock.
|
|
func roundKeeper(t *testing.T, node string, order []string) (*Keeper, *fakeStates, *fakeClock) {
|
|
t.Helper()
|
|
db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}}
|
|
db.set("glibc 2.42-1\n", "")
|
|
k, states, _, clock, _ := newTestKeeper(db)
|
|
k.Node = node
|
|
k.Machine.Now = clock.Now
|
|
k.MakeHooks = keeperHooks
|
|
k.Logf = t.Logf
|
|
k.Config = KeeperConfig{RecordDays: 180, RoundStartMinutes: 120, UpgradeOrder: order}
|
|
k.AskRecord = func(string) (*stdio.StateEntry, error) { return nil, nil }
|
|
return k, states, clock
|
|
}
|
|
|
|
func until(t *testing.T, what string, ok func() bool) {
|
|
t.Helper()
|
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
|
if ok() {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("never: %s", what)
|
|
}
|
|
|
|
func roundNodeOf(s *fakeStore, round, node string) RoundNode {
|
|
var rn RoundNode
|
|
if e, _ := s.Get(RoundNodeKey(round, node)); e != nil {
|
|
_ = json.Unmarshal(e.Value, &rn)
|
|
}
|
|
return rn
|
|
}
|
|
|
|
// 1841/AC6 through the keeper's loop: the first of the order opens the week's round at its start on Saturday
|
|
// 04:00 and starts its run in it; the round's parts get the operator channel's reader and the rounds state.
|
|
//
|
|
// ADR 0585 §2, §3: it ranks the mesh's mirrors inside the write that opens the round, before the round's key, which names
|
|
// the list's revision; and its keeper writes the list to its node's file.
|
|
func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) {
|
|
k, states, _ := roundKeeper(t, "node-a", []string{"node-a", "node-b"})
|
|
root := &fakeRoot{}
|
|
k.Root = root
|
|
servers := []string{"https://nl.mirror.example/archlinux/$repo/os/$arch"}
|
|
k.Rank = func(conf []byte, round string) (MirrorList, error) {
|
|
if string(conf) != "--sort rate\n" {
|
|
t.Errorf("ranked with %q", conf)
|
|
}
|
|
return MirrorList{Servers: servers, RankedAt: t0, RankedBy: "node-a", Round: round}, nil
|
|
}
|
|
k.Machine.ReadFile = func(path string) ([]byte, error) {
|
|
if path == ReflectorConf {
|
|
return []byte("--sort rate\n"), nil
|
|
}
|
|
return nil, errNoFile
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = k.Run(ctx) }()
|
|
rounds := states.store(StateRounds)
|
|
round := RoundID(t0)
|
|
until(t, "the round opened", func() bool { e, _ := rounds.Get(round); return e != nil })
|
|
until(t, "node-a's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-a").Run != "" })
|
|
if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" {
|
|
t.Errorf("node-b started in node-a's keeper: %+v", rn)
|
|
}
|
|
var opened Round
|
|
e, _ := rounds.Get(round)
|
|
_ = json.Unmarshal(e.Value, &opened)
|
|
list, rev, err := readMirrorList(states.store(StateMirrors))
|
|
if err != nil || list == nil || list.Round != round || opened.Mirrors != rev || rev == 0 || opened.MirrorsNote != "" {
|
|
t.Errorf("the round opened with mirrors %d (%q), the list is %+v at %d (%v)", opened.Mirrors, opened.MirrorsNote, list, rev, err)
|
|
}
|
|
until(t, "the list written to the node's file", func() bool { w := root.written(); return len(w) == 1 && w[0][0] == servers[0] })
|
|
}
|
|
|
|
// 1841/AC6 through the keeper's loop: a node waits until the node before it passed its check, and starts on
|
|
// the change of `runs` that says so, with no wait for a timer.
|
|
func TestTheNextNodesKeeperStartsWhenTheRunBeforeItPassed(t *testing.T) {
|
|
k, states, clock := roundKeeper(t, "node-b", []string{"node-a", "node-b"})
|
|
round := RoundID(t0)
|
|
rounds, runs := states.store(StateRounds), states.store(StateRuns)
|
|
runA := RunID(t0)
|
|
rounds.Put(round, Round{Round: round, Opened: t0, OpenedBy: "node-a", Order: []string{"node-a", "node-b"}})
|
|
rounds.Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: runA})
|
|
a := Run{Node: "node-a", Run: runA, Kind: RunUpgrade, From: FromRound, Round: round,
|
|
Steps: []Step{{Step: StepRecorded, At: t0}, {Step: StepStarted, At: t0}}}
|
|
runs.Put(NodeKey("node-a", runA), a)
|
|
clock.Advance(10 * time.Minute)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go func() { _ = k.Run(ctx) }()
|
|
time.Sleep(30 * time.Millisecond)
|
|
if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" {
|
|
t.Fatalf("node-b started before node-a passed: %+v", rn)
|
|
}
|
|
a.Steps = append(a.Steps, Step{Step: StepFinished, At: clock.Now()}, Step{Step: StepCheckedAfter, At: clock.Now(), Check: &Check{Passed: true}})
|
|
runs.Put(NodeKey("node-a", runA), a)
|
|
until(t, "node-b's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-b").Run != "" })
|
|
}
|
|
|
|
// The pass of a node is answered from the operator channel's record: the keeper hands its reader on.
|
|
func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) {
|
|
k, states, _ := roundKeeper(t, "node-a", nil)
|
|
k.State = states.State
|
|
h := keeperHooks(k)
|
|
if k.upgrades == nil || k.upgrades.AskRecord == nil || k.upgrades.Rounds == nil {
|
|
t.Fatalf("the round's parts lack the channel's reader or the rounds state: %+v", k.upgrades)
|
|
}
|
|
if len(h.Wakes) < 2 {
|
|
t.Fatalf("no wake for the round: %d wakes", len(h.Wakes))
|
|
}
|
|
// ADR 0585 §3: the node's file is written from the shared list when they differ, and never from a list that is empty
|
|
// or unreadable; the file is written again when it differs (the watch's MirrorsRewritten).
|
|
if len(h.MirrorsRewritten) != 1 {
|
|
t.Fatalf("the mirror list file is not written again when it changes: %d hooks", len(h.MirrorsRewritten))
|
|
}
|
|
root := &fakeRoot{}
|
|
u := k.upgrades
|
|
u.Root = root
|
|
file := []byte(nil)
|
|
u.Machine.ReadFile = func(path string) ([]byte, error) {
|
|
if path == Mirrorlist && file != nil {
|
|
return file, nil
|
|
}
|
|
return nil, errNoFile
|
|
}
|
|
mirrors := states.store(StateMirrors)
|
|
good := []string{"https://a.example/$repo/os/$arch", "https://b.example/archlinux/$repo/os/$arch"}
|
|
for _, step := range []struct {
|
|
name string
|
|
put any
|
|
file []byte
|
|
wrote bool
|
|
failed bool
|
|
}{
|
|
{name: "no shared list yet writes nothing"},
|
|
{name: "an empty list is never written", put: MirrorList{Servers: []string{}}, failed: true},
|
|
{name: "an unreadable list is never written", put: "not a list", failed: true},
|
|
{name: "a list the file differs from is written whole", put: MirrorList{Servers: good}, file: []byte(mirrorlistReflector), wrote: true},
|
|
{name: "a file holding the list is not written again", put: MirrorList{Servers: good}, file: RenderMirrorlist(good)},
|
|
} {
|
|
if step.put != nil {
|
|
mirrors.Put(MirrorsKey, step.put)
|
|
}
|
|
file = step.file
|
|
before := len(root.written())
|
|
wrote, err := u.WriteMirrors()
|
|
if wrote != step.wrote || (err != nil) != step.failed || len(root.written()) != before+map[bool]int{true: 1}[step.wrote] {
|
|
t.Errorf("%s: wrote %v (%v), the root helper wrote %v", step.name, wrote, err, root.written())
|
|
}
|
|
}
|
|
// ADR 0585 §4: a refresh asked of the keeper is refused while a round runs, and ranks the list when none does; the
|
|
// ask is taken once either way.
|
|
asks, rounds := states.store(StateAsks), states.store(StateRounds)
|
|
ranked := 0
|
|
u.Rank = func([]byte, string) (MirrorList, error) { ranked++; return MirrorList{Servers: good}, nil }
|
|
u.Machine.ReadFile = func(string) ([]byte, error) { return []byte("--sort rate\n"), nil }
|
|
rounds.Put("2026-W43", Round{Round: "2026-W43", Opened: t0, OpenedBy: "node-a", Order: []string{"node-a"}})
|
|
for _, running := range []bool{true, false} {
|
|
if !running {
|
|
rounds.Delete("2026-W43")
|
|
}
|
|
key := NodeKey("node-a", "ask-m")
|
|
asks.Put(key, Ask{ID: "ask-m", Node: "node-a", Act: ActMirrors, At: t0})
|
|
e, _ := asks.Get(key)
|
|
err := u.TakeAsk(stdio.StateChange{Op: "put", Key: key, Value: e.Value})
|
|
if left, _ := asks.Get(key); left != nil || (err != nil) != running || ranked != map[bool]int{true: 0, false: 1}[running] ||
|
|
(running && !strings.Contains(err.Error(), "round 2026-W43 runs")) {
|
|
t.Errorf("a refresh with a round running %v: %v, ranked %d, ask left %v", running, err, ranked, left)
|
|
}
|
|
}
|
|
}
|