diff --git a/.gitignore b/.gitignore index 6f3d049b..bab11407 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules/ dist/ # Go tool bundles built in place (go build in a module's cmd/-tools) are build output. +modules/pacman/pacman-tools modules/slack/cmd/slack-tools/slack-tools modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools modules/messenger/cmd/messenger/messenger diff --git a/modules/pacman/cmd/pacman-tools/keeper_hooks_test.go b/modules/pacman/cmd/pacman-tools/keeper_hooks_test.go index a314babc..5b2a0565 100644 --- a/modules/pacman/cmd/pacman-tools/keeper_hooks_test.go +++ b/modules/pacman/cmd/pacman-tools/keeper_hooks_test.go @@ -6,6 +6,7 @@ package main import ( "context" "encoding/json" + "errors" "strings" "testing" "time" @@ -132,9 +133,13 @@ func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) { } return MirrorList{Servers: servers, RankedAt: t0, RankedBy: "node-a", Round: round}, nil } + // The node's mirror list file holds what the root helper last wrote. k.Machine.ReadFile = func(path string) ([]byte, error) { - if path == ReflectorConf { + switch w := root.written(); { + case path == ReflectorConf: return []byte("--sort rate\n"), nil + case path == Mirrorlist && len(w) > 0: + return RenderMirrorlist(w[len(w)-1]), nil } return nil, errNoFile } @@ -156,6 +161,17 @@ func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) { t.Errorf("the round opened with mirrors %d (%q), the list is %+v at %d (%v)", opened.Mirrors, opened.MirrorsNote, list, rev, err) } until(t, "the list written to the node's file", func() bool { w := root.written(); return len(w) == 1 && w[0][0] == servers[0] }) + // The opener's run starts only once its file holds the round's list (written before the run, not refused for it). + var run Run + if e, _ := states.store(StateRuns).Get(NodeKey("node-a", roundNodeOf(rounds, round, "node-a").Run)); e != nil { + _ = json.Unmarshal(e.Value, &run) + } + if lastStep(run) == StepRefused && strings.Contains(run.Steps[len(run.Steps)-1].Why, "mirror") { + t.Errorf("node-a's run was refused for its mirror list: %+v", run.Steps) + } + if len(root.written()) != 1 { + t.Errorf("the list was written %d times", len(root.written())) + } } // 1841/AC6 through the keeper's loop: a node waits until the node before it passed its check, and starts on @@ -235,8 +251,34 @@ func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) { t.Errorf("%s: wrote %v (%v), the root helper wrote %v", step.name, wrote, err, root.written()) } } + // ADR 0585 §2: a round's run starts only once the node's file holds the list the round opened with: none asked of a + // round that opened with no list; refused, naming why, when the bus holds another revision or the file is not written. + rounds0 := states.store(StateRounds) + _, listRev, _ := readMirrorList(mirrors) + file = RenderMirrorlist(good) + for _, c := range []struct { + name, refusal string + opened uint64 + fail error + }{ + {name: "a round that opened with no list", opened: 0}, + {name: "a round whose list the file holds", opened: listRev}, + {name: "a round that opened with another revision", opened: listRev + 7, refusal: "the bus holds revision"}, + {name: "a file that cannot be written", opened: listRev, fail: errors.New("no root"), refusal: "is not written"}, + } { + rounds0.Put("2026-W40", Round{Round: "2026-W40", Opened: t0, Order: []string{"node-a"}, Mirrors: c.opened}) + if c.fail != nil { + file = []byte(mirrorlistReflector) + } + root.fail = c.fail + if why := u.roundMirrorsWritten("2026-W40"); (c.refusal == "") != (why == "") || !strings.Contains(why, c.refusal) { + t.Errorf("%s: %q", c.name, why) + } + } + root.fail = nil + rounds0.Delete("2026-W40") // ADR 0585 §4: a refresh asked of the keeper is refused while a round runs, and ranks the list when none does; the - // ask is taken once either way. + // ask is taken once either way. A round that opens while it ranks refuses its write, naming the round. asks, rounds := states.store(StateAsks), states.store(StateRounds) ranked := 0 u.Rank = func([]byte, string) (MirrorList, error) { ranked++; return MirrorList{Servers: good}, nil } @@ -255,4 +297,15 @@ func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) { t.Errorf("a refresh with a round running %v: %v, ranked %d, ask left %v", running, err, ranked, left) } } + _, before, _ := readMirrorList(mirrors) + u.Rank = func([]byte, string) (MirrorList, error) { + rounds.Put("2026-W44", Round{Round: "2026-W44", Opened: t0, OpenedBy: "node-a", Order: []string{"node-a"}}) + return MirrorList{Servers: good}, nil + } + asks.Put(NodeKey("node-a", "ask-r"), Ask{ID: "ask-r", Node: "node-a", Act: ActMirrors, At: t0}) + e, _ := asks.Get(NodeKey("node-a", "ask-r")) + err := u.TakeAsk(stdio.StateChange{Op: "put", Key: NodeKey("node-a", "ask-r"), Value: e.Value}) + if _, after, _ := readMirrorList(mirrors); err == nil || !strings.Contains(err.Error(), "round 2026-W44 runs") || after != before { + t.Errorf("a refresh whose round opened while it ranked: %v, the list at %d (was %d)", err, after, before) + } } diff --git a/modules/pacman/cmd/pacman-tools/mirrors.go b/modules/pacman/cmd/pacman-tools/mirrors.go index eaebcb7c..27564916 100644 --- a/modules/pacman/cmd/pacman-tools/mirrors.go +++ b/modules/pacman/cmd/pacman-tools/mirrors.go @@ -221,8 +221,9 @@ func readMirrorList(r interface { // ---- the keeper's part ------------------------------------------------------------------------------------------- -// rankMirrors ranks the mesh's list and writes it to the shared key, answering its revision. -func (u *Upgrades) rankMirrors(round string) (uint64, error) { +// rankMirrors ranks the mesh's list and writes it to the shared key, answering its revision. guard, when set, is asked +// right before the write and refuses it with its error: ranking takes minutes, and what held when it began may not. +func (u *Upgrades) rankMirrors(round string, guard func() error) (uint64, error) { if u.Rank == nil || u.Mirrors == nil { return 0, errors.New("this keeper ranks no mirrors") } @@ -234,13 +235,18 @@ func (u *Upgrades) rankMirrors(round string) (uint64, error) { if err != nil { return 0, err } + if guard != nil { + if err := guard(); err != nil { + return 0, err + } + } return u.Mirrors.Put(MirrorsKey, list) } // mirrorsForRound ranks the list in the write that opens round (ADR 0585 §2): the revision the round opens with, and // why it was not ranked again when it was not, the round then opening with the list there is. func (u *Upgrades) mirrorsForRound(round string) (uint64, string) { - rev, err := u.rankMirrors(round) + rev, err := u.rankMirrors(round, nil) if err == nil { return rev, "" } @@ -278,17 +284,59 @@ func refreshRefused(round string) error { "again only once every node of it passed (pacman_round says where it stands)", round) } -// takeMirrorsAsk ranks the list again for the mesh on a refresh, unless a round runs. +// takeMirrorsAsk ranks the list again for the mesh on a refresh, unless a round runs: asked before the ranking and again +// right before its write, so a refresh that began before a round opened never writes after the round's key exists. func (u *Upgrades) takeMirrorsAsk() error { - if round, err := runningRound(u.Rounds, u.Runs, u.Machine.Now()); err != nil { - return err - } else if round != "" { - return refreshRefused(round) + noRound := func() error { + if round, err := runningRound(u.Rounds, u.Runs, u.Machine.Now()); err != nil { + return err + } else if round != "" { + return refreshRefused(round) + } + return nil } - _, err := u.rankMirrors("") + if err := noRound(); err != nil { + return err + } + _, err := u.rankMirrors("", noRound) return err } +// roundMirrorsWritten is why this node may not start its run in round yet, empty when it may: its mirror list file +// must hold the list the round opened with (Round.Mirrors), written now when it does not (ADR 0585 §2: every node of +// the round upgrades from that list). A round that opened with no list (none ranked yet) asks nothing of the file. +// A refusal is a refused run, tried again an hour later as every refused run of a round is. +func (u *Upgrades) roundMirrorsWritten(round string) string { + e, err := u.Rounds.Get(round) + if err != nil || e == nil { + return fmt.Sprintf("round %s cannot be read to know its mirror list: %v", round, err) + } + var r Round + if err := json.Unmarshal(e.Value, &r); err != nil { + return fmt.Sprintf("round %s is not readable: %v", round, err) + } + if r.Mirrors == 0 { + return "" + } + if u.Mirrors == nil { + return "this keeper reads no mirror list, and round " + round + " opened with one" + } + list, rev, err := readMirrorList(u.Mirrors) + switch { + case err != nil: + return "the mesh's mirror list cannot be read: " + err.Error() + case list == nil || rev != r.Mirrors: + return fmt.Sprintf("round %s opened with the mirror list at revision %d, and the bus holds revision %d", round, r.Mirrors, rev) + } + if _, err := u.WriteMirrors(); err != nil { + return "the round's mirror list is not written to " + Mirrorlist + ": " + err.Error() + } + if have, err := u.Machine.ReadFile(Mirrorlist); err != nil || !bytes.Equal(have, RenderMirrorlist(list.Servers)) { + return fmt.Sprintf("%s does not hold the mirror list round %s opened with (revision %d)", Mirrorlist, round, r.Mirrors) + } + return "" +} + // WriteMirrors writes this node's mirror list file whole from the shared list when they differ, through the root // helper, answering whether it wrote. No list yet writes nothing; a list that is empty or cannot be read is never // written, and the node keeps its last (ADR 0585 §3). diff --git a/modules/pacman/cmd/pacman-tools/round_integration_test.go b/modules/pacman/cmd/pacman-tools/round_integration_test.go index 099f22ff..5eec1f51 100644 --- a/modules/pacman/cmd/pacman-tools/round_integration_test.go +++ b/modules/pacman/cmd/pacman-tools/round_integration_test.go @@ -26,12 +26,15 @@ type countingRoot struct { mu sync.Mutex started []string mirrors []string + // mirrorsAtStart is the mirror list the node's file held when each transaction started (ADR 0585 §2). + mirrorsAtStart [][]string } func (r *countingRoot) Transaction(kind, run string, ignore []string) error { r.mu.Lock() defer r.mu.Unlock() r.started = append(r.started, kind+" "+run) + r.mirrorsAtStart = append(r.mirrorsAtStart, append([]string(nil), r.mirrors...)) return nil } func (r *countingRoot) Rollback(string, []RollbackItem) error { return nil } @@ -83,16 +86,20 @@ func startRoundNode(t *testing.T, ctx context.Context, url, node string, order [ m := machine(run, 1000) began := time.Now() m.Now = func() time.Time { return t0.Add(time.Since(began)) } + n := &roundNode{root: &countingRoot{}, signals: make(chan Signal)} m.ReadFile = func(path string) ([]byte, error) { switch path { case OSRelease: return []byte("7.2.8-arch1-2\n"), nil case ReflectorConf: return []byte("--sort rate\n"), nil + case Mirrorlist: // the file holds what the root helper last wrote + if l := n.root.mirrorlist(); l != nil { + return RenderMirrorlist(l), nil + } } return nil, errNoFile } - n := &roundNode{root: &countingRoot{}, signals: make(chan Signal)} k := &Keeper{Config: KeeperConfig{UpgradeOrder: order, RecordDays: 180, RoundStartMinutes: 120}, Machine: m, Clock: realClock{}, Watcher: fakeWatcher{n.signals}, Logf: func(f string, a ...any) { t.Logf(node+": "+f, a...) }, MakeHooks: keeperHooks, Root: n.root, Check: func(Run, *Record) Check { return Check{Passed: passes} }, @@ -192,6 +199,13 @@ func TestTheRoundGoesNodeByNodeOnARealBus(t *testing.T) { eventually(t, "each node wrote the mesh's mirror list", 10*time.Second, func() bool { return strings.Join(n.root.mirrorlist(), " ") == strings.Join(roundMirrors, " ") }) + // And each started its run only once its file held the round's list, the opener too. + n.root.mu.Lock() + at := n.root.mirrorsAtStart + n.root.mu.Unlock() + if len(at) != 1 || strings.Join(at[0], " ") != strings.Join(roundMirrors, " ") { + t.Errorf("a node's run started with the mirror list file holding %v, want the round's %v", at, roundMirrors) + } } } diff --git a/modules/pacman/cmd/pacman-tools/transaction.go b/modules/pacman/cmd/pacman-tools/transaction.go index 515394eb..0d724693 100644 --- a/modules/pacman/cmd/pacman-tools/transaction.go +++ b/modules/pacman/cmd/pacman-tools/transaction.go @@ -140,6 +140,12 @@ func (u *Upgrades) startUpgrade(r *openRun) error { if u.Machine.locked() { return u.refuse(r, "another pacman holds "+DBLock+": a transaction is running, or one was killed and left its lock") } + // A round's run upgrades from the mirror list the round opened with, written to this node's file first (ADR 0585 §2). + if r.Run.From == FromRound { + if why := u.roundMirrorsWritten(r.Run.Round); why != "" { + return u.refuse(r, why) + } + } rec, why := u.TakeRecord(r.Run.Run) if why != "" { return u.refuse(r, why)