pacman starts a round's run only once the node's file holds the round's mirror list, and a refresh never writes after a round opened (ADR 0585 §2, §4; review round 1)
mesh/delivery-group group feat/node-packages rejected: composed together: the checker's verdict at the members' heads (98ce55efa892): unit: i3/cmd/i3-tools.Tes…
mesh/delivery closed: a newer head of the same pull request
mesh/check failed: unit: i3/cmd/i3-tools.TestAtItsStartTheWatcherReloadsWhatTheRunningI3HasNotLoaded: loaded map[/tmp/TestAtItsStartTheWatcherReloadsW…

The opener ranked the list and took its turn in the same step, so its upgrade started
before the keeper's change hook wrote /etc/pacman.d/mirrorlist, and a failed write on
any node was only logged. A round's run now writes the list first and checks the file
holds the revision the round opened with; otherwise the run is refused with why and
tried again an hour later, as every refused run is. A refresh asks again for a running
round right before its write, since ranking takes minutes.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
2026-11-01 16:32:59 +01:00
parent 7375d613b4
commit 3d1a450406
5 changed files with 134 additions and 12 deletions
+1
View File
@@ -2,6 +2,7 @@ node_modules/
dist/
# Go tool bundles built in place (go build in a module's cmd/<name>-tools) are build output.
modules/pacman/pacman-tools
modules/slack/cmd/slack-tools/slack-tools
modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
modules/messenger/cmd/messenger/messenger
@@ -6,6 +6,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
@@ -132,9 +133,13 @@ func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) {
}
return MirrorList{Servers: servers, RankedAt: t0, RankedBy: "node-a", Round: round}, nil
}
// The node's mirror list file holds what the root helper last wrote.
k.Machine.ReadFile = func(path string) ([]byte, error) {
if path == ReflectorConf {
switch w := root.written(); {
case path == ReflectorConf:
return []byte("--sort rate\n"), nil
case path == Mirrorlist && len(w) > 0:
return RenderMirrorlist(w[len(w)-1]), nil
}
return nil, errNoFile
}
@@ -156,6 +161,17 @@ func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) {
t.Errorf("the round opened with mirrors %d (%q), the list is %+v at %d (%v)", opened.Mirrors, opened.MirrorsNote, list, rev, err)
}
until(t, "the list written to the node's file", func() bool { w := root.written(); return len(w) == 1 && w[0][0] == servers[0] })
// The opener's run starts only once its file holds the round's list (written before the run, not refused for it).
var run Run
if e, _ := states.store(StateRuns).Get(NodeKey("node-a", roundNodeOf(rounds, round, "node-a").Run)); e != nil {
_ = json.Unmarshal(e.Value, &run)
}
if lastStep(run) == StepRefused && strings.Contains(run.Steps[len(run.Steps)-1].Why, "mirror") {
t.Errorf("node-a's run was refused for its mirror list: %+v", run.Steps)
}
if len(root.written()) != 1 {
t.Errorf("the list was written %d times", len(root.written()))
}
}
// 1841/AC6 through the keeper's loop: a node waits until the node before it passed its check, and starts on
@@ -235,8 +251,34 @@ func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) {
t.Errorf("%s: wrote %v (%v), the root helper wrote %v", step.name, wrote, err, root.written())
}
}
// ADR 0585 §2: a round's run starts only once the node's file holds the list the round opened with: none asked of a
// round that opened with no list; refused, naming why, when the bus holds another revision or the file is not written.
rounds0 := states.store(StateRounds)
_, listRev, _ := readMirrorList(mirrors)
file = RenderMirrorlist(good)
for _, c := range []struct {
name, refusal string
opened uint64
fail error
}{
{name: "a round that opened with no list", opened: 0},
{name: "a round whose list the file holds", opened: listRev},
{name: "a round that opened with another revision", opened: listRev + 7, refusal: "the bus holds revision"},
{name: "a file that cannot be written", opened: listRev, fail: errors.New("no root"), refusal: "is not written"},
} {
rounds0.Put("2026-W40", Round{Round: "2026-W40", Opened: t0, Order: []string{"node-a"}, Mirrors: c.opened})
if c.fail != nil {
file = []byte(mirrorlistReflector)
}
root.fail = c.fail
if why := u.roundMirrorsWritten("2026-W40"); (c.refusal == "") != (why == "") || !strings.Contains(why, c.refusal) {
t.Errorf("%s: %q", c.name, why)
}
}
root.fail = nil
rounds0.Delete("2026-W40")
// ADR 0585 §4: a refresh asked of the keeper is refused while a round runs, and ranks the list when none does; the
// ask is taken once either way.
// ask is taken once either way. A round that opens while it ranks refuses its write, naming the round.
asks, rounds := states.store(StateAsks), states.store(StateRounds)
ranked := 0
u.Rank = func([]byte, string) (MirrorList, error) { ranked++; return MirrorList{Servers: good}, nil }
@@ -255,4 +297,15 @@ func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) {
t.Errorf("a refresh with a round running %v: %v, ranked %d, ask left %v", running, err, ranked, left)
}
}
_, before, _ := readMirrorList(mirrors)
u.Rank = func([]byte, string) (MirrorList, error) {
rounds.Put("2026-W44", Round{Round: "2026-W44", Opened: t0, OpenedBy: "node-a", Order: []string{"node-a"}})
return MirrorList{Servers: good}, nil
}
asks.Put(NodeKey("node-a", "ask-r"), Ask{ID: "ask-r", Node: "node-a", Act: ActMirrors, At: t0})
e, _ := asks.Get(NodeKey("node-a", "ask-r"))
err := u.TakeAsk(stdio.StateChange{Op: "put", Key: NodeKey("node-a", "ask-r"), Value: e.Value})
if _, after, _ := readMirrorList(mirrors); err == nil || !strings.Contains(err.Error(), "round 2026-W44 runs") || after != before {
t.Errorf("a refresh whose round opened while it ranked: %v, the list at %d (was %d)", err, after, before)
}
}
+57 -9
View File
@@ -221,8 +221,9 @@ func readMirrorList(r interface {
// ---- the keeper's part -------------------------------------------------------------------------------------------
// rankMirrors ranks the mesh's list and writes it to the shared key, answering its revision.
func (u *Upgrades) rankMirrors(round string) (uint64, error) {
// rankMirrors ranks the mesh's list and writes it to the shared key, answering its revision. guard, when set, is asked
// right before the write and refuses it with its error: ranking takes minutes, and what held when it began may not.
func (u *Upgrades) rankMirrors(round string, guard func() error) (uint64, error) {
if u.Rank == nil || u.Mirrors == nil {
return 0, errors.New("this keeper ranks no mirrors")
}
@@ -234,13 +235,18 @@ func (u *Upgrades) rankMirrors(round string) (uint64, error) {
if err != nil {
return 0, err
}
if guard != nil {
if err := guard(); err != nil {
return 0, err
}
}
return u.Mirrors.Put(MirrorsKey, list)
}
// mirrorsForRound ranks the list in the write that opens round (ADR 0585 §2): the revision the round opens with, and
// why it was not ranked again when it was not, the round then opening with the list there is.
func (u *Upgrades) mirrorsForRound(round string) (uint64, string) {
rev, err := u.rankMirrors(round)
rev, err := u.rankMirrors(round, nil)
if err == nil {
return rev, ""
}
@@ -278,17 +284,59 @@ func refreshRefused(round string) error {
"again only once every node of it passed (pacman_round says where it stands)", round)
}
// takeMirrorsAsk ranks the list again for the mesh on a refresh, unless a round runs.
// takeMirrorsAsk ranks the list again for the mesh on a refresh, unless a round runs: asked before the ranking and again
// right before its write, so a refresh that began before a round opened never writes after the round's key exists.
func (u *Upgrades) takeMirrorsAsk() error {
if round, err := runningRound(u.Rounds, u.Runs, u.Machine.Now()); err != nil {
return err
} else if round != "" {
return refreshRefused(round)
noRound := func() error {
if round, err := runningRound(u.Rounds, u.Runs, u.Machine.Now()); err != nil {
return err
} else if round != "" {
return refreshRefused(round)
}
return nil
}
_, err := u.rankMirrors("")
if err := noRound(); err != nil {
return err
}
_, err := u.rankMirrors("", noRound)
return err
}
// roundMirrorsWritten is why this node may not start its run in round yet, empty when it may: its mirror list file
// must hold the list the round opened with (Round.Mirrors), written now when it does not (ADR 0585 §2: every node of
// the round upgrades from that list). A round that opened with no list (none ranked yet) asks nothing of the file.
// A refusal is a refused run, tried again an hour later as every refused run of a round is.
func (u *Upgrades) roundMirrorsWritten(round string) string {
e, err := u.Rounds.Get(round)
if err != nil || e == nil {
return fmt.Sprintf("round %s cannot be read to know its mirror list: %v", round, err)
}
var r Round
if err := json.Unmarshal(e.Value, &r); err != nil {
return fmt.Sprintf("round %s is not readable: %v", round, err)
}
if r.Mirrors == 0 {
return ""
}
if u.Mirrors == nil {
return "this keeper reads no mirror list, and round " + round + " opened with one"
}
list, rev, err := readMirrorList(u.Mirrors)
switch {
case err != nil:
return "the mesh's mirror list cannot be read: " + err.Error()
case list == nil || rev != r.Mirrors:
return fmt.Sprintf("round %s opened with the mirror list at revision %d, and the bus holds revision %d", round, r.Mirrors, rev)
}
if _, err := u.WriteMirrors(); err != nil {
return "the round's mirror list is not written to " + Mirrorlist + ": " + err.Error()
}
if have, err := u.Machine.ReadFile(Mirrorlist); err != nil || !bytes.Equal(have, RenderMirrorlist(list.Servers)) {
return fmt.Sprintf("%s does not hold the mirror list round %s opened with (revision %d)", Mirrorlist, round, r.Mirrors)
}
return ""
}
// WriteMirrors writes this node's mirror list file whole from the shared list when they differ, through the root
// helper, answering whether it wrote. No list yet writes nothing; a list that is empty or cannot be read is never
// written, and the node keeps its last (ADR 0585 §3).
@@ -26,12 +26,15 @@ type countingRoot struct {
mu sync.Mutex
started []string
mirrors []string
// mirrorsAtStart is the mirror list the node's file held when each transaction started (ADR 0585 §2).
mirrorsAtStart [][]string
}
func (r *countingRoot) Transaction(kind, run string, ignore []string) error {
r.mu.Lock()
defer r.mu.Unlock()
r.started = append(r.started, kind+" "+run)
r.mirrorsAtStart = append(r.mirrorsAtStart, append([]string(nil), r.mirrors...))
return nil
}
func (r *countingRoot) Rollback(string, []RollbackItem) error { return nil }
@@ -83,16 +86,20 @@ func startRoundNode(t *testing.T, ctx context.Context, url, node string, order [
m := machine(run, 1000)
began := time.Now()
m.Now = func() time.Time { return t0.Add(time.Since(began)) }
n := &roundNode{root: &countingRoot{}, signals: make(chan Signal)}
m.ReadFile = func(path string) ([]byte, error) {
switch path {
case OSRelease:
return []byte("7.2.8-arch1-2\n"), nil
case ReflectorConf:
return []byte("--sort rate\n"), nil
case Mirrorlist: // the file holds what the root helper last wrote
if l := n.root.mirrorlist(); l != nil {
return RenderMirrorlist(l), nil
}
}
return nil, errNoFile
}
n := &roundNode{root: &countingRoot{}, signals: make(chan Signal)}
k := &Keeper{Config: KeeperConfig{UpgradeOrder: order, RecordDays: 180, RoundStartMinutes: 120}, Machine: m, Clock: realClock{},
Watcher: fakeWatcher{n.signals}, Logf: func(f string, a ...any) { t.Logf(node+": "+f, a...) }, MakeHooks: keeperHooks,
Root: n.root, Check: func(Run, *Record) Check { return Check{Passed: passes} },
@@ -192,6 +199,13 @@ func TestTheRoundGoesNodeByNodeOnARealBus(t *testing.T) {
eventually(t, "each node wrote the mesh's mirror list", 10*time.Second, func() bool {
return strings.Join(n.root.mirrorlist(), " ") == strings.Join(roundMirrors, " ")
})
// And each started its run only once its file held the round's list, the opener too.
n.root.mu.Lock()
at := n.root.mirrorsAtStart
n.root.mu.Unlock()
if len(at) != 1 || strings.Join(at[0], " ") != strings.Join(roundMirrors, " ") {
t.Errorf("a node's run started with the mirror list file holding %v, want the round's %v", at, roundMirrors)
}
}
}
@@ -140,6 +140,12 @@ func (u *Upgrades) startUpgrade(r *openRun) error {
if u.Machine.locked() {
return u.refuse(r, "another pacman holds "+DBLock+": a transaction is running, or one was killed and left its lock")
}
// A round's run upgrades from the mirror list the round opened with, written to this node's file first (ADR 0585 §2).
if r.Run.From == FromRound {
if why := u.roundMirrorsWritten(r.Run.Round); why != "" {
return u.refuse(r, why)
}
}
rec, why := u.TakeRecord(r.Run.Run)
if why != "" {
return u.refuse(r, why)