package main // The upgrade parts joined to the keeper's loop (keeperHooks): the log read into runs and old records pruned at // the start, an ask for this node taken from `asks`, and the records pruned again a day later. import ( "context" "encoding/json" "strings" "testing" "time" stdio "git.novox.be/novox/mesh-sdk/go" ) func TestTheKeeperRunsTheUpgradePartsAtItsStartOnItsAsksAndDaily(t *testing.T) { db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} db.set("glibc 2.42-1\n", "") k, states, _, clock, _ := newTestKeeper(db) log := capturedLog(t) k.Machine.ReadFile = func(path string) ([]byte, error) { if path == PacmanLog { return []byte(log), nil } return nil, errNoFile } k.MakeHooks = keeperHooks k.Logf = t.Logf k.Config = KeeperConfig{RecordDays: 180} records := states.store(StateRecords) records.Put(NodeKey("node-a", "20250101T040000Z"), Record{Node: "node-a", Run: "20250101T040000Z"}) records.Put(NodeKey("node-b", "20250101T040000Z"), Record{Node: "node-b", Run: "20250101T040000Z"}) done := Run{Node: "node-a", Run: "20261003T040000Z", Kind: RunUpgrade, From: FromAsk, Ask: "ask-1", Steps: []Step{{Step: StepRefused, At: t0, Why: "a test"}}} ctx, cancel := context.WithCancel(context.Background()) defer cancel() ended := make(chan error, 1) go func() { ended <- k.Run(ctx) }() until := func(what string, ok func() bool) { t.Helper() for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { if ok() { return } } t.Fatalf("never: %s", what) } until("the log read into runs", func() bool { keys, _ := states.store(StateRuns).Keys() return len(keys) > 1 }) until("the old record pruned at the start", func() bool { e, _ := records.Get(NodeKey("node-a", "20250101T040000Z")) return e == nil }) if e, _ := records.Get(NodeKey("node-b", "20250101T040000Z")); e == nil { t.Error("another node's record was pruned") } // a run an ask started, ended; its ask left behind by a keeper that went away. states.store(StateRuns).Put(NodeKey("node-a", done.Run), done) asks := states.store(StateAsks) asks.Put(NodeKey("node-a", "ask-1"), Ask{ID: "ask-1", Node: "node-a", Act: ActUpgrade, At: t0}) until("the ask a run names deleted", func() bool { e, _ := asks.Get(NodeKey("node-a", "ask-1")) return e == nil }) records.Put(NodeKey("node-a", "20250102T040000Z"), Record{Node: "node-a", Run: "20250102T040000Z"}) time.Sleep(20 * time.Millisecond) if e, _ := records.Get(NodeKey("node-a", "20250102T040000Z")); e == nil { t.Fatal("pruned before a day passed") } clock.Advance(UpdatesEvery) until("the old record pruned a day later", func() bool { e, _ := records.Get(NodeKey("node-a", "20250102T040000Z")) return e == nil }) cancel() if err := <-ended; err == nil || !strings.Contains(err.Error(), "canceled") { t.Errorf("Run: %v", err) } } // roundKeeper is a keeper of node in a round of the made-up nodes order, its machine's time the keeper's clock. func roundKeeper(t *testing.T, node string, order []string) (*Keeper, *fakeStates, *fakeClock) { t.Helper() db := &pacmanDB{sl: syncDBs, checkup: Ran{Status: 2}} db.set("glibc 2.42-1\n", "") k, states, _, clock, _ := newTestKeeper(db) k.Node = node k.Machine.Now = clock.Now k.MakeHooks = keeperHooks k.Logf = t.Logf k.Config = KeeperConfig{RecordDays: 180, RoundStartMinutes: 120, UpgradeOrder: order} k.AskRecord = func(string) (*stdio.StateEntry, error) { return nil, nil } return k, states, clock } func until(t *testing.T, what string, ok func() bool) { t.Helper() for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { if ok() { return } } t.Fatalf("never: %s", what) } func roundNodeOf(s *fakeStore, round, node string) RoundNode { var rn RoundNode if e, _ := s.Get(RoundNodeKey(round, node)); e != nil { _ = json.Unmarshal(e.Value, &rn) } return rn } // 1841/AC6 through the keeper's loop: the first of the order opens the week's round at its start on Saturday // 04:00 and starts its run in it; the round's parts get the operator channel's reader and the rounds state. func TestTheFirstNodesKeeperOpensTheRoundAndTakesItsTurn(t *testing.T) { k, states, _ := roundKeeper(t, "node-a", []string{"node-a", "node-b"}) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { _ = k.Run(ctx) }() rounds := states.store(StateRounds) round := RoundID(t0) until(t, "the round opened", func() bool { e, _ := rounds.Get(round); return e != nil }) until(t, "node-a's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-a").Run != "" }) if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" { t.Errorf("node-b started in node-a's keeper: %+v", rn) } } // 1841/AC6 through the keeper's loop: a node waits until the node before it passed its check, and starts on // the change of `runs` that says so, with no wait for a timer. func TestTheNextNodesKeeperStartsWhenTheRunBeforeItPassed(t *testing.T) { k, states, clock := roundKeeper(t, "node-b", []string{"node-a", "node-b"}) round := RoundID(t0) rounds, runs := states.store(StateRounds), states.store(StateRuns) runA := RunID(t0) rounds.Put(round, Round{Round: round, Opened: t0, OpenedBy: "node-a", Order: []string{"node-a", "node-b"}}) rounds.Put(RoundNodeKey(round, "node-a"), RoundNode{Round: round, Node: "node-a", Run: runA}) a := Run{Node: "node-a", Run: runA, Kind: RunUpgrade, From: FromRound, Round: round, Steps: []Step{{Step: StepRecorded, At: t0}, {Step: StepStarted, At: t0}}} runs.Put(NodeKey("node-a", runA), a) clock.Advance(10 * time.Minute) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { _ = k.Run(ctx) }() time.Sleep(30 * time.Millisecond) if rn := roundNodeOf(rounds, round, "node-b"); rn.Run != "" { t.Fatalf("node-b started before node-a passed: %+v", rn) } a.Steps = append(a.Steps, Step{Step: StepFinished, At: clock.Now()}, Step{Step: StepCheckedAfter, At: clock.Now(), Check: &Check{Passed: true}}) runs.Put(NodeKey("node-a", runA), a) until(t, "node-b's run named in the round", func() bool { return roundNodeOf(rounds, round, "node-b").Run != "" }) } // The pass of a node is answered from the operator channel's record: the keeper hands its reader on. func TestTheKeeperHandsTheOperatorChannelsReaderToTheRound(t *testing.T) { k, states, _ := roundKeeper(t, "node-a", nil) k.State = states.State h := keeperHooks(k) if k.upgrades == nil || k.upgrades.AskRecord == nil || k.upgrades.Rounds == nil { t.Fatalf("the round's parts lack the channel's reader or the rounds state: %+v", k.upgrades) } if len(h.Wakes) < 2 { t.Fatalf("no wake for the round: %d wakes", len(h.Wakes)) } }