package main // The week's round (novox/hq ADR 0508 §5, §6): the nodes upgrade one at a time in the order of the mesh-level key // `upgrade-order`, each after the node before it passed its after-upgrade check. On Saturday at 04:00 local time // the first node of the order opens the week's round in `rounds` (a key created once, so one round a week); every // node's keeper watches `rounds` and `runs` and starts its own run when its turn came. A failed run or check // stops the round at that node. Only the operator passes a node: `pacman_round` with pass and a reason asks the // operator through the operator's channel (ADR 0259), and the keeper of the node waiting on it lets the round go // on only on the operator's Approve, read from the channel's own record. With no order, no round opens. import ( "encoding/json" "errors" "fmt" "os" "sort" "strings" "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" ) // Module is this module's name: the asker of its asks on the operator's channel. const Module = "pacman" // MessengerAsksBucket is the operator channel's records bucket, where the keeper reads the answer to a pass. const MessengerAsksBucket = "messenger_asks" // The operator's answers to a pass, besides PassApproved and PassDeclined: an ask nobody answered in time. const PassExpired = "expired" // passLasts is how long a pass waits for the operator's answer. const passLasts = 12 * time.Hour // refusedRetry is how long after a run of the round was refused (it started nothing) the node tries again. const refusedRetry = time.Hour // passCheckEvery is how often a keeper reads the channel's record of a pass that waits for its answer. const passCheckEvery = time.Minute // defaultRoundStartMinutes is round-start-minutes' default in the manifest. const defaultRoundStartMinutes = 120 // RoundOpening is the Saturday at 04:00 of the ISO week that holds at, in at's own time zone. func RoundOpening(at time.Time) time.Time { day := int(at.Weekday()+6) % 7 // Monday 0 … Sunday 6 monday := time.Date(at.Year(), at.Month(), at.Day()-day, 0, 0, 0, 0, at.Location()) return monday.AddDate(0, 0, 5).Add(4 * time.Hour) } // PassOperatorAsk is the ask on the operator's channel for a pass: composed from the pass alone, so the tool that // publishes it and the keeper that reads its answer compose the same ask, with the same digest. func PassOperatorAsk(round, node, next, reason, id string, at time.Time) asks.Ask { headline := "Upgrade round: pass " + node if len([]rune(headline)) > asks.HeadlineLength { headline = "Upgrade round: pass a node" } binds, _ := asks.ActDigest(passAct(round, node, reason, id)) return asks.Ask{ ID: id, Headline: headline, Explanation: fmt.Sprintf("The week's upgrade round %s stands at %s. Passing it lets %s and the nodes after it upgrade; "+ "%s is not upgraded in this round.\nThe reason given: %s", round, node, next, node, reason), Options: []asks.Option{ {ID: "pass", Label: "Pass it", Does: "the round goes on past " + node, Level: asks.Approve, Binds: binds}, {ID: "keep", Label: "Keep it waiting", Does: "the round stays at " + node, Level: asks.Acknowledge}, }, Who: asks.Operator, Expires: at.UTC().Add(passLasts), OnExpiry: "the round stays at " + node, } } func passAct(round, node, reason, id string) asks.Act { return asks.Act{"verb": Module + ".pass", "round": round, "node": node, "reason": reason, "ask": id} } // roundView is one round as the bus holds it: the round, each node's place, and each node's run in it. type roundView struct { Round Round Nodes map[string]RoundNode revs map[string]uint64 Runs map[string]Run } // latestRound is the newest round's id, or none. func latestRound(rounds Bucket) (string, error) { keys, err := rounds.Keys() if err != nil { return "", fmt.Errorf("listing rounds: %w", err) } latest := "" for _, k := range keys { if !strings.Contains(k, ".") && k > latest { latest = k } } return latest, nil } func readRound(rounds, runs Bucket, id string) (*roundView, error) { e, err := rounds.Get(id) if err != nil || e == nil { return nil, err } v := &roundView{Nodes: map[string]RoundNode{}, revs: map[string]uint64{}, Runs: map[string]Run{}} if err := json.Unmarshal(e.Value, &v.Round); err != nil { return nil, fmt.Errorf("round %s: %w", id, err) } for _, node := range v.Round.Order { e, err := rounds.Get(RoundNodeKey(id, node)) if err != nil { return nil, err } if e == nil { continue } var rn RoundNode if err := json.Unmarshal(e.Value, &rn); err != nil { return nil, fmt.Errorf("round %s, node %s: %w", id, node, err) } v.Nodes[node], v.revs[node] = rn, e.Revision if rn.Run == "" { continue } re, err := runs.Get(NodeKey(node, rn.Run)) if err != nil { return nil, err } if re != nil { var r Run if err := json.Unmarshal(re.Value, &r); err != nil { return nil, fmt.Errorf("run %s of %s: %w", rn.Run, node, err) } v.Runs[node] = r } } return v, nil } // checkedAt is when a run's after-upgrade check passed, or zero. func checkedAt(r Run) time.Time { for _, s := range r.Steps { if s.Step == StepCheckedAfter && s.Check != nil && s.Check.Passed { return s.At } } return time.Time{} } // passedAt is when a node let the round go on: its check passed, or the operator approved its pass; zero when // it has not. func (v *roundView) passedAt(node string) time.Time { if rn := v.Nodes[node]; rn.Pass != nil && rn.Pass.Answer == PassApproved && rn.Pass.Answered != nil { return *rn.Pass.Answered } if r, ok := v.Runs[node]; ok { return checkedAt(r) } return time.Time{} } // turnSince is when a node's turn came: the round's opening for the first, the passing of the node before it // for the rest; zero when it has not come. func (v *roundView) turnSince(node string) time.Time { for i, n := range v.Round.Order { if n != node { continue } if i == 0 { return v.Round.Opened } return v.passedAt(v.Round.Order[i-1]) } return time.Time{} } // next is the node after node in the order, or none. func (v *roundView) next(node string) string { for i, n := range v.Round.Order { if n == node && i+1 < len(v.Round.Order) { return v.Round.Order[i+1] } } return "" } // putRoundNode writes a node's place in a round from the revision read, so a pass and a run written by two // keepers never overwrite each other: a lost race is read again and changed again. func putRoundNode(rounds Bucket, round, node string, change func(*RoundNode)) error { key := RoundNodeKey(round, node) for attempt := 0; attempt < 5; attempt++ { e, err := rounds.Get(key) if err != nil { return err } rn := RoundNode{Round: round, Node: node} if e != nil { if err := json.Unmarshal(e.Value, &rn); err != nil { return fmt.Errorf("%s: %w", key, err) } change(&rn) _, err = rounds.Update(key, rn, e.Revision) } else { change(&rn) _, err = rounds.Create(key, rn) } if !errors.Is(err, stdio.ErrChanged) { return err } } return fmt.Errorf("%s was written by another keeper five times in a row", key) } // RoundStep does what the node's keeper has to do for the rounds now, and answers when it should be called // again (zero: only when `rounds` or `runs` change). The keeper calls it at its start, at the time it answered, // and whenever `rounds` or `runs` change. The first node of the order opens the week's round once it is // Saturday 04:00; a node whose turn came starts its run; a node waiting on a pass reads the operator's answer. func (u *Upgrades) RoundStep() (time.Time, error) { now := u.Machine.Now() order := u.Config.UpgradeOrder var next time.Time soonest := func(t time.Time) { if !t.IsZero() && (next.IsZero() || t.Before(next)) { next = t } } if len(order) == 0 { return time.Time{}, nil // no order: no round (the health says so) } var errs []error if order[0] == u.Node { opening := RoundOpening(now) if now.Before(opening) { soonest(opening) } else { soonest(opening.AddDate(0, 0, 7)) id := RoundID(now) round := Round{Round: id, Opened: now.UTC(), OpenedBy: u.Node, Order: append([]string(nil), order...)} if _, err := u.Rounds.Create(id, round); err != nil && !errors.Is(err, stdio.ErrChanged) { errs = append(errs, fmt.Errorf("opening round %s: %w", id, err)) } } } id, err := latestRound(u.Rounds) if err != nil || id == "" { return next, errors.Join(append(errs, err)...) } v, err := readRound(u.Rounds, u.Runs, id) if err != nil || v == nil { return next, errors.Join(append(errs, err)...) } // A pass of the node before this one that waits for the operator's answer. for i, node := range v.Round.Order { if i+1 >= len(v.Round.Order) || v.Round.Order[i+1] != u.Node { continue } if rn := v.Nodes[node]; rn.Pass != nil && rn.Pass.Answer == "" { answered, err := u.readPass(v, node) if err != nil { errs = append(errs, err) } if !answered { soonest(now.Add(passCheckEvery)) } else if v, err = readRound(u.Rounds, u.Runs, id); err != nil { return next, errors.Join(append(errs, err)...) } } } t, err := u.takeTurn(v, now) soonest(t) return next, errors.Join(append(errs, err)...) } // takeTurn starts this node's run in the round when its turn came and it has none, and again an hour after a // run of it was refused (it started nothing); it answers when to look again. func (u *Upgrades) takeTurn(v *roundView, now time.Time) (time.Time, error) { if !contains(v.Round.Order, u.Node) || v.turnSince(u.Node).IsZero() || !v.passedAt(u.Node).IsZero() { return time.Time{}, nil } if rn := v.Nodes[u.Node]; rn.Pass != nil && rn.Pass.Answer == "" { return time.Time{}, nil // the operator is asked to pass this node: it waits for the answer } own, err := u.ownRuns() if err != nil { return time.Time{}, err } var latest *Run for i := range own { if own[i].Round == v.Round.Round && (latest == nil || own[i].Run.Run > latest.Run) { latest = &own[i].Run } } if latest != nil { if v.Nodes[u.Node].Run != latest.Run { // a keeper that went away between starting its run and naming it in the round if err := putRoundNode(u.Rounds, v.Round.Round, u.Node, func(rn *RoundNode) { rn.Run = latest.Run }); err != nil { return time.Time{}, err } } if lastStep(*latest) != StepRefused { return time.Time{}, nil // running, or ended: a failure stops the round here } retry := latest.Steps[len(latest.Steps)-1].At.Add(refusedRetry) if now.Before(retry) { return retry, nil } } r, err := u.StartUpgrade(FromRound, v.Round.Round, "") if len(r.Steps) == 0 { return time.Time{}, err } if perr := putRoundNode(u.Rounds, v.Round.Round, u.Node, func(rn *RoundNode) { rn.Run = r.Run }); perr != nil { err = errors.Join(err, perr) } if lastStep(r) == StepRefused { return r.Steps[len(r.Steps)-1].At.Add(refusedRetry), err } return time.Time{}, err } // readPass reads the operator's answer to a pass of node from the channel's record, and writes it into the // round once there is one. It answers whether the pass is answered. func (u *Upgrades) readPass(v *roundView, node string) (bool, error) { rn := v.Nodes[node] p := rn.Pass if u.AskRecord == nil { return false, errors.New("no reader of the operator channel's records: a pass cannot be answered") } e, err := u.AskRecord(p.Ask) if err != nil { return false, fmt.Errorf("reading the operator's answer to the pass of %s: %w", node, err) } if e == nil { return false, nil } var rec struct { State string `json:"state"` Warrant *asks.Warrant `json:"warrant"` } if err := json.Unmarshal(e.Value, &rec); err != nil { return false, fmt.Errorf("the channel's record of the pass of %s: %w", node, err) } if rec.State == "" || rec.State == "open" || rec.Warrant == nil { return false, nil } w := *rec.Warrant answer := PassDeclined switch { case w.Outcome == asks.OutcomeExpired: answer = PassExpired case w.Outcome == asks.OutcomeChosen: ask := PassOperatorAsk(v.Round.Round, node, v.next(node), p.Reason, p.Ask, p.Asked) option, err := w.For(Module, ask) if err == nil { err = option.Performs(passAct(v.Round.Round, node, p.Reason, p.Ask)) } if err != nil { return false, fmt.Errorf("the answer to the pass of %s is not the operator's to this pass: %w", node, err) } if option.ID == "pass" { answer = PassApproved } } at := u.Machine.Now().UTC() return true, putRoundNode(u.Rounds, v.Round.Round, node, func(rn *RoundNode) { if rn.Pass != nil && rn.Pass.Ask == p.Ask && rn.Pass.Answer == "" { rn.Pass.Answer, rn.Pass.Answered = answer, &at } }) } // takePass is a pass ask taken by the keeper of the node waiting on the passed node: the pass is written into // the round, waiting for the operator's answer (the tool asked the operator). func (u *Upgrades) takePass(a Ask) error { v, err := readRound(u.Rounds, u.Runs, a.Round) if err != nil { return err } switch { case v == nil: return fmt.Errorf("ask %s passes %s in round %s, which is not open", a.ID, a.Pass, a.Round) case v.next(a.Pass) != u.Node: return fmt.Errorf("ask %s passes %s, on which %s does not wait", a.ID, a.Pass, u.Node) } return putRoundNode(u.Rounds, a.Round, a.Pass, func(rn *RoundNode) { if rn.Pass == nil || rn.Pass.Ask != a.ID { // taken again after a restart: the answer already read stays rn.Pass = &Pass{Reason: a.Reason, Ask: a.ID, Asked: a.At.UTC()} } }) } // NodeInRound is where one node stands in a round, as pacman_round and the health say it. type NodeInRound struct { Node string `json:"node"` State string `json:"state"` Run string `json:"run,omitempty"` Since *time.Time `json:"since,omitempty"` WaitsFor string `json:"waits_for,omitempty"` Pass *Pass `json:"pass,omitempty"` } // The states of a node in a round. const ( InRoundWaiting = "waiting" // the node before it has not passed InRoundTurn = "turn" // its turn came, its run has not started InRoundStalled = "stalled" // its turn came longer than round-start-minutes ago, and it has not started InRoundRunning = "running" // its run goes on InRoundPassed = "passed" // its check passed, or the operator passed it InRoundFailed = "failed" // its run failed or its check did not pass: the round stops here InRoundRefused = "refused" // its run did not start (it is tried again) InRoundOff = "off-order" // not in the round's order ) // RoundState is where every node of the order stands in a round at now. func RoundState(v *roundView, now time.Time, startMinutes int) []NodeInRound { if startMinutes < 1 { startMinutes = defaultRoundStartMinutes } out := []NodeInRound{} for i, node := range v.Round.Order { s := NodeInRound{Node: node, Run: v.Nodes[node].Run, Pass: v.Nodes[node].Pass} at := func(t time.Time) *time.Time { if t.IsZero() { return nil } t = t.UTC() return &t } r, hasRun := v.Runs[node] switch { case !v.passedAt(node).IsZero(): s.State, s.Since = InRoundPassed, at(v.passedAt(node)) case hasRun && lastStep(r) == StepRefused: s.State, s.Since = InRoundRefused, at(r.Steps[len(r.Steps)-1].At) case hasRun && lastStep(r) == StepCheckedAfter: s.State, s.Since = InRoundFailed, at(r.Steps[len(r.Steps)-1].At) case hasRun: s.State = InRoundRunning if len(r.Steps) > 0 { s.Since = at(r.Steps[0].At) } case !v.turnSince(node).IsZero(): since := v.turnSince(node) s.State, s.Since = InRoundTurn, at(since) if now.Sub(since) > time.Duration(startMinutes)*time.Minute { s.State = InRoundStalled } default: s.State = InRoundWaiting for j := i - 1; j >= 0; j-- { if v.passedAt(v.Round.Order[j]).IsZero() { s.WaitsFor = v.Round.Order[j] } } if s.WaitsFor != "" { // since the node it waits for stopped or its turn came prev := out[indexOf(v.Round.Order, s.WaitsFor)] s.Since = prev.Since } } out = append(out, s) } return out } func indexOf(list []string, s string) int { for i, x := range list { if x == s { return i } } return -1 } // seatPublish publishes on a seat for the tools bundle; a test replaces it. var seatPublish = stdio.SeatPublish // ListRounds answers pacman_round's read: the newest rounds, or one, each with every node's state. func ListRounds(rounds, runs Bucket, round string, now time.Time, startMinutes, limit int) (map[string]any, error) { keys, err := rounds.Keys() if err != nil { return nil, err } var ids []string for _, k := range keys { if !strings.Contains(k, ".") && (round == "" || k == round) { ids = append(ids, k) } } if round != "" && len(ids) == 0 { return nil, fmt.Errorf("no round %s", round) } sort.Sort(sort.Reverse(sort.StringSlice(ids))) if len(ids) > limit { ids = ids[:limit] } listed := []map[string]any{} for _, id := range ids { v, err := readRound(rounds, runs, id) if err != nil || v == nil { return nil, err } listed = append(listed, map[string]any{"round": id, "opened": v.Round.Opened, "opened_by": v.Round.OpenedBy, "order": v.Round.Order, "nodes": RoundState(v, now, startMinutes)}) } return map[string]any{"rounds": listed}, nil } // AskPass asks the operator to let a round go on past a node: the ask on the operator's channel, and the ask in // `asks` for the keeper of the node waiting on it, which reads the answer. A reason is required. func AskPass(rounds, runs, asksState Bucket, round, node, reason string, now time.Time) (map[string]any, error) { if strings.TrimSpace(reason) == "" { return nil, errors.New("a pass needs a reason: the operator is asked to let the round go on past a node, and is told why") } if round == "" { id, err := latestRound(rounds) if err != nil { return nil, err } round = id } v, err := readRound(rounds, runs, round) if err != nil { return nil, err } if v == nil { return nil, fmt.Errorf("no round %s is open", round) } if !contains(v.Round.Order, node) { return nil, fmt.Errorf("%s is not in round %s's order (%s)", node, round, strings.Join(v.Round.Order, ", ")) } if !v.passedAt(node).IsZero() { return nil, fmt.Errorf("%s has passed already in round %s", node, round) } next := v.next(node) if next == "" { return nil, fmt.Errorf("%s is the last of the order: no node waits on it", node) } a := Ask{ID: AskID(now), Node: next, Act: ActPass, At: now.UTC(), Round: round, Pass: node, Reason: reason} opAsk := PassOperatorAsk(round, node, next, reason, a.ID, a.At) if err := opAsk.Check(now); err != nil { return nil, err } if err := seatPublish(asks.AskSubject(Module), opAsk, opAsk.ID); err != nil { return nil, fmt.Errorf("the operator could not be asked: %w; nothing was passed", err) } if _, err := asksState.Create(NodeKey(next, a.ID), a); err != nil { return nil, fmt.Errorf("the operator was asked, and the ask for %s's keeper could not be written: %w; an Approve passes nothing", next, err) } return map[string]any{"asked": opAsk.ID, "round": round, "pass": node, "waiting": next, "next": "the round goes on past " + node + " only when the operator approves on their channel; pacman_round says where it stands"}, nil } // roundStartMinutes is round-start-minutes as the tools bundle reads it: from the keeper's config file, which // the manifest names to the bundle too (EnvConfig); its default when the file cannot be read. var roundStartMinutes = func() int { raw, err := os.ReadFile(os.Getenv(EnvConfig)) var c KeeperConfig if err != nil || json.Unmarshal(raw, &c) != nil || c.RoundStartMinutes < 1 { return defaultRoundStartMinutes } return c.RoundStartMinutes }