jochen
d1de0edd4a
A pull request's head announced again from the bus's history at mesh-delivery's first start was proposed with no verdict and nothing ever asked its check: the controller had taken that announcement long before, and stalled raised it after an hour for the operator. A proposed delivery with no verdict and no check asked is now asked through delivery-check once it has waited past a grace longer than a check takes; an announcement carrying its head's decided gate status takes it. The proposed bound runs from the ask, and H2's close may re-ask once (a table row) before the delivery is the operator's.
435 lines
17 KiB
Go
435 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// The state table (novox/hq ADR 0239 decision 2, to-be 47): every transition a delivery may take, each with
|
|
// its guard, compiled once. The code moves a delivery only through Apply, which finds the row for the
|
|
// delivery's state and the event, runs its guard and refuses anything else, naming what was asked. A test
|
|
// walks the table: every row it holds, and every pair it does not.
|
|
|
|
// State is where a delivery is.
|
|
type State string
|
|
|
|
// The states, in the order a delivery passes through them. None is a delivery not yet made: the rows from
|
|
// it are how one comes to exist.
|
|
const (
|
|
None State = ""
|
|
Proposed State = "proposed"
|
|
Checked State = "checked"
|
|
Ready State = "ready"
|
|
Rejected State = "rejected"
|
|
Published State = "published"
|
|
Delivering State = "delivering"
|
|
Held State = "held"
|
|
Delivered State = "delivered"
|
|
Failed State = "failed"
|
|
Superseded State = "superseded"
|
|
Stopped State = "stopped"
|
|
)
|
|
|
|
// AllStates are every state a delivery can be in, for the table's own test and its verb.
|
|
var AllStates = []State{Proposed, Checked, Ready, Rejected, Published, Delivering, Held, Delivered, Failed,
|
|
Superseded, Stopped}
|
|
|
|
// Final is whether nothing follows a state.
|
|
func (s State) Final() bool {
|
|
return s == Delivered || s == Failed || s == Superseded || s == Stopped
|
|
}
|
|
|
|
// Event is what moves a delivery.
|
|
type Event string
|
|
|
|
// The events. Observed ones are taken by settle whenever their guard holds; the others are acts — a
|
|
// person's, or healer H2's — taken only when asked.
|
|
const (
|
|
EvAnnounced Event = "announced" // the forge announced a pull request's head
|
|
EvAppeared Event = "appeared" // a trunk commit the mesh is delivering, with no pull request known
|
|
EvAdopted Event = "adopted" // a walk already running at the switch, or on the controller's own path
|
|
EvChecked Event = "checked" // its check's verdict arrived
|
|
EvAccepted Event = "accepted" // the verdict passes
|
|
EvRefused Event = "refused" // the verdict fails, or the check could not run
|
|
EvRecheck Event = "recheck" // a person asked for it again, or its group changed
|
|
EvReask Event = "re-ask" // healer H2 asked a check asked and not answered once more
|
|
EvNewHead Event = "new-head" // a newer head of the same pull request
|
|
EvClosed Event = "closed" // the pull request closed unmerged
|
|
EvMerged Event = "merged" // it reached the trunk
|
|
EvMergedUnchecked Event = "merged-unchecked" // it reached the trunk without a passing check
|
|
EvGo Event = "go" // its walk started
|
|
EvHold Event = "hold" // something holds it for a person
|
|
EvRelease Event = "release" // a person's word that it goes on
|
|
EvDone Event = "done" // every machine of its deploy plan took it, or nothing was to be sent
|
|
EvFailed Event = "failed" // its walk failed: a gate, a build, a machine
|
|
EvSuperseded Event = "superseded" // a newer delivery to the same trunk took over its walk
|
|
EvStop Event = "stop" // a person stopped it, or its group did
|
|
)
|
|
|
|
// Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act.
|
|
type Facts struct {
|
|
Now time.Time
|
|
By string
|
|
Why string
|
|
}
|
|
|
|
// Row is one transition the table holds.
|
|
type Row struct {
|
|
From []State
|
|
Event Event
|
|
To State
|
|
// Guard says, in words, what must be true; Holds checks it, answering why not.
|
|
Guard string
|
|
Holds func(d *Delivery, f Facts) error
|
|
// Act says the row is taken only when asked — a person's or H2's — never by settle.
|
|
Act bool
|
|
}
|
|
|
|
func unless(cond bool, why string) error {
|
|
if cond {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%s", why)
|
|
}
|
|
|
|
var notFinal = []State{Proposed, Checked, Ready, Rejected, Published, Delivering, Held}
|
|
|
|
// Table is every transition there is. The order within one state is the order settle tries them.
|
|
var Table = []Row{
|
|
{From: []State{None}, Event: EvAnnounced, To: Proposed,
|
|
Guard: "a pull request's head the forge announced",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Number > 0 && d.Commit != "", "no pull request's head")
|
|
}},
|
|
{From: []State{None}, Event: EvAppeared, To: Held,
|
|
Guard: "a commit on the trunk, its walk waiting, and no pull request known: it waits for a person",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.MergedAs != "" && d.HeldWhy != "", "no trunk commit, or nothing says why it is held")
|
|
}},
|
|
{From: []State{None}, Event: EvAdopted, To: Delivering,
|
|
Guard: "a walk already running: at the switch, or on the controller's own path",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Walk != nil && d.Walk.Started(), "no running walk")
|
|
}},
|
|
|
|
{From: []State{Proposed, Checked, Ready, Rejected}, Event: EvNewHead, To: Superseded,
|
|
Guard: "a newer head of the same pull request",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(d.NewerHead != "", "no newer head") }},
|
|
{From: []State{Proposed, Checked, Ready, Rejected}, Event: EvClosed, To: Stopped,
|
|
Guard: "the pull request closed unmerged",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(d.ClosedUnmerged, "the pull request is open") }},
|
|
{From: []State{Ready}, Event: EvMerged, To: Published,
|
|
Guard: "on the trunk its modules follow: merged there, and its walk opened — or nothing for a walk to move",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.MergedAs != "" && (d.Walk != nil || d.MovesNothing()), "not on the trunk yet")
|
|
}},
|
|
{From: []State{Proposed}, Event: EvChecked, To: Checked,
|
|
Guard: "the verdict names this commit",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(d.Check != nil, "no verdict") }},
|
|
{From: []State{Checked}, Event: EvAccepted, To: Ready,
|
|
Guard: "the gate passed or warned, and the repository's own check did not fail",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(d.Check.Passes(), "the verdict does not pass") }},
|
|
{From: []State{Checked}, Event: EvRefused, To: Rejected,
|
|
Guard: "the gate failed or could not run, or the repository's own check failed",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(!d.Check.Passes(), "the verdict passes") }},
|
|
{From: []State{Proposed, Checked, Rejected}, Event: EvMergedUnchecked, To: Held,
|
|
Guard: "merged without a passing check — a verdict known with it is taken first: only a person decides that it goes on",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(d.MergedAs != "", "not merged") }},
|
|
{From: []State{Rejected, Ready}, Event: EvRecheck, To: Proposed, Act: true,
|
|
Guard: "a person asked, with why, or its group changed",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "", "a recheck says why") }},
|
|
{From: []State{Proposed}, Event: EvReask, To: Proposed, Act: true,
|
|
Guard: "its check was asked and no verdict came within its bound, and it was not re-asked before: healer H2 " +
|
|
"asks it once more, with why — after that it is the operator's",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
switch {
|
|
case f.Why == "" || f.By == "":
|
|
return errors.New("a re-ask says who and why")
|
|
case d.Check != nil:
|
|
return errors.New("its verdict is known")
|
|
case d.CheckAsk == nil:
|
|
return errors.New("its check was never asked: the tick asks it, not H2")
|
|
case d.CheckAsk.Reasked > 0:
|
|
return errors.New("its check was re-asked once already: the operator's")
|
|
}
|
|
return nil
|
|
}},
|
|
|
|
{From: []State{Published}, Event: EvSuperseded, To: Superseded,
|
|
Guard: "a newer delivery to the same trunk took over its walk",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded")
|
|
}},
|
|
{From: []State{Published}, Event: EvDone, To: Delivered,
|
|
Guard: "nothing for a walk to move",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(d.Walk == nil && d.MovesNothing(), "a walk moves it") }},
|
|
{From: []State{Published, Ready}, Event: EvHold, To: Held,
|
|
Guard: "something holds it for a person: its group's composed check did not pass for the heads that merged, " +
|
|
"or it merged and no walk was opened for it",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.HeldWhy != "" && d.MergedAs != "", "nothing holds it, or it is not on the trunk")
|
|
}},
|
|
{From: []State{Published}, Event: EvGo, To: Delivering,
|
|
Guard: "its walk started: let go by this owner in its group's order, by a person, or on the controller's own path",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(d.Walk != nil && d.Walk.Started(), "its walk waits") }},
|
|
{From: []State{Held}, Event: EvGo, To: Delivering,
|
|
Guard: "its walk started on a word that was not this owner's: the controller's own path, which waits for " +
|
|
"nobody, or a person's `plans go`",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Walk != nil && d.Walk.Started() && (!d.Walk.Waited() || d.Walk.LetGoByAPerson()),
|
|
"its walk was not started by a person or the controller's own path")
|
|
}},
|
|
{From: []State{Held}, Event: EvRelease, To: Delivering, Act: true,
|
|
Guard: "a person's decision, with why",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(f.Why != "" && f.By != "", "a release is a person's word, with why")
|
|
}},
|
|
|
|
{From: []State{Delivering}, Event: EvSuperseded, To: Superseded,
|
|
Guard: "a newer delivery to the same trunk took over its walk",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded")
|
|
}},
|
|
{From: []State{Delivering}, Event: EvDone, To: Delivered,
|
|
Guard: "its walk is done: every machine of its deploy plan passed, or was left as its policy says",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Walk != nil && d.Walk.State == walkDone, "its walk is not done")
|
|
}},
|
|
{From: []State{Delivering}, Event: EvStop, To: Stopped,
|
|
Guard: "its walk was stopped through this owner",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy != "", "its walk was not stopped")
|
|
}},
|
|
{From: []State{Delivering}, Event: EvFailed, To: Failed,
|
|
Guard: "its walk failed: a gate on a first machine (what it carried put back), a build, a machine",
|
|
Holds: func(d *Delivery, f Facts) error {
|
|
return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "", "its walk did not fail")
|
|
}},
|
|
|
|
{From: notFinal, Event: EvStop, To: Stopped, Act: true,
|
|
Guard: "a person, with why, or its group stopped by a member before it",
|
|
Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "" && f.By != "", "a stop says who and why") }},
|
|
}
|
|
|
|
// rowFor is the row a state and event name; nil when the table holds none.
|
|
func rowFor(from State, ev Event, act bool) *Row {
|
|
for i := range Table {
|
|
r := &Table[i]
|
|
if r.Event != ev || r.Act != act {
|
|
continue
|
|
}
|
|
for _, s := range r.From {
|
|
if s == from {
|
|
return r
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ErrRefused is a transition the table does not hold, or whose guard does not.
|
|
type ErrRefused struct {
|
|
ID string
|
|
From State
|
|
Event Event
|
|
Why string
|
|
}
|
|
|
|
func (e ErrRefused) Error() string {
|
|
from := string(e.From)
|
|
if from == "" {
|
|
from = "nothing"
|
|
}
|
|
return fmt.Sprintf("%s: %s from %s is refused — %s", e.ID, e.Event, from, e.Why)
|
|
}
|
|
|
|
// Apply takes one transition, or refuses it. The delivery keeps the transition, bounded.
|
|
func Apply(d *Delivery, ev Event, act bool, f Facts) (Transition, error) {
|
|
r := rowFor(d.State, ev, act)
|
|
if r == nil {
|
|
kind := "observed"
|
|
if act {
|
|
kind = "asked"
|
|
}
|
|
return Transition{}, ErrRefused{ID: d.ID, From: d.State, Event: ev,
|
|
Why: "the table holds no such transition (" + kind + ")"}
|
|
}
|
|
if err := r.Holds(d, f); err != nil {
|
|
return Transition{}, ErrRefused{ID: d.ID, From: d.State, Event: ev, Why: "its guard does not hold: " + err.Error()}
|
|
}
|
|
why := f.Why
|
|
if why == "" {
|
|
why = r.Guard
|
|
}
|
|
t := Transition{At: f.Now, From: d.State, To: r.To, Event: ev, Why: why, By: f.By}
|
|
d.State, d.Since = r.To, f.Now
|
|
d.Transitions = append(d.Transitions, t)
|
|
if len(d.Transitions) > keptTransitions {
|
|
d.Transitions = d.Transitions[len(d.Transitions)-keptTransitions:]
|
|
}
|
|
return t, nil
|
|
}
|
|
|
|
// keptTransitions is how many transitions a delivery keeps; its note on the commit keeps every one.
|
|
const keptTransitions = 100
|
|
|
|
// settle takes every observed transition whose guard holds, in the table's order, until none does — so a
|
|
// delivery always stands where its facts put it, whatever order they arrived in.
|
|
func settle(d *Delivery, f Facts) []Transition {
|
|
var taken []Transition
|
|
for range len(Table) + 1 {
|
|
moved := false
|
|
for i := range Table {
|
|
r := &Table[i]
|
|
if r.Act || !contains(r.From, d.State) {
|
|
continue
|
|
}
|
|
if t, err := Apply(d, r.Event, false, f); err == nil {
|
|
taken = append(taken, t)
|
|
moved = true
|
|
break
|
|
}
|
|
}
|
|
if !moved {
|
|
break
|
|
}
|
|
}
|
|
return taken
|
|
}
|
|
|
|
func contains(states []State, s State) bool {
|
|
for _, x := range states {
|
|
if x == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Bounds are how long a delivery may be in a state before `stalled` lists it, and what healer H2 may do
|
|
// then — always a transition the table holds, never one of its own (novox/hq ADR 0239 decision 9).
|
|
type Bound struct {
|
|
For time.Duration
|
|
// H2 is the transitions H2 may take after the bound, by event; empty when the state is the operator's.
|
|
H2 []Event
|
|
Says string
|
|
}
|
|
|
|
// Bounds are every state's.
|
|
var Bounds = map[State]Bound{
|
|
Proposed: {For: time.Hour, H2: []Event{EvReask}, Says: "its check was asked and not answered: H2 asks it once " +
|
|
"more, then the check's own watchdog (S6) and the operator speak for it"},
|
|
Checked: {For: time.Minute, Says: "a verdict is decided at once"},
|
|
Published: {For: 30 * time.Minute, H2: []Event{EvSuperseded, EvGo}, Says: "its walk waits for its turn or its word"},
|
|
Held: {For: 24 * time.Hour, Says: "it waits for the operator"},
|
|
Delivering: {For: 2 * time.Hour, H2: []Event{EvDone, EvSuperseded, EvFailed}, Says: "its walk runs"},
|
|
}
|
|
|
|
// checkGrace is how long a proposed delivery is left to the controller's own check of its announcement before
|
|
// this owner asks the check itself: longer than a check takes from announcement to verdict. A head adopted while
|
|
// it already waited — announced again from the bus's history — is asked once it has waited this long too.
|
|
const checkGrace = 15 * time.Minute
|
|
|
|
// boundSince is when a delivery's bound runs from: its state's start, except for a proposed delivery whose
|
|
// check this owner asked — its bound is "asked and not answered", so it runs from the ask. One never asked
|
|
// runs from its start plus the grace before it is asked: an ask that cannot be made is still listed.
|
|
func boundSince(d *Delivery) time.Time {
|
|
if d.State == Proposed {
|
|
if d.CheckAsk != nil {
|
|
return d.CheckAsk.At
|
|
}
|
|
return d.Since.Add(checkGrace)
|
|
}
|
|
return d.Since
|
|
}
|
|
|
|
// pastBound is whether a delivery has been where it is longer than its state's bound.
|
|
func pastBound(d *Delivery, now time.Time) bool {
|
|
b, bounded := Bounds[d.State]
|
|
return bounded && !d.State.Final() && now.Sub(boundSince(d)) > b.For
|
|
}
|
|
|
|
// StepState is where one machine stands in a delivering walk.
|
|
type StepState string
|
|
|
|
// The machine steps (ADR 0239 decision 2): sent, judged, passed or failed, and put back.
|
|
const (
|
|
StepSent StepState = "sent"
|
|
StepJudging StepState = "judging"
|
|
StepPassed StepState = "passed"
|
|
StepFailed StepState = "failed"
|
|
StepRolledBack StepState = "rolled-back"
|
|
)
|
|
|
|
// StepTable is every move a machine's step may make. From "" is a step first heard of.
|
|
var StepTable = map[StepState][]StepState{
|
|
"": {StepSent, StepJudging, StepPassed, StepFailed, StepRolledBack},
|
|
StepSent: {StepJudging, StepPassed, StepFailed},
|
|
StepJudging: {StepPassed, StepFailed},
|
|
StepFailed: {StepRolledBack},
|
|
}
|
|
|
|
// StepMay is whether a step may move from one state to another: directly, or through states the walk's
|
|
// record passed between two readings of it (sent, judged and failed and put back, read once).
|
|
func StepMay(from, to StepState) bool {
|
|
if from == to {
|
|
return true
|
|
}
|
|
seen := map[StepState]bool{}
|
|
next := []StepState{from}
|
|
for len(next) > 0 {
|
|
s := next[0]
|
|
next = next[1:]
|
|
for _, t := range StepTable[s] {
|
|
if t == to {
|
|
return true
|
|
}
|
|
if !seen[t] {
|
|
seen[t] = true
|
|
next = append(next, t)
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// TableText is the table as the `table` verb answers it.
|
|
func TableText() map[string]any {
|
|
var rows []map[string]any
|
|
for _, r := range Table {
|
|
from := make([]string, 0, len(r.From))
|
|
for _, s := range r.From {
|
|
if s == None {
|
|
from = append(from, "(none)")
|
|
continue
|
|
}
|
|
from = append(from, string(s))
|
|
}
|
|
kind := "observed"
|
|
if r.Act {
|
|
kind = "asked"
|
|
}
|
|
rows = append(rows, map[string]any{"from": strings.Join(from, ", "), "event": r.Event, "to": r.To,
|
|
"guard": r.Guard, "taken": kind})
|
|
}
|
|
bounds := map[string]any{}
|
|
for s, b := range Bounds {
|
|
var h2 []string
|
|
for _, e := range b.H2 {
|
|
h2 = append(h2, string(e))
|
|
}
|
|
bounds[string(s)] = map[string]any{"bound": b.For.String(), "h2": h2, "says": b.Says}
|
|
}
|
|
steps := map[string][]StepState{}
|
|
for from, to := range StepTable {
|
|
name := string(from)
|
|
if name == "" {
|
|
name = "(first heard)"
|
|
}
|
|
steps[name] = to
|
|
}
|
|
return map[string]any{"transitions": rows, "bounds": bounds, "machine-steps": steps}
|
|
}
|