jschoubben
60c7ffc981
mesh-grants moves to the same mesh-sdk (3203358c), so its shadow user list grants the checker and the builder the DEAD_LETTERS reads the controller's list will, and the take-over is not blocked by a difference. The checker and the builder hear DEAD_LETTERS through buildrun.DeadLetterHearing: in the background, again at every reconnect and every ask waited for while deaf, and keep whether they hear it in their new state hearing, which checker_status and builder_stuck say as dead_letters. checker_try says that a try ended by the not-taken bound leaves its ask in the stream. Not done as asked: a module health tool. The catalogue refuses a tool check on a module with no runtime of its own and no other check beside it (ADR 0227 rule 8, ADR 0366 §1), so the deaf state is said in the status tools and the log.
565 lines
20 KiB
Go
565 lines
20 KiB
Go
package check
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
)
|
|
|
|
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
|
// status it owes in full, so a check is read live without a shell. They write nothing.
|
|
|
|
// The read tools, named as the module is: checker_ and the noun.
|
|
const (
|
|
VerdictsReadTool = "checker_verdicts"
|
|
PullReadTool = "checker_pull"
|
|
KeptReadTool = "checker_kept"
|
|
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker runs (issue 1685: nothing holds it since R4g).
|
|
PreviewsReadTool = "checker_previews"
|
|
StatusReadTool = "checker_status"
|
|
)
|
|
|
|
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
|
|
type StateRead struct {
|
|
Keys func() ([]string, error)
|
|
Get func(key string) (json.RawMessage, error)
|
|
}
|
|
|
|
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
|
|
type Reader struct {
|
|
VerdictState, KeptState, CandidateState StateRead
|
|
|
|
PreviewState StateRead // the checker's merge previews
|
|
StepStates map[string]StateRead // the build modules' steps states, by module
|
|
HaveState StateRead // the node-engine's have: whether a build module is on a node and running
|
|
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
|
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
|
Trunks func() (map[string]string, error)
|
|
// TurnState is the gate's merge turns (ADR 0548 §3); without it no check is named as held by a turn.
|
|
TurnState StateRead
|
|
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
|
|
// node-engine a verdict names is compared at its running commit, not at the trunk's (ADR 0548 §6); nil knows none.
|
|
Runs interface{ CommitOf(module string) string }
|
|
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
|
|
HearingState StateRead
|
|
Now func() time.Time
|
|
}
|
|
|
|
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
|
// candidate still holds those heads; without it the report is that of the verdict alone.
|
|
type VerdictRead struct {
|
|
Key string `json:"key"`
|
|
Verdict json.RawMessage `json:"verdict"`
|
|
Reports map[string]string `json:"reports,omitempty"`
|
|
Note string `json:"note,omitempty"`
|
|
}
|
|
|
|
func (r Reader) candidate(group string) (delivery.Candidate, bool) {
|
|
key, err := delivery.CandidatesKey(group)
|
|
if err != nil || r.CandidateState.Get == nil {
|
|
return delivery.Candidate{}, false
|
|
}
|
|
raw, err := r.CandidateState.Get(key)
|
|
if err != nil || len(raw) == 0 {
|
|
return delivery.Candidate{}, false
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
return c, err == nil
|
|
}
|
|
|
|
func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
|
out := VerdictRead{Key: key, Verdict: raw}
|
|
v, err := delivery.DecodeVerdict(raw)
|
|
if err != nil {
|
|
return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
|
|
}
|
|
c, ok := r.candidate(v.Group)
|
|
if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) {
|
|
out.Note = "the candidate no longer holds these heads"
|
|
return out, nil
|
|
}
|
|
cs := CaseOf(c, v.Trunk)
|
|
out.Reports = map[string]string{}
|
|
if cs == nil {
|
|
return out, nil
|
|
}
|
|
for _, p := range cs.Pulls {
|
|
out.Reports[pullKey(p)] = Report(v, cs, p, nil, nil, nil)
|
|
}
|
|
for _, l := range cs.Left {
|
|
out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group
|
|
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there.
|
|
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
|
|
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
|
return nil, "", fmt.Errorf("the verdicts state cannot be read here")
|
|
}
|
|
keys, err := r.VerdictState.Keys()
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
sort.Strings(keys)
|
|
var matching []string
|
|
for _, k := range keys {
|
|
if group != "" {
|
|
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
|
continue
|
|
}
|
|
}
|
|
matching = append(matching, k)
|
|
}
|
|
matching, cut = capKeys(matching, "give a group to narrow the answer")
|
|
rows := make([]VerdictRead, len(matching))
|
|
found := make([]bool, len(matching))
|
|
if err := parallel(len(matching), func(i int) error {
|
|
raw, err := r.VerdictState.Get(matching[i])
|
|
if err != nil || len(raw) == 0 {
|
|
return err
|
|
}
|
|
rows[i], err = r.read(matching[i], raw)
|
|
found[i] = err == nil
|
|
return err
|
|
}); err != nil {
|
|
return nil, "", err
|
|
}
|
|
for i, row := range rows {
|
|
if found[i] {
|
|
out = append(out, row)
|
|
}
|
|
}
|
|
return out, cut, nil
|
|
}
|
|
|
|
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
|
|
// in no candidate, that none does.
|
|
func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
|
if r.CandidateState.Keys == nil || r.CandidateState.Get == nil {
|
|
return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here")
|
|
}
|
|
keys, err := r.CandidateState.Keys()
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
sort.Strings(keys)
|
|
keys, cut := capKeys(keys, "")
|
|
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
|
|
for from := 0; from < len(keys); from += ReadWorkers * 4 {
|
|
batch := keys[from:min(from+ReadWorkers*4, len(keys))]
|
|
cands := make([]delivery.Candidate, len(batch))
|
|
ok := make([]bool, len(batch))
|
|
_ = parallel(len(batch), func(i int) error {
|
|
raw, err := r.CandidateState.Get(batch[i])
|
|
if err != nil || len(raw) == 0 {
|
|
return nil
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
cands[i], ok[i] = c, err == nil
|
|
return nil
|
|
})
|
|
for i, c := range cands {
|
|
if !ok[i] {
|
|
continue
|
|
}
|
|
for _, p := range c.Pulls {
|
|
if p.Repository != repository || p.Number != number {
|
|
continue
|
|
}
|
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
vraw, err := r.VerdictState.Get(vkey)
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
if len(vraw) == 0 {
|
|
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
|
}
|
|
return r.read(vkey, vraw)
|
|
}
|
|
}
|
|
}
|
|
if cut != "" {
|
|
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
|
|
}
|
|
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
|
}
|
|
|
|
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
|
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) {
|
|
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
|
return nil, "", fmt.Errorf("the kept state cannot be read here")
|
|
}
|
|
if digest != "" {
|
|
key, err := delivery.KeptKey(level, digest)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
raw, err := r.KeptState.Get(key)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if len(raw) == 0 {
|
|
return nil, "", fmt.Errorf("no kept verdict %s", key)
|
|
}
|
|
return map[string]json.RawMessage{key: raw}, "", nil
|
|
}
|
|
keys, err := r.KeptState.Keys()
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
sort.Strings(keys)
|
|
var matching []string
|
|
for _, k := range keys {
|
|
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
|
|
matching = append(matching, k)
|
|
}
|
|
}
|
|
matching, cut := capKeys(matching, "give a level and a digest to read one")
|
|
values := make([]json.RawMessage, len(matching))
|
|
if err := parallel(len(matching), func(i int) (err error) {
|
|
values[i], err = r.KeptState.Get(matching[i])
|
|
return err
|
|
}); err != nil {
|
|
return nil, "", err
|
|
}
|
|
out := map[string]json.RawMessage{}
|
|
for i, k := range matching {
|
|
if len(values[i]) > 0 {
|
|
out[k] = values[i]
|
|
}
|
|
}
|
|
return out, cut, nil
|
|
}
|
|
|
|
// PreviewRead is one preview as stored, with the parts of its key.
|
|
type PreviewRead struct {
|
|
Key string `json:"key"`
|
|
Repository string `json:"repository"`
|
|
Head string `json:"head"`
|
|
Trunk string `json:"trunk"`
|
|
Preview json.RawMessage `json:"preview"`
|
|
}
|
|
|
|
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
|
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
|
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) {
|
|
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
|
return nil, nil, "", fmt.Errorf("the previews state cannot be read here")
|
|
}
|
|
keys, err := r.PreviewState.Keys()
|
|
if err != nil {
|
|
return nil, nil, "", err
|
|
}
|
|
sort.Strings(keys)
|
|
type part struct{ repo, head, trunk string }
|
|
parts := map[string]part{}
|
|
var matching, bad []string
|
|
for _, k := range keys {
|
|
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
|
if err != nil {
|
|
bad = append(bad, k)
|
|
continue
|
|
}
|
|
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
|
continue
|
|
}
|
|
parts[k] = part{repo, h, trunk}
|
|
matching = append(matching, k)
|
|
}
|
|
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer")
|
|
values := make([]json.RawMessage, len(matching))
|
|
if err := parallel(len(matching), func(i int) (err error) {
|
|
values[i], err = r.PreviewState.Get(matching[i])
|
|
return err
|
|
}); err != nil {
|
|
return nil, nil, "", err
|
|
}
|
|
out := []PreviewRead{}
|
|
for i, k := range matching {
|
|
if len(values[i]) > 0 {
|
|
p := parts[k]
|
|
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
|
|
}
|
|
}
|
|
return out, bad, cut, nil
|
|
}
|
|
|
|
// RunningStep is a step in flight on a build module.
|
|
type RunningStep struct {
|
|
Module string `json:"module"`
|
|
Node string `json:"node"`
|
|
ID string `json:"id"`
|
|
Tool string `json:"tool"`
|
|
Since string `json:"since,omitempty"`
|
|
}
|
|
|
|
// LeftStep is a key a build module left running that no step behind it runs any more, and why.
|
|
type LeftStep struct {
|
|
RunningStep
|
|
Why string `json:"why"`
|
|
}
|
|
|
|
// Status is what runs now, read from the buckets: the steps in flight on the build modules and the candidates without a
|
|
// verdict for their heads. Nothing holds a check since R4g: a check is an ask on the build seats' streams, whose backlog
|
|
// the delivery view shows (ADR 0525 §9).
|
|
type Status struct {
|
|
Running []RunningStep `json:"running_steps"`
|
|
LeftRunning []LeftStep `json:"left_running"`
|
|
RunningByNode map[string]int `json:"running_by_node"`
|
|
Unjudged []string `json:"unjudged_candidates"`
|
|
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
|
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
|
Waiting map[string][]string `json:"waiting_by_kind"`
|
|
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
|
|
// the group of the merge that waits. Their check is claimed when that turn goes.
|
|
HeldByTurn map[string]string `json:"held_by_turn"`
|
|
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
|
|
// whose ask is given up waits until its bound.
|
|
DeadLetters DeadLettersHeard `json:"dead_letters"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// DeadLettersHeard is the checker's hearing of DEAD_LETTERS as the status says it: healthy or not, and why.
|
|
type DeadLettersHeard struct {
|
|
Heard bool `json:"heard"`
|
|
Why string `json:"why"`
|
|
}
|
|
|
|
// deadLetters reads the checker's kept hearing (buildrun.HearingHealth).
|
|
func (r Reader) deadLetters() DeadLettersHeard {
|
|
if r.HearingState.Get == nil {
|
|
return DeadLettersHeard{Why: "the checker's state hearing is not readable here"}
|
|
}
|
|
raw, err := r.HearingState.Get(buildrun.HearingKey)
|
|
if err != nil {
|
|
return DeadLettersHeard{Why: "the checker's state hearing cannot be read: " + err.Error()}
|
|
}
|
|
a := buildrun.HearingHealth(raw)
|
|
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
|
|
}
|
|
|
|
// turns reads the gate's merge turns, by repository; nil when they cannot be read, said in a note.
|
|
func (r Reader) turns(st *Status) *Turns {
|
|
if r.TurnState.Keys == nil || r.TurnState.Get == nil {
|
|
st.Notes = append(st.Notes, "the gate's merge turns are not readable here, so no check is named as held by one")
|
|
return nil
|
|
}
|
|
keys, err := r.TurnState.Keys()
|
|
if err != nil {
|
|
st.Notes = append(st.Notes, "the gate's merge turns cannot be listed, so no check is named as held by one: "+err.Error())
|
|
return nil
|
|
}
|
|
t := &Turns{}
|
|
for _, k := range keys {
|
|
if raw, err := r.TurnState.Get(k); err == nil && len(raw) > 0 {
|
|
t.Put(k, raw, false)
|
|
}
|
|
}
|
|
return t
|
|
}
|
|
|
|
// heldBy says the turn that holds a candidate's check off, or "".
|
|
func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
|
|
cs := CaseOf(c, heads)
|
|
if t == nil || cs == nil {
|
|
return ""
|
|
}
|
|
repos := make([]string, 0, len(cs.Trunk))
|
|
for r := range cs.Trunk {
|
|
repos = append(repos, r)
|
|
}
|
|
if repo, w, ok := t.Holding(repos, cs.Heads); ok {
|
|
return repo + ": the approved merge of " + w.Group + " waits on it"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Status reads the build modules' steps and the candidates against their verdicts.
|
|
func (r Reader) Status() (Status, error) {
|
|
st := Status{RunningByNode: map[string]int{}}
|
|
mods := make([]string, 0, len(r.StepStates))
|
|
for m := range r.StepStates {
|
|
mods = append(mods, m)
|
|
}
|
|
sort.Strings(mods)
|
|
st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{}
|
|
now, bound := time.Now(), r.StepBound
|
|
if r.Now != nil {
|
|
now = r.Now()
|
|
}
|
|
if bound <= 0 {
|
|
bound = DefaultStepBound
|
|
}
|
|
for _, m := range mods {
|
|
sr := r.StepStates[m]
|
|
if sr.Keys == nil || sr.Get == nil {
|
|
continue
|
|
}
|
|
keys, err := sr.Keys()
|
|
if err != nil {
|
|
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
|
continue
|
|
}
|
|
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
|
|
newestFirst(keys)
|
|
keys, cut := capKeys(keys, "")
|
|
if cut != "" {
|
|
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
|
|
}
|
|
steps := make([]delivery.Step, len(keys))
|
|
isRunning := make([]bool, len(keys))
|
|
_ = parallel(len(keys), func(i int) error {
|
|
raw, err := sr.Get(keys[i])
|
|
if err != nil || len(raw) == 0 {
|
|
return nil
|
|
}
|
|
step, err := delivery.DecodeStep(raw)
|
|
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
|
|
return nil
|
|
})
|
|
for i := range keys {
|
|
if !isRunning[i] {
|
|
continue
|
|
}
|
|
step := steps[i]
|
|
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
|
if !step.Started.IsZero() {
|
|
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
|
}
|
|
if why := r.leftWhy(m, step, now, bound); why != "" {
|
|
st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why})
|
|
continue
|
|
}
|
|
st.Running = append(st.Running, rs)
|
|
st.RunningByNode[step.Node]++
|
|
}
|
|
}
|
|
if r.HaveState.Get == nil {
|
|
st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node")
|
|
}
|
|
if len(st.LeftRunning) > 0 {
|
|
st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them")
|
|
}
|
|
st.Unjudged = []string{}
|
|
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
|
st.HeldByTurn = map[string]string{}
|
|
turns := r.turns(&st)
|
|
var heads map[string]string
|
|
if r.Trunks == nil {
|
|
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
|
} else if h, err := r.Trunks(); err != nil {
|
|
st.Notes = append(st.Notes, "the trunk heads cannot be read, so no check is named as waiting for a trunk move: "+err.Error())
|
|
} else {
|
|
heads = h
|
|
}
|
|
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
|
|
keys, err := r.CandidateState.Keys()
|
|
if err != nil {
|
|
return st, err
|
|
}
|
|
sort.Strings(keys)
|
|
total := len(keys)
|
|
keys, cut := capKeys(keys, "")
|
|
if cut != "" {
|
|
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
|
|
}
|
|
type judged struct {
|
|
c delivery.Candidate
|
|
raw json.RawMessage
|
|
ok bool
|
|
}
|
|
rows := make([]judged, len(keys))
|
|
_ = parallel(len(keys), func(i int) error {
|
|
raw, err := r.CandidateState.Get(keys[i])
|
|
if err != nil || len(raw) == 0 {
|
|
return nil
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
rows[i].c = c
|
|
if vraw, err := r.VerdictState.Get(vkey); err == nil {
|
|
rows[i].raw, rows[i].ok = vraw, true
|
|
}
|
|
return nil
|
|
})
|
|
for _, row := range rows {
|
|
if !row.ok {
|
|
continue
|
|
}
|
|
c, raw := row.c, row.raw
|
|
if len(raw) == 0 {
|
|
st.Unjudged = append(st.Unjudged, c.Group)
|
|
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
|
if why := heldBy(turns, c, heads); why != "" {
|
|
st.HeldByTurn[c.Group] = why
|
|
}
|
|
continue
|
|
}
|
|
v, err := delivery.DecodeVerdict(raw)
|
|
switch {
|
|
case err != nil || v.State == delivery.VerdictRunning:
|
|
case IsNodesFailure(v):
|
|
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
|
case heads != nil:
|
|
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
|
|
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
|
if why := heldBy(turns, c, heads); why != "" {
|
|
st.HeldByTurn[c.Group] = why
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
st.DeadLetters = r.deadLetters()
|
|
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
|
return st, nil
|
|
}
|
|
|
|
// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch,
|
|
// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node.
|
|
const DefaultStepBound = 30 * time.Minute
|
|
|
|
// leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module
|
|
// is not on its node or is not running there (the node-engine's have), or it started longer ago than the step bound, when
|
|
// the checker stopped waiting for it.
|
|
func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound time.Duration) string {
|
|
if r.HaveState.Get != nil {
|
|
key, err := delivery.HaveKey(step.Node, module)
|
|
if err == nil {
|
|
raw, err := r.HaveState.Get(key)
|
|
switch {
|
|
case err != nil:
|
|
case len(raw) == 0:
|
|
return fmt.Sprintf("%s is not on %s", module, step.Node)
|
|
default:
|
|
if h, err := delivery.DecodeHave(raw); err == nil && h.Phase != delivery.PhaseRunning {
|
|
return fmt.Sprintf("%s on %s is %s", module, step.Node, h.Phase)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if step.Started.IsZero() {
|
|
return fmt.Sprintf("it says no start, so no wait of the checker's (bound %s) can be behind it", bound)
|
|
}
|
|
if end := step.Started.Add(bound); now.After(end) {
|
|
return fmt.Sprintf("started %s, past the step bound of %s: the checker stopped waiting for it at %s", step.Started.UTC().Format(time.RFC3339), bound, end.UTC().Format(time.RFC3339))
|
|
}
|
|
return ""
|
|
}
|