432 lines
14 KiB
Go
432 lines
14 KiB
Go
package check
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
|
// status it owes in full, so a check is read live without a shell. They write nothing.
|
|
|
|
// The read tools, named as the module is: checker_ and the noun.
|
|
const (
|
|
VerdictsReadTool = "checker_verdicts"
|
|
PullReadTool = "checker_pull"
|
|
KeptReadTool = "checker_kept"
|
|
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685).
|
|
PreviewsReadTool = "checker_previews"
|
|
StatusReadTool = "checker_status"
|
|
)
|
|
|
|
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
|
|
type StateRead struct {
|
|
Keys func() ([]string, error)
|
|
Get func(key string) (json.RawMessage, error)
|
|
}
|
|
|
|
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
|
|
type Reader struct {
|
|
VerdictState, KeptState, CandidateState StateRead
|
|
|
|
PreviewState StateRead // the checker's merge previews
|
|
StepStates map[string]StateRead // the build modules' steps states, by module
|
|
HaveState StateRead // the node-engine's have: whether a build module is on a node and running
|
|
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
|
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
|
Trunks func() (map[string]string, error)
|
|
Now func() time.Time
|
|
}
|
|
|
|
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
|
// candidate still holds those heads; without it the report is that of the verdict alone.
|
|
type VerdictRead struct {
|
|
Key string `json:"key"`
|
|
Verdict json.RawMessage `json:"verdict"`
|
|
Reports map[string]string `json:"reports,omitempty"`
|
|
Note string `json:"note,omitempty"`
|
|
}
|
|
|
|
func (r Reader) candidate(group string) (delivery.Candidate, bool) {
|
|
key, err := delivery.CandidatesKey(group)
|
|
if err != nil || r.CandidateState.Get == nil {
|
|
return delivery.Candidate{}, false
|
|
}
|
|
raw, err := r.CandidateState.Get(key)
|
|
if err != nil || len(raw) == 0 {
|
|
return delivery.Candidate{}, false
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
return c, err == nil
|
|
}
|
|
|
|
func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
|
out := VerdictRead{Key: key, Verdict: raw}
|
|
v, err := delivery.DecodeVerdict(raw)
|
|
if err != nil {
|
|
return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
|
|
}
|
|
c, ok := r.candidate(v.Group)
|
|
if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) {
|
|
out.Note = "the candidate no longer holds these heads"
|
|
return out, nil
|
|
}
|
|
cs := CaseOf(c, v.Trunk)
|
|
out.Reports = map[string]string{}
|
|
if cs == nil {
|
|
return out, nil
|
|
}
|
|
for _, p := range cs.Pulls {
|
|
out.Reports[pullKey(p)] = Report(v, cs, p, nil, nil, nil)
|
|
}
|
|
for _, l := range cs.Left {
|
|
out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
|
|
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
|
|
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
|
return nil, fmt.Errorf("the verdicts state cannot be read here")
|
|
}
|
|
keys, err := r.VerdictState.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
var out []VerdictRead
|
|
for _, k := range keys {
|
|
if group != "" {
|
|
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
|
continue
|
|
}
|
|
}
|
|
raw, err := r.VerdictState.Get(k)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
continue
|
|
}
|
|
vr, err := r.read(k, raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, vr)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for an hq pull request,
|
|
// that hq is judged by merge-check.sh until issue 1296 (HQNote); for one the indexer holds in no candidate, that none does.
|
|
func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
|
if r.CandidateState.Keys == nil || r.CandidateState.Get == nil {
|
|
return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here")
|
|
}
|
|
keys, err := r.CandidateState.Keys()
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
raw, err := r.CandidateState.Get(k)
|
|
if err != nil || len(raw) == 0 {
|
|
continue
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, p := range c.Pulls {
|
|
if p.Repository != repository || p.Number != number {
|
|
continue
|
|
}
|
|
if repository == HQRepository {
|
|
return VerdictRead{Key: k, Note: HQNote}, nil
|
|
}
|
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
vraw, err := r.VerdictState.Get(vkey)
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
if len(vraw) == 0 {
|
|
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
|
}
|
|
return r.read(vkey, vraw)
|
|
}
|
|
}
|
|
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
|
}
|
|
|
|
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
|
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
|
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
|
return nil, fmt.Errorf("the kept state cannot be read here")
|
|
}
|
|
if digest != "" {
|
|
key, err := delivery.KeptKey(level, digest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
raw, err := r.KeptState.Get(key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
return nil, fmt.Errorf("no kept verdict %s", key)
|
|
}
|
|
return map[string]json.RawMessage{key: raw}, nil
|
|
}
|
|
keys, err := r.KeptState.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
out := map[string]json.RawMessage{}
|
|
for _, k := range keys {
|
|
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
|
|
continue
|
|
}
|
|
raw, err := r.KeptState.Get(k)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) > 0 {
|
|
out[k] = raw
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// PreviewRead is one preview as stored, with the parts of its key.
|
|
type PreviewRead struct {
|
|
Key string `json:"key"`
|
|
Repository string `json:"repository"`
|
|
Head string `json:"head"`
|
|
Trunk string `json:"trunk"`
|
|
Preview json.RawMessage `json:"preview"`
|
|
}
|
|
|
|
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
|
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
|
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
|
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
|
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
|
}
|
|
keys, err := r.PreviewState.Keys()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
out := []PreviewRead{}
|
|
var bad []string
|
|
for _, k := range keys {
|
|
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
|
if err != nil {
|
|
bad = append(bad, k)
|
|
continue
|
|
}
|
|
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
|
continue
|
|
}
|
|
raw, err := r.PreviewState.Get(k)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if len(raw) > 0 {
|
|
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
|
}
|
|
}
|
|
return out, bad, nil
|
|
}
|
|
|
|
// RunningStep is a step in flight on a build module.
|
|
type RunningStep struct {
|
|
Module string `json:"module"`
|
|
Node string `json:"node"`
|
|
ID string `json:"id"`
|
|
Tool string `json:"tool"`
|
|
Since string `json:"since,omitempty"`
|
|
}
|
|
|
|
// LeftStep is a key a build module left running that no step behind it runs any more, and why.
|
|
type LeftStep struct {
|
|
RunningStep
|
|
Why string `json:"why"`
|
|
}
|
|
|
|
// Status is what runs now, read from the buckets: the steps in flight on the build modules and the candidates without a
|
|
// verdict for their heads. Nothing holds a check since R4g: a check is an ask on the build seats' streams, whose backlog
|
|
// the delivery view shows (ADR 0525 §9).
|
|
type Status struct {
|
|
Running []RunningStep `json:"running_steps"`
|
|
LeftRunning []LeftStep `json:"left_running"`
|
|
RunningByNode map[string]int `json:"running_by_node"`
|
|
Unjudged []string `json:"unjudged_candidates"`
|
|
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
|
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
|
Waiting map[string][]string `json:"waiting_by_kind"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// Status reads the build modules' steps and the candidates against their verdicts.
|
|
func (r Reader) Status() (Status, error) {
|
|
st := Status{RunningByNode: map[string]int{}}
|
|
mods := make([]string, 0, len(r.StepStates))
|
|
for m := range r.StepStates {
|
|
mods = append(mods, m)
|
|
}
|
|
sort.Strings(mods)
|
|
st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{}
|
|
now, bound := time.Now(), r.StepBound
|
|
if r.Now != nil {
|
|
now = r.Now()
|
|
}
|
|
if bound <= 0 {
|
|
bound = DefaultStepBound
|
|
}
|
|
for _, m := range mods {
|
|
sr := r.StepStates[m]
|
|
if sr.Keys == nil || sr.Get == nil {
|
|
continue
|
|
}
|
|
keys, err := sr.Keys()
|
|
if err != nil {
|
|
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
|
continue
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
raw, err := sr.Get(k)
|
|
if err != nil || len(raw) == 0 {
|
|
continue
|
|
}
|
|
step, err := delivery.DecodeStep(raw)
|
|
if err != nil || step.State != delivery.StepRunning {
|
|
continue
|
|
}
|
|
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
|
if !step.Started.IsZero() {
|
|
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
|
}
|
|
if why := r.leftWhy(m, step, now, bound); why != "" {
|
|
st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why})
|
|
continue
|
|
}
|
|
st.Running = append(st.Running, rs)
|
|
st.RunningByNode[step.Node]++
|
|
}
|
|
}
|
|
if r.HaveState.Get == nil {
|
|
st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node")
|
|
}
|
|
if len(st.LeftRunning) > 0 {
|
|
st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them")
|
|
}
|
|
st.Unjudged = []string{}
|
|
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
|
var heads map[string]string
|
|
if r.Trunks == nil {
|
|
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
|
} else if h, err := r.Trunks(); err != nil {
|
|
st.Notes = append(st.Notes, "the trunk heads cannot be read, so no check is named as waiting for a trunk move: "+err.Error())
|
|
} else {
|
|
heads = h
|
|
}
|
|
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
|
|
keys, err := r.CandidateState.Keys()
|
|
if err != nil {
|
|
return st, err
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
raw, err := r.CandidateState.Get(k)
|
|
if err != nil || len(raw) == 0 {
|
|
continue
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if !checksAny(c) {
|
|
continue // hq's pull requests alone: merge-check.sh judges them, the checker never does
|
|
}
|
|
raw, err = r.VerdictState.Get(vkey)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if len(raw) == 0 {
|
|
st.Unjudged = append(st.Unjudged, c.Group)
|
|
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
|
continue
|
|
}
|
|
v, err := delivery.DecodeVerdict(raw)
|
|
switch {
|
|
case err != nil || v.State == delivery.VerdictRunning:
|
|
case IsNodesFailure(v):
|
|
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
|
case heads != nil:
|
|
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
|
|
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
st.Notes = append(st.Notes, "the places the checker holds are in its process and in no bucket: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
|
|
return st, nil
|
|
}
|
|
|
|
// DefaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a test
|
|
// level runs a repository's tests); past it the check fails naming the step, and the step's place on its node is free.
|
|
const DefaultStepBound = 30 * time.Minute
|
|
|
|
// leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module
|
|
// is not on its node or is not running there (the node-engine's have), or it started longer ago than the step bound, when
|
|
// the checker stopped waiting for it.
|
|
func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound time.Duration) string {
|
|
if r.HaveState.Get != nil {
|
|
key, err := delivery.HaveKey(step.Node, module)
|
|
if err == nil {
|
|
raw, err := r.HaveState.Get(key)
|
|
switch {
|
|
case err != nil:
|
|
case len(raw) == 0:
|
|
return fmt.Sprintf("%s is not on %s", module, step.Node)
|
|
default:
|
|
if h, err := delivery.DecodeHave(raw); err == nil && h.Phase != delivery.PhaseRunning {
|
|
return fmt.Sprintf("%s on %s is %s", module, step.Node, h.Phase)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if step.Started.IsZero() {
|
|
return fmt.Sprintf("it says no start, so no wait of the checker's (bound %s) can be behind it", bound)
|
|
}
|
|
if end := step.Started.Add(bound); now.After(end) {
|
|
return fmt.Sprintf("started %s, past the step bound of %s: the checker stopped waiting for it at %s", step.Started.UTC().Format(time.RFC3339), bound, end.UTC().Format(time.RFC3339))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// checksAny says whether a candidate has a pull request the checker judges: one outside hq.
|
|
func checksAny(c delivery.Candidate) bool {
|
|
for _, p := range c.Pulls {
|
|
if p.Repository != HQRepository {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|