jschoubben
2bdb8d0e00
A change to mesh-build-go was judged by the installed module it replaces, so such pull requests were merged by hand. ADR 0535 §1, §2, §4, §5.
428 lines
15 KiB
Go
428 lines
15 KiB
Go
package check
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
)
|
|
|
|
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
|
// status it owes in full, so a check is read live without a shell. They write nothing.
|
|
|
|
// The read tools, named as the module is: checker_ and the noun.
|
|
const (
|
|
VerdictsReadTool = "checker_verdicts"
|
|
PullReadTool = "checker_pull"
|
|
KeptReadTool = "checker_kept"
|
|
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685).
|
|
PreviewsReadTool = "checker_previews"
|
|
StatusReadTool = "checker_status"
|
|
)
|
|
|
|
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
|
|
type StateRead struct {
|
|
Keys func() ([]string, error)
|
|
Get func(key string) (json.RawMessage, error)
|
|
}
|
|
|
|
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
|
|
type Reader struct {
|
|
VerdictState, KeptState, CandidateState StateRead
|
|
|
|
PreviewState StateRead // the checker's merge previews
|
|
PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent
|
|
StepStates map[string]StateRead // the build modules' steps states, by module
|
|
Capacity func() (string, error) // the capacity setting as written, `node=steps,...`
|
|
HaveState StateRead // the node-engine's have: whether a build module is on a node and running
|
|
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
|
Now func() time.Time
|
|
}
|
|
|
|
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
|
// candidate still holds those heads; without it the report is that of the verdict alone.
|
|
type VerdictRead struct {
|
|
Key string `json:"key"`
|
|
Verdict json.RawMessage `json:"verdict"`
|
|
Reports map[string]string `json:"reports,omitempty"`
|
|
Note string `json:"note,omitempty"`
|
|
}
|
|
|
|
func (r Reader) candidate(group string) (delivery.Candidate, bool) {
|
|
key, err := delivery.CandidatesKey(group)
|
|
if err != nil || r.CandidateState.Get == nil {
|
|
return delivery.Candidate{}, false
|
|
}
|
|
raw, err := r.CandidateState.Get(key)
|
|
if err != nil || len(raw) == 0 {
|
|
return delivery.Candidate{}, false
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
return c, err == nil
|
|
}
|
|
|
|
func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
|
out := VerdictRead{Key: key, Verdict: raw}
|
|
v, err := delivery.DecodeVerdict(raw)
|
|
if err != nil {
|
|
return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
|
|
}
|
|
c, ok := r.candidate(v.Group)
|
|
if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) {
|
|
out.Note = "the candidate no longer holds these heads"
|
|
return out, nil
|
|
}
|
|
cs := CaseOf(c, v.Trunk)
|
|
out.Reports = map[string]string{}
|
|
if cs == nil {
|
|
return out, nil
|
|
}
|
|
for _, p := range cs.Pulls {
|
|
out.Reports[pullKey(p)] = Report(v, cs, p, nil, nil, nil)
|
|
}
|
|
for _, l := range cs.Left {
|
|
out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
|
|
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
|
|
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
|
return nil, fmt.Errorf("the verdicts state cannot be read here")
|
|
}
|
|
keys, err := r.VerdictState.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
var out []VerdictRead
|
|
for _, k := range keys {
|
|
if group != "" {
|
|
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
|
continue
|
|
}
|
|
}
|
|
raw, err := r.VerdictState.Get(k)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
continue
|
|
}
|
|
vr, err := r.read(k, raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, vr)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for an hq pull request,
|
|
// that hq is judged by merge-check.sh until issue 1296 (HQNote); for one the indexer holds in no candidate, that none does.
|
|
func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
|
if r.CandidateState.Keys == nil || r.CandidateState.Get == nil {
|
|
return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here")
|
|
}
|
|
keys, err := r.CandidateState.Keys()
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
raw, err := r.CandidateState.Get(k)
|
|
if err != nil || len(raw) == 0 {
|
|
continue
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, p := range c.Pulls {
|
|
if p.Repository != repository || p.Number != number {
|
|
continue
|
|
}
|
|
if repository == HQRepository {
|
|
return VerdictRead{Key: k, Note: HQNote}, nil
|
|
}
|
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
vraw, err := r.VerdictState.Get(vkey)
|
|
if err != nil {
|
|
return VerdictRead{}, err
|
|
}
|
|
if len(vraw) == 0 {
|
|
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
|
}
|
|
return r.read(vkey, vraw)
|
|
}
|
|
}
|
|
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
|
}
|
|
|
|
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
|
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
|
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
|
return nil, fmt.Errorf("the kept state cannot be read here")
|
|
}
|
|
if digest != "" {
|
|
key, err := delivery.KeptKey(level, digest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
raw, err := r.KeptState.Get(key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) == 0 {
|
|
return nil, fmt.Errorf("no kept verdict %s", key)
|
|
}
|
|
return map[string]json.RawMessage{key: raw}, nil
|
|
}
|
|
keys, err := r.KeptState.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
out := map[string]json.RawMessage{}
|
|
for _, k := range keys {
|
|
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
|
|
continue
|
|
}
|
|
raw, err := r.KeptState.Get(k)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) > 0 {
|
|
out[k] = raw
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// PreviewRead is one preview as stored, with the parts of its key.
|
|
type PreviewRead struct {
|
|
Key string `json:"key"`
|
|
Repository string `json:"repository"`
|
|
Head string `json:"head"`
|
|
Trunk string `json:"trunk"`
|
|
Preview json.RawMessage `json:"preview"`
|
|
}
|
|
|
|
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
|
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
|
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
|
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
|
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
|
}
|
|
keys, err := r.PreviewState.Keys()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
sort.Strings(keys)
|
|
out := []PreviewRead{}
|
|
var bad []string
|
|
for _, k := range keys {
|
|
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
|
if err != nil {
|
|
bad = append(bad, k)
|
|
continue
|
|
}
|
|
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
|
continue
|
|
}
|
|
raw, err := r.PreviewState.Get(k)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if len(raw) > 0 {
|
|
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
|
}
|
|
}
|
|
return out, bad, nil
|
|
}
|
|
|
|
// RunningStep is a step in flight on a build module.
|
|
type RunningStep struct {
|
|
Module string `json:"module"`
|
|
Node string `json:"node"`
|
|
ID string `json:"id"`
|
|
Tool string `json:"tool"`
|
|
Since string `json:"since,omitempty"`
|
|
}
|
|
|
|
// LeftStep is a key a build module left running that no step behind it runs any more, and why.
|
|
type LeftStep struct {
|
|
RunningStep
|
|
Why string `json:"why"`
|
|
}
|
|
|
|
// Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in
|
|
// its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a
|
|
// verdict for their heads, and the delivery plans in their build phase), not as the process's own counters.
|
|
type Status struct {
|
|
Capacity map[string]int `json:"capacity"`
|
|
CapacityNote string `json:"capacity_note,omitempty"`
|
|
DefaultPerNode int `json:"default_per_node"`
|
|
PlansBuilding struct {
|
|
Readable bool `json:"readable"`
|
|
Keys []string `json:"keys"`
|
|
Holds bool `json:"holds"`
|
|
} `json:"plans_building"`
|
|
Running []RunningStep `json:"running_steps"`
|
|
LeftRunning []LeftStep `json:"left_running"`
|
|
RunningByNode map[string]int `json:"running_by_node"`
|
|
Unjudged []string `json:"unjudged_candidates"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts.
|
|
func (r Reader) Status() (Status, error) {
|
|
st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}}
|
|
if r.Capacity == nil {
|
|
st.CapacityNote = "the capacity setting cannot be read from here"
|
|
} else if setting, err := r.Capacity(); err != nil {
|
|
st.CapacityNote = "the capacity setting cannot be read: " + err.Error()
|
|
} else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil {
|
|
st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error()
|
|
}
|
|
if r.PlansState.Keys == nil {
|
|
st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker")
|
|
} else if keys, err := r.PlansState.Keys(); err != nil {
|
|
st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error())
|
|
} else {
|
|
sort.Strings(keys)
|
|
st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0
|
|
}
|
|
mods := make([]string, 0, len(r.StepStates))
|
|
for m := range r.StepStates {
|
|
mods = append(mods, m)
|
|
}
|
|
sort.Strings(mods)
|
|
st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{}
|
|
now, bound := time.Now(), r.StepBound
|
|
if r.Now != nil {
|
|
now = r.Now()
|
|
}
|
|
if bound <= 0 {
|
|
bound = DefaultStepBound
|
|
}
|
|
for _, m := range mods {
|
|
sr := r.StepStates[m]
|
|
if sr.Keys == nil || sr.Get == nil {
|
|
continue
|
|
}
|
|
keys, err := sr.Keys()
|
|
if err != nil {
|
|
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
|
continue
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
raw, err := sr.Get(k)
|
|
if err != nil || len(raw) == 0 {
|
|
continue
|
|
}
|
|
step, err := delivery.DecodeStep(raw)
|
|
if err != nil || step.State != delivery.StepRunning {
|
|
continue
|
|
}
|
|
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
|
if !step.Started.IsZero() {
|
|
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
|
}
|
|
if why := r.leftWhy(m, step, now, bound); why != "" {
|
|
st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why})
|
|
continue
|
|
}
|
|
st.Running = append(st.Running, rs)
|
|
st.RunningByNode[step.Node]++
|
|
}
|
|
}
|
|
if r.HaveState.Get == nil {
|
|
st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node")
|
|
}
|
|
if len(st.LeftRunning) > 0 {
|
|
st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them")
|
|
}
|
|
st.Unjudged = []string{}
|
|
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
|
|
keys, err := r.CandidateState.Keys()
|
|
if err != nil {
|
|
return st, err
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
raw, err := r.CandidateState.Get(k)
|
|
if err != nil || len(raw) == 0 {
|
|
continue
|
|
}
|
|
c, err := delivery.DecodeCandidate(raw)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if !checksAny(c) {
|
|
continue // hq's pull requests alone: merge-check.sh judges them, the checker never does
|
|
}
|
|
if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 {
|
|
st.Unjudged = append(st.Unjudged, c.Group)
|
|
}
|
|
}
|
|
}
|
|
st.Notes = append(st.Notes, "the places the checker holds are in its process and in no bucket: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
|
|
return st, nil
|
|
}
|
|
|
|
// DefaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a test
|
|
// level runs a repository's tests); past it the check fails naming the step, and the step's place on its node is free.
|
|
const DefaultStepBound = 30 * time.Minute
|
|
|
|
// leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module
|
|
// is not on its node or is not running there (the node-engine's have), or it started longer ago than the step bound, when
|
|
// the checker stopped waiting for it.
|
|
func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound time.Duration) string {
|
|
if r.HaveState.Get != nil {
|
|
key, err := delivery.HaveKey(step.Node, module)
|
|
if err == nil {
|
|
raw, err := r.HaveState.Get(key)
|
|
switch {
|
|
case err != nil:
|
|
case len(raw) == 0:
|
|
return fmt.Sprintf("%s is not on %s", module, step.Node)
|
|
default:
|
|
if h, err := delivery.DecodeHave(raw); err == nil && h.Phase != delivery.PhaseRunning {
|
|
return fmt.Sprintf("%s on %s is %s", module, step.Node, h.Phase)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if step.Started.IsZero() {
|
|
return fmt.Sprintf("it says no start, so no wait of the checker's (bound %s) can be behind it", bound)
|
|
}
|
|
if end := step.Started.Add(bound); now.After(end) {
|
|
return fmt.Sprintf("started %s, past the step bound of %s: the checker stopped waiting for it at %s", step.Started.UTC().Format(time.RFC3339), bound, end.UTC().Format(time.RFC3339))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// checksAny says whether a candidate has a pull request the checker judges: one outside hq.
|
|
func checksAny(c delivery.Candidate) bool {
|
|
for _, p := range c.Pulls {
|
|
if p.Repository != HQRepository {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|