package check import ( "encoding/json" "fmt" "sort" "strings" "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the // status it owes in full, so a check is read live without a shell. They write nothing. // The read tools, named as the module is: checker_ and the noun. const ( VerdictsReadTool = "checker_verdicts" PullReadTool = "checker_pull" KeptReadTool = "checker_kept" // PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685). PreviewsReadTool = "checker_previews" StatusReadTool = "checker_status" ) // StateRead reads one state: its keys and a key's stored value (nil for a key with none). type StateRead struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, error) } // Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates. type Reader struct { VerdictState, KeptState, CandidateState StateRead PreviewState StateRead // the checker's merge previews PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent StepStates map[string]StateRead // the build modules' steps states, by module Capacity func() (string, error) // the capacity setting as written, `node=steps,...` HaveState StateRead // the node-engine's have: whether a build module is on a node and running StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0 Now func() time.Time } // VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the // candidate still holds those heads; without it the report is that of the verdict alone. type VerdictRead struct { Key string `json:"key"` Verdict json.RawMessage `json:"verdict"` Reports map[string]string `json:"reports,omitempty"` Note string `json:"note,omitempty"` } func (r Reader) candidate(group string) (delivery.Candidate, bool) { key, err := delivery.CandidatesKey(group) if err != nil || r.CandidateState.Get == nil { return delivery.Candidate{}, false } raw, err := r.CandidateState.Get(key) if err != nil || len(raw) == 0 { return delivery.Candidate{}, false } c, err := delivery.DecodeCandidate(raw) return c, err == nil } func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) { out := VerdictRead{Key: key, Verdict: raw} v, err := delivery.DecodeVerdict(raw) if err != nil { return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err) } c, ok := r.candidate(v.Group) if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) { out.Note = "the candidate no longer holds these heads" return out, nil } cs := CaseOf(c, v.Trunk) out.Reports = map[string]string{} if cs == nil { return out, nil } for _, p := range cs.Pulls { out.Reports[pullKey(p)] = Report(v, cs, p, nil, nil, nil) } for _, l := range cs.Left { out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1] } return out, nil } // Verdicts are the verdicts the checker holds, by key; with a group only that group's. func (r Reader) Verdicts(group string) ([]VerdictRead, error) { if r.VerdictState.Keys == nil || r.VerdictState.Get == nil { return nil, fmt.Errorf("the verdicts state cannot be read here") } keys, err := r.VerdictState.Keys() if err != nil { return nil, err } sort.Strings(keys) var out []VerdictRead for _, k := range keys { if group != "" { if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group { continue } } raw, err := r.VerdictState.Get(k) if err != nil { return nil, err } if len(raw) == 0 { continue } vr, err := r.read(k, raw) if err != nil { return nil, err } out = append(out, vr) } return out, nil } // Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for an hq pull request, // that hq is judged by merge-check.sh until issue 1296 (HQNote); for one the indexer holds in no candidate, that none does. func (r Reader) Pull(repository string, number int) (VerdictRead, error) { if r.CandidateState.Keys == nil || r.CandidateState.Get == nil { return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here") } keys, err := r.CandidateState.Keys() if err != nil { return VerdictRead{}, err } sort.Strings(keys) for _, k := range keys { raw, err := r.CandidateState.Get(k) if err != nil || len(raw) == 0 { continue } c, err := delivery.DecodeCandidate(raw) if err != nil { continue } for _, p := range c.Pulls { if p.Repository != repository || p.Number != number { continue } if repository == HQRepository { return VerdictRead{Key: k, Note: HQNote}, nil } vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) if err != nil { return VerdictRead{}, err } vraw, err := r.VerdictState.Get(vkey) if err != nil { return VerdictRead{}, err } if len(vraw) == 0 { return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil } return r.read(vkey, vraw) } } return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number) } // Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values. func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) { if r.KeptState.Keys == nil || r.KeptState.Get == nil { return nil, fmt.Errorf("the kept state cannot be read here") } if digest != "" { key, err := delivery.KeptKey(level, digest) if err != nil { return nil, err } raw, err := r.KeptState.Get(key) if err != nil { return nil, err } if len(raw) == 0 { return nil, fmt.Errorf("no kept verdict %s", key) } return map[string]json.RawMessage{key: raw}, nil } keys, err := r.KeptState.Keys() if err != nil { return nil, err } sort.Strings(keys) out := map[string]json.RawMessage{} for _, k := range keys { if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) { continue } raw, err := r.KeptState.Get(k) if err != nil { return nil, err } if len(raw) > 0 { out[k] = raw } } return out, nil } // PreviewRead is one preview as stored, with the parts of its key. type PreviewRead struct { Key string `json:"key"` Repository string `json:"repository"` Head string `json:"head"` Trunk string `json:"trunk"` Preview json.RawMessage `json:"preview"` } // Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a // prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable. func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) { if r.PreviewState.Keys == nil || r.PreviewState.Get == nil { return nil, nil, fmt.Errorf("the previews state cannot be read here") } keys, err := r.PreviewState.Keys() if err != nil { return nil, nil, err } sort.Strings(keys) out := []PreviewRead{} var bad []string for _, k := range keys { repo, h, trunk, err := delivery.ParsePreviewKey(k) if err != nil { bad = append(bad, k) continue } if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) { continue } raw, err := r.PreviewState.Get(k) if err != nil { return nil, nil, err } if len(raw) > 0 { out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw}) } } return out, bad, nil } // RunningStep is a step in flight on a build module. type RunningStep struct { Module string `json:"module"` Node string `json:"node"` ID string `json:"id"` Tool string `json:"tool"` Since string `json:"since,omitempty"` } // LeftStep is a key a build module left running that no step behind it runs any more, and why. type LeftStep struct { RunningStep Why string `json:"why"` } // Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in // its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a // verdict for their heads, and the delivery plans in their build phase), not as the process's own counters. type Status struct { Capacity map[string]int `json:"capacity"` CapacityNote string `json:"capacity_note,omitempty"` DefaultPerNode int `json:"default_per_node"` PlansBuilding struct { Readable bool `json:"readable"` Keys []string `json:"keys"` Holds bool `json:"holds"` } `json:"plans_building"` Running []RunningStep `json:"running_steps"` LeftRunning []LeftStep `json:"left_running"` RunningByNode map[string]int `json:"running_by_node"` Unjudged []string `json:"unjudged_candidates"` Notes []string `json:"notes,omitempty"` } // Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts. func (r Reader) Status() (Status, error) { st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}} if r.Capacity == nil { st.CapacityNote = "the capacity setting cannot be read from here" } else if setting, err := r.Capacity(); err != nil { st.CapacityNote = "the capacity setting cannot be read: " + err.Error() } else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil { st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error() } if r.PlansState.Keys == nil { st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker") } else if keys, err := r.PlansState.Keys(); err != nil { st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error()) } else { sort.Strings(keys) st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0 } mods := make([]string, 0, len(r.StepStates)) for m := range r.StepStates { mods = append(mods, m) } sort.Strings(mods) st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{} now, bound := time.Now(), r.StepBound if r.Now != nil { now = r.Now() } if bound <= 0 { bound = DefaultStepBound } for _, m := range mods { sr := r.StepStates[m] if sr.Keys == nil || sr.Get == nil { continue } keys, err := sr.Keys() if err != nil { st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error()) continue } sort.Strings(keys) for _, k := range keys { raw, err := sr.Get(k) if err != nil || len(raw) == 0 { continue } step, err := delivery.DecodeStep(raw) if err != nil || step.State != delivery.StepRunning { continue } rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool} if !step.Started.IsZero() { rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z") } if why := r.leftWhy(m, step, now, bound); why != "" { st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why}) continue } st.Running = append(st.Running, rs) st.RunningByNode[step.Node]++ } } if r.HaveState.Get == nil { st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node") } if len(st.LeftRunning) > 0 { st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them") } st.Unjudged = []string{} if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil { keys, err := r.CandidateState.Keys() if err != nil { return st, err } sort.Strings(keys) for _, k := range keys { raw, err := r.CandidateState.Get(k) if err != nil || len(raw) == 0 { continue } c, err := delivery.DecodeCandidate(raw) if err != nil { continue } vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) if err != nil { continue } if !checksAny(c) { continue // hq's pull requests alone: merge-check.sh judges them, the checker never does } if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 { st.Unjudged = append(st.Unjudged, c.Group) } } } st.Notes = append(st.Notes, "the places the checker holds are in its process and in no bucket: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held") return st, nil } // DefaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a test // level runs a repository's tests); past it the check fails naming the step, and the step's place on its node is free. const DefaultStepBound = 30 * time.Minute // leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module // is not on its node or is not running there (the node-engine's have), or it started longer ago than the step bound, when // the checker stopped waiting for it. func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound time.Duration) string { if r.HaveState.Get != nil { key, err := delivery.HaveKey(step.Node, module) if err == nil { raw, err := r.HaveState.Get(key) switch { case err != nil: case len(raw) == 0: return fmt.Sprintf("%s is not on %s", module, step.Node) default: if h, err := delivery.DecodeHave(raw); err == nil && h.Phase != delivery.PhaseRunning { return fmt.Sprintf("%s on %s is %s", module, step.Node, h.Phase) } } } } if step.Started.IsZero() { return fmt.Sprintf("it says no start, so no wait of the checker's (bound %s) can be behind it", bound) } if end := step.Started.Add(bound); now.After(end) { return fmt.Sprintf("started %s, past the step bound of %s: the checker stopped waiting for it at %s", step.Started.UTC().Format(time.RFC3339), bound, end.UTC().Format(time.RFC3339)) } return "" } // checksAny says whether a candidate has a pull request the checker judges: one outside hq. func checksAny(c delivery.Candidate) bool { for _, p := range c.Pulls { if p.Repository != HQRepository { return true } } return false }