package check import ( "encoding/json" "fmt" "sort" "strings" "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) // The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the // status it owes in full, so a check is read live without a shell. They write nothing. // The read tools, named as the module is: checker_ and the noun. const ( VerdictsReadTool = "checker_verdicts" PullReadTool = "checker_pull" KeptReadTool = "checker_kept" // PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker runs (issue 1685: nothing holds it since R4g). PreviewsReadTool = "checker_previews" StatusReadTool = "checker_status" ) // StateRead reads one state: its keys and a key's stored value (nil for a key with none). type StateRead struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, error) } // Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates. type Reader struct { VerdictState, KeptState, CandidateState StateRead PreviewState StateRead // the checker's merge previews StepStates map[string]StateRead // the build modules' steps states, by module HaveState StateRead // the node-engine's have: whether a build module is on a node and running StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0 // Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move. Trunks func() (map[string]string, error) Now func() time.Time } // VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the // candidate still holds those heads; without it the report is that of the verdict alone. type VerdictRead struct { Key string `json:"key"` Verdict json.RawMessage `json:"verdict"` Reports map[string]string `json:"reports,omitempty"` Note string `json:"note,omitempty"` } func (r Reader) candidate(group string) (delivery.Candidate, bool) { key, err := delivery.CandidatesKey(group) if err != nil || r.CandidateState.Get == nil { return delivery.Candidate{}, false } raw, err := r.CandidateState.Get(key) if err != nil || len(raw) == 0 { return delivery.Candidate{}, false } c, err := delivery.DecodeCandidate(raw) return c, err == nil } func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) { out := VerdictRead{Key: key, Verdict: raw} v, err := delivery.DecodeVerdict(raw) if err != nil { return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err) } c, ok := r.candidate(v.Group) if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) { out.Note = "the candidate no longer holds these heads" return out, nil } cs := CaseOf(c, v.Trunk) out.Reports = map[string]string{} if cs == nil { return out, nil } for _, p := range cs.Pulls { out.Reports[pullKey(p)] = Report(v, cs, p, nil, nil, nil) } for _, l := range cs.Left { out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1] } return out, nil } // Verdicts are the verdicts the checker holds, by key; with a group only that group's. func (r Reader) Verdicts(group string) ([]VerdictRead, error) { if r.VerdictState.Keys == nil || r.VerdictState.Get == nil { return nil, fmt.Errorf("the verdicts state cannot be read here") } keys, err := r.VerdictState.Keys() if err != nil { return nil, err } sort.Strings(keys) var out []VerdictRead for _, k := range keys { if group != "" { if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group { continue } } raw, err := r.VerdictState.Get(k) if err != nil { return nil, err } if len(raw) == 0 { continue } vr, err := r.read(k, raw) if err != nil { return nil, err } out = append(out, vr) } return out, nil } // Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds // in no candidate, that none does. func (r Reader) Pull(repository string, number int) (VerdictRead, error) { if r.CandidateState.Keys == nil || r.CandidateState.Get == nil { return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here") } keys, err := r.CandidateState.Keys() if err != nil { return VerdictRead{}, err } sort.Strings(keys) for _, k := range keys { raw, err := r.CandidateState.Get(k) if err != nil || len(raw) == 0 { continue } c, err := delivery.DecodeCandidate(raw) if err != nil { continue } for _, p := range c.Pulls { if p.Repository != repository || p.Number != number { continue } vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) if err != nil { return VerdictRead{}, err } vraw, err := r.VerdictState.Get(vkey) if err != nil { return VerdictRead{}, err } if len(vraw) == 0 { return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil } return r.read(vkey, vraw) } } return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number) } // Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values. func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) { if r.KeptState.Keys == nil || r.KeptState.Get == nil { return nil, fmt.Errorf("the kept state cannot be read here") } if digest != "" { key, err := delivery.KeptKey(level, digest) if err != nil { return nil, err } raw, err := r.KeptState.Get(key) if err != nil { return nil, err } if len(raw) == 0 { return nil, fmt.Errorf("no kept verdict %s", key) } return map[string]json.RawMessage{key: raw}, nil } keys, err := r.KeptState.Keys() if err != nil { return nil, err } sort.Strings(keys) out := map[string]json.RawMessage{} for _, k := range keys { if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) { continue } raw, err := r.KeptState.Get(k) if err != nil { return nil, err } if len(raw) > 0 { out[k] = raw } } return out, nil } // PreviewRead is one preview as stored, with the parts of its key. type PreviewRead struct { Key string `json:"key"` Repository string `json:"repository"` Head string `json:"head"` Trunk string `json:"trunk"` Preview json.RawMessage `json:"preview"` } // Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a // prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable. func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) { if r.PreviewState.Keys == nil || r.PreviewState.Get == nil { return nil, nil, fmt.Errorf("the previews state cannot be read here") } keys, err := r.PreviewState.Keys() if err != nil { return nil, nil, err } sort.Strings(keys) out := []PreviewRead{} var bad []string for _, k := range keys { repo, h, trunk, err := delivery.ParsePreviewKey(k) if err != nil { bad = append(bad, k) continue } if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) { continue } raw, err := r.PreviewState.Get(k) if err != nil { return nil, nil, err } if len(raw) > 0 { out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw}) } } return out, bad, nil } // RunningStep is a step in flight on a build module. type RunningStep struct { Module string `json:"module"` Node string `json:"node"` ID string `json:"id"` Tool string `json:"tool"` Since string `json:"since,omitempty"` } // LeftStep is a key a build module left running that no step behind it runs any more, and why. type LeftStep struct { RunningStep Why string `json:"why"` } // Status is what runs now, read from the buckets: the steps in flight on the build modules and the candidates without a // verdict for their heads. Nothing holds a check since R4g: a check is an ask on the build seats' streams, whose backlog // the delivery view shows (ADR 0525 §9). type Status struct { Running []RunningStep `json:"running_steps"` LeftRunning []LeftStep `json:"left_running"` RunningByNode map[string]int `json:"running_by_node"` Unjudged []string `json:"unjudged_candidates"` // Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed // for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since). Waiting map[string][]string `json:"waiting_by_kind"` Notes []string `json:"notes,omitempty"` } // Status reads the build modules' steps and the candidates against their verdicts. func (r Reader) Status() (Status, error) { st := Status{RunningByNode: map[string]int{}} mods := make([]string, 0, len(r.StepStates)) for m := range r.StepStates { mods = append(mods, m) } sort.Strings(mods) st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{} now, bound := time.Now(), r.StepBound if r.Now != nil { now = r.Now() } if bound <= 0 { bound = DefaultStepBound } for _, m := range mods { sr := r.StepStates[m] if sr.Keys == nil || sr.Get == nil { continue } keys, err := sr.Keys() if err != nil { st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error()) continue } sort.Strings(keys) for _, k := range keys { raw, err := sr.Get(k) if err != nil || len(raw) == 0 { continue } step, err := delivery.DecodeStep(raw) if err != nil || step.State != delivery.StepRunning { continue } rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool} if !step.Started.IsZero() { rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z") } if why := r.leftWhy(m, step, now, bound); why != "" { st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why}) continue } st.Running = append(st.Running, rs) st.RunningByNode[step.Node]++ } } if r.HaveState.Get == nil { st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node") } if len(st.LeftRunning) > 0 { st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them") } st.Unjudged = []string{} st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}} var heads map[string]string if r.Trunks == nil { st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move") } else if h, err := r.Trunks(); err != nil { st.Notes = append(st.Notes, "the trunk heads cannot be read, so no check is named as waiting for a trunk move: "+err.Error()) } else { heads = h } if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil { keys, err := r.CandidateState.Keys() if err != nil { return st, err } sort.Strings(keys) for _, k := range keys { raw, err := r.CandidateState.Get(k) if err != nil || len(raw) == 0 { continue } c, err := delivery.DecodeCandidate(raw) if err != nil { continue } vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) if err != nil { continue } raw, err = r.VerdictState.Get(vkey) if err != nil { continue } if len(raw) == 0 { st.Unjudged = append(st.Unjudged, c.Group) st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group) continue } v, err := delivery.DecodeVerdict(raw) switch { case err != nil || v.State == delivery.VerdictRunning: case IsNodesFailure(v): st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group) case heads != nil: if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) { st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group) } } } } st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)") return st, nil } // DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch, // the digests and the levels), never counting its wait in the stream; past it the check fails naming the node. const DefaultStepBound = 30 * time.Minute // leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module // is not on its node or is not running there (the node-engine's have), or it started longer ago than the step bound, when // the checker stopped waiting for it. func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound time.Duration) string { if r.HaveState.Get != nil { key, err := delivery.HaveKey(step.Node, module) if err == nil { raw, err := r.HaveState.Get(key) switch { case err != nil: case len(raw) == 0: return fmt.Sprintf("%s is not on %s", module, step.Node) default: if h, err := delivery.DecodeHave(raw); err == nil && h.Phase != delivery.PhaseRunning { return fmt.Sprintf("%s on %s is %s", module, step.Node, h.Phase) } } } } if step.Started.IsZero() { return fmt.Sprintf("it says no start, so no wait of the checker's (bound %s) can be behind it", bound) } if end := step.Started.Add(bound); now.After(end) { return fmt.Sprintf("started %s, past the step bound of %s: the checker stopped waiting for it at %s", step.Started.UTC().Format(time.RFC3339), bound, end.UTC().Format(time.RFC3339)) } return "" }