The checker wrote a try's asks only at its end, so a 31-minute try could not be told queued from running (issue 1890). The try's record now names each ask as it is put, then the node that took it, its state and the phase its step says, written on the step key's own changes. build_usage names a running try's asks and takes a build module's capacity.<node> (configured bounded by its node's build CPU, issue 1806's follow-up) when it is current with mesh-config.
719 lines
26 KiB
Go
719 lines
26 KiB
Go
package view
|
|
|
|
// The build nodes' use, in one answer (the operator, 2026-10-26: "how much are our build agents being used, and is ace
|
|
// eligible for building?" took six calls over four tools). Read from the bus's state alone: which build seats each node
|
|
// holds (the node-engine's have of mesh-build-<language>), its capacity for each (mesh-config's resolved value of the
|
|
// module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each
|
|
// build module's steps state (ADR 0525 §6: an ask ends on <node>.<ask id>) the asks running now by kind, the ones left
|
|
// running, and the steps started and finished in the last hour and day with their total and median time and how long the
|
|
// node was at its capacity. A try's asks are told from a candidate's by the checker's tries state (each try names the
|
|
// asks it put). The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver),
|
|
// asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the
|
|
// bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
)
|
|
|
|
// BuildUsageTool is the view's tool of the build nodes' use.
|
|
const BuildUsageTool = "build_usage"
|
|
|
|
// endedStale says why a step's end was written by its build module for a key left running (mesh-sdk buildstep's ending
|
|
// of stale steps): a restart's, or the step bound's; "" for a step that ran to its own end. Such an end is the moment the
|
|
// module noticed, not the step's, so it is counted apart and in neither the medians nor the time at capacity.
|
|
func endedStale(st delivery.Step) string {
|
|
if st.State != delivery.StepFailed {
|
|
return ""
|
|
}
|
|
if st.Reason == buildstep.RestartReason || strings.HasPrefix(st.Reason, "past the step bound") {
|
|
return st.Reason
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker
|
|
// put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity).
|
|
const (
|
|
KindTry = "try"
|
|
KindHand = "hand"
|
|
)
|
|
|
|
// UsageReader reads what build_usage answers from.
|
|
type UsageReader struct {
|
|
// Steps and Pauses are each build module's states by language (go, node, image).
|
|
Steps map[string]StateRead
|
|
Pauses map[string]StateRead
|
|
// Resolved is mesh-config's resolved values (<node>.<module>.<key>); Haves the node-engine's have.
|
|
Resolved StateRead
|
|
Haves StateRead
|
|
// Tries is the checker's tries (try-<unix ns>), each naming the asks it put.
|
|
Tries StateRead
|
|
// Backlog is the seats' streams' waiting and held asks, as the view program reads them.
|
|
Backlog func() ([]BacklogRow, error)
|
|
Now func() time.Time
|
|
}
|
|
|
|
// UsageAnswer is build_usage's answer.
|
|
type UsageAnswer struct {
|
|
At time.Time `json:"at"`
|
|
Nodes []NodeUsage `json:"nodes"`
|
|
Backlog []BacklogRow `json:"backlog"`
|
|
// TriesRunning are the tries the checker runs now, each with its asks and where each stands (a checker that names its
|
|
// asks as they are put, issue 1890; an older one names them only at the end, and they are counted under check until then).
|
|
TriesRunning []TryRunning `json:"tries_running"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// TryRunning is one try the checker runs now.
|
|
type TryRunning struct {
|
|
ID string `json:"id"`
|
|
Repository string `json:"repository"`
|
|
Started time.Time `json:"started,omitzero"`
|
|
Asks []string `json:"asks,omitempty"`
|
|
// Steps are the try's own words for each ask: asked, or the node that took it, its state and the phase its step says.
|
|
Steps json.RawMessage `json:"steps,omitempty"`
|
|
}
|
|
|
|
// NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU.
|
|
type NodeUsage struct {
|
|
Node string `json:"node"`
|
|
// Eligible is the node's verdict: "eligible" when a seat of it is, "not eligible" when every seat is known not to be,
|
|
// and "not known" when none is known eligible and this tool cannot tell for some seat.
|
|
Eligible string `json:"eligible"`
|
|
// Said is one line per seat, the answer in words.
|
|
Said []string `json:"said"`
|
|
Seats []SeatUsage `json:"seats"`
|
|
CPU string `json:"cpu"`
|
|
}
|
|
|
|
// SeatUsage is one build seat on one node.
|
|
type SeatUsage struct {
|
|
Seat string `json:"seat"`
|
|
Module string `json:"module"`
|
|
// Holds says the node runs the seat's module (its have says running); Phase is what its have says, or absent.
|
|
Holds bool `json:"holds"`
|
|
Phase string `json:"phase,omitempty"`
|
|
// Capacity is the asks it runs at once; CapacityFrom where that came from.
|
|
Capacity int `json:"capacity"`
|
|
CapacityFrom string `json:"capacity_from"`
|
|
// Configured is mesh-config's capacity when the module says it judges by a lower one (capacity.<node> of its pauses
|
|
// state: the configured capacity bounded by its node's build CPU, issue 1806's follow-up); Capacity is then the
|
|
// module's and CapacityFrom its why.
|
|
Configured int `json:"configured,omitempty"`
|
|
Paused bool `json:"paused"`
|
|
// Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not.
|
|
// EligibleKnown is false when this tool cannot tell (it cannot read mesh-config's capacities): Eligible is then
|
|
// absent, never false, and WhyNot starts "not known: ".
|
|
Eligible *bool `json:"eligible,omitempty"`
|
|
EligibleKnown bool `json:"eligible_known"`
|
|
WhyNot string `json:"why_not,omitempty"`
|
|
Running RunningUsage `json:"running"`
|
|
LeftRunning []StepUsage `json:"left_running"`
|
|
// EndedStale are the steps of the last day whose end their build module wrote for a key left running (a restart, or
|
|
// past the step bound): counted here alone, never as finished.
|
|
EndedStale []StepUsage `json:"ended_stale"`
|
|
LastHour WindowUsage `json:"last_hour"`
|
|
LastDay WindowUsage `json:"last_day"`
|
|
Steps []StepUsage `json:"running_steps"`
|
|
}
|
|
|
|
// usageStep is a step read, with the kind it is counted under.
|
|
type usageStep struct {
|
|
step delivery.Step
|
|
kind string
|
|
}
|
|
|
|
// RunningUsage is the asks running now: how many in all and by kind, against the capacity.
|
|
type RunningUsage struct {
|
|
Total int `json:"total"`
|
|
ByKind map[string]int `json:"by_kind"`
|
|
}
|
|
|
|
// StepUsage is one step running now, or one left running and why.
|
|
type StepUsage struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"`
|
|
Tool string `json:"tool"`
|
|
Started time.Time `json:"started"`
|
|
Secs int64 `json:"secs"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// WindowUsage is the steps of one window: started and finished in it, the finished ones' total and median time, by kind,
|
|
// and how long the node ran its capacity of asks (a hand step takes no slot).
|
|
type WindowUsage struct {
|
|
Started int `json:"started"`
|
|
Finished int `json:"finished"`
|
|
Failed int `json:"failed"`
|
|
TotalSecs int64 `json:"total_secs"`
|
|
MedianSecs int64 `json:"median_secs"`
|
|
ByKind map[string]KindWindow `json:"by_kind"`
|
|
AtCapacitySecs int64 `json:"at_capacity_secs"`
|
|
AtCapacityShare float64 `json:"at_capacity_share"`
|
|
}
|
|
|
|
// KindWindow is one kind's finished steps in a window.
|
|
type KindWindow struct {
|
|
Finished int `json:"finished"`
|
|
TotalSecs int64 `json:"total_secs"`
|
|
MedianSecs int64 `json:"median_secs"`
|
|
}
|
|
|
|
// stepIDTime is when a step id was made: its base-36 milliseconds before the six hex digits (buildstep.NewID).
|
|
func stepIDTime(id string) (time.Time, bool) {
|
|
if len(id) <= 6 {
|
|
return time.Time{}, false
|
|
}
|
|
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
return time.UnixMilli(ms).UTC(), true
|
|
}
|
|
|
|
// stepKind is the kind a step is counted under.
|
|
func stepKind(s delivery.Step, tries map[string]bool) string {
|
|
kind, ask := strings.CutSuffix(s.Tool, " ask")
|
|
if !ask {
|
|
return KindHand
|
|
}
|
|
if kind == delivery.AskCheck && tries[s.ID] {
|
|
return KindTry
|
|
}
|
|
return kind
|
|
}
|
|
|
|
// Usage reads every state and answers. A state it cannot read is said in a note, never left out silently.
|
|
func (u UsageReader) Usage() UsageAnswer {
|
|
now := time.Now().UTC()
|
|
if u.Now != nil {
|
|
now = u.Now().UTC()
|
|
}
|
|
ans := UsageAnswer{At: now, Nodes: []NodeUsage{}, Backlog: []BacklogRow{}, TriesRunning: []TryRunning{}}
|
|
note := func(f string, a ...any) { ans.Notes = append(ans.Notes, fmt.Sprintf(f, a...)) }
|
|
|
|
tries := u.tries(now, &ans, note)
|
|
seats := map[string]map[string]*SeatUsage{} // node -> language -> seat
|
|
seat := func(node, lang string) *SeatUsage {
|
|
if seats[node] == nil {
|
|
seats[node] = map[string]*SeatUsage{}
|
|
}
|
|
if s := seats[node][lang]; s != nil {
|
|
return s
|
|
}
|
|
name, _ := delivery.BuildSeat(lang)
|
|
module, _ := delivery.BuildModuleOf(lang)
|
|
s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, EndedStale: []StepUsage{}, Steps: []StepUsage{}}
|
|
seats[node][lang] = s
|
|
return s
|
|
}
|
|
|
|
// Which node runs which build module: the have, <node>.mesh-build-<language>.
|
|
haves := map[string]delivery.Have{}
|
|
if u.Haves.Keys == nil || u.Haves.Get == nil {
|
|
note("the node-engine's have is not readable here, so no seat is said held")
|
|
} else if keys, err := u.Haves.Keys(); err != nil {
|
|
note("the node-engine's have cannot be listed, so no seat is said held: %v", err)
|
|
} else {
|
|
var want []string
|
|
for _, k := range keys {
|
|
for _, lang := range delivery.AskLanguages {
|
|
if m, _ := delivery.BuildModuleOf(lang); strings.HasSuffix(k, "."+m) {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
}
|
|
raws, err := u.Haves.values(want)
|
|
if err != nil {
|
|
note("the node-engine's have cannot be read: %v", err)
|
|
}
|
|
for i, k := range want {
|
|
if i < len(raws) && len(raws[i]) > 0 {
|
|
h, err := delivery.DecodeHave(raws[i])
|
|
if err != nil {
|
|
note("the have %s cannot be read, so its seat is not said held: %v", k, err)
|
|
continue
|
|
}
|
|
haves[k] = h
|
|
}
|
|
}
|
|
}
|
|
for k, h := range haves {
|
|
node, module, _ := strings.Cut(k, ".")
|
|
lang := strings.TrimPrefix(module, "mesh-build-")
|
|
s := seat(node, lang)
|
|
s.Phase, s.Holds = string(h.Phase), h.Phase == delivery.PhaseRunning
|
|
}
|
|
|
|
// Capacity: mesh-config's resolved value of the module's `capacity` on each node.
|
|
resolved := map[string]string{} // <node>.<module> -> setting
|
|
unknown := "" // why this tool cannot say a capacity: mesh-config's values cannot be read here
|
|
unreadable := map[string]string{} // <node>.<module> -> why its resolved key holds no readable value: the puller uses 0
|
|
if u.Resolved.Keys == nil || u.Resolved.Get == nil {
|
|
unknown = "mesh-config's resolved values are not readable here"
|
|
} else if keys, err := u.Resolved.Keys(); err != nil {
|
|
unknown = "mesh-config's resolved values cannot be listed: " + err.Error()
|
|
} else {
|
|
var want []string
|
|
for _, k := range keys {
|
|
if strings.HasSuffix(k, "."+buildstep.CapacitySetting) && strings.Contains(k, ".mesh-build-") {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
raws, err := u.Resolved.values(want)
|
|
if err != nil {
|
|
unknown = "mesh-config's resolved capacities cannot be read: " + err.Error()
|
|
}
|
|
for i, k := range want {
|
|
if err != nil {
|
|
break // the read failed: the values are unknown to this tool, not unreadable in mesh-config
|
|
}
|
|
var r struct {
|
|
Value json.RawMessage `json:"value"`
|
|
Source string `json:"source"`
|
|
}
|
|
key := strings.TrimSuffix(k, "."+buildstep.CapacitySetting)
|
|
setting := ""
|
|
switch {
|
|
case i >= len(raws) || len(raws[i]) == 0:
|
|
unreadable[key] = "its resolved key holds no value"
|
|
case json.Unmarshal(raws[i], &r) != nil:
|
|
unreadable[key] = "its resolved key is not a resolved value"
|
|
case r.Source == "unreadable":
|
|
unreadable[key] = "mesh-config resolves it as unreadable"
|
|
case len(r.Value) == 0 || string(r.Value) == "null" || json.Unmarshal(r.Value, &setting) != nil || strings.TrimSpace(setting) == "":
|
|
unreadable[key] = fmt.Sprintf("its resolved value %s is empty or not a capacity", strings.TrimSpace(string(r.Value)))
|
|
default:
|
|
resolved[key] = setting
|
|
}
|
|
}
|
|
}
|
|
if unknown != "" {
|
|
note("capacity unknown to this tool: %s; eligibility is not answered for a seat that holds its module", unknown)
|
|
}
|
|
for key := range unreadable {
|
|
node, module, _ := strings.Cut(key, ".")
|
|
seat(node, strings.TrimPrefix(module, "mesh-build-"))
|
|
}
|
|
// Every node a capacity names is listed, so "is this node eligible" is answered for any node: the nodes each module's
|
|
// resolved value names (as resolved on the first node in name order), or the modules' default when mesh-config
|
|
// resolves none for the module.
|
|
keys := make([]string, 0, len(resolved))
|
|
for key := range resolved {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
named := map[string]string{} // language -> the module's setting, as resolved on some node
|
|
namedOn := map[string]string{} // language -> that node
|
|
for _, key := range keys {
|
|
node, module, _ := strings.Cut(key, ".")
|
|
lang := strings.TrimPrefix(module, "mesh-build-")
|
|
seat(node, lang)
|
|
if _, ok := named[lang]; !ok {
|
|
named[lang], namedOn[lang] = resolved[key], node
|
|
}
|
|
}
|
|
for _, lang := range delivery.AskLanguages {
|
|
setting, ok := named[lang]
|
|
if !ok {
|
|
setting = buildstep.DefaultCapacities
|
|
}
|
|
if caps, err := buildrun.ParseCapacity(setting); err == nil {
|
|
for node := range caps {
|
|
seat(node, lang)
|
|
}
|
|
}
|
|
}
|
|
// The steps of each build module.
|
|
steps := map[string]map[string][]usageStep{} // node -> language -> steps
|
|
for _, lang := range delivery.AskLanguages {
|
|
module, _ := delivery.BuildModuleOf(lang)
|
|
sr, ok := u.Steps[lang]
|
|
if !ok || sr.Keys == nil || sr.Get == nil {
|
|
note("the steps of %s are not readable here", module)
|
|
continue
|
|
}
|
|
keys, err := sr.Keys()
|
|
if err != nil {
|
|
note("the steps of %s cannot be listed: %v", module, err)
|
|
continue
|
|
}
|
|
// A step id is its start in base-36 milliseconds, so the newest keys are the ones of the last day; at most
|
|
// MaxStateReads of them are read, newest first.
|
|
type keyAt struct {
|
|
key string
|
|
at time.Time
|
|
}
|
|
var recent []keyAt
|
|
for _, k := range keys {
|
|
_, id, err := delivery.ParseStepKey(k)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+delivery.StepBound {
|
|
recent = append(recent, keyAt{k, at})
|
|
}
|
|
}
|
|
sort.Slice(recent, func(i, j int) bool { return recent[i].at.After(recent[j].at) })
|
|
if len(recent) > MaxStateReads {
|
|
note("the steps of %s: read the newest %d of %d keys of the last day; the older ones are not counted", module, MaxStateReads, len(recent))
|
|
recent = recent[:MaxStateReads]
|
|
}
|
|
want := make([]string, len(recent))
|
|
for i, r := range recent {
|
|
want[i] = r.key
|
|
}
|
|
raws, err := sr.values(want)
|
|
if err != nil {
|
|
note("the steps of %s cannot all be read: %v", module, err)
|
|
}
|
|
for i := range want {
|
|
if i >= len(raws) || len(raws[i]) == 0 {
|
|
continue
|
|
}
|
|
s, err := delivery.DecodeStep(raws[i])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if steps[s.Node] == nil {
|
|
steps[s.Node] = map[string][]usageStep{}
|
|
}
|
|
steps[s.Node][lang] = append(steps[s.Node][lang], usageStep{s, stepKind(s, tries)})
|
|
seat(s.Node, lang)
|
|
}
|
|
}
|
|
|
|
// Pauses: paused.<node> of each module; and capacity.<node>, the capacity the module says it judges by.
|
|
paused := map[string]bool{} // <node>.<language>
|
|
judged := map[string]delivery.Capacity{}
|
|
for _, lang := range delivery.AskLanguages {
|
|
module, _ := delivery.BuildModuleOf(lang)
|
|
pr, ok := u.Pauses[lang]
|
|
if !ok || pr.Get == nil {
|
|
note("the pauses of %s are not readable here, so none is said paused", module)
|
|
continue
|
|
}
|
|
for node := range seats {
|
|
if seats[node][lang] == nil {
|
|
continue
|
|
}
|
|
key, err := delivery.PauseKey(node)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
raw, err := pr.Get(key)
|
|
if err != nil {
|
|
note("the pause of %s on %s cannot be read: %v", module, node, err)
|
|
continue
|
|
}
|
|
if len(raw) > 0 {
|
|
if p, err := delivery.DecodePause(raw); err == nil {
|
|
paused[node+"."+lang] = p.Paused
|
|
}
|
|
}
|
|
if key, err := delivery.CapacityKey(node); err == nil {
|
|
if raw, err := pr.Get(key); err == nil && len(raw) > 0 {
|
|
if c, err := delivery.DecodeCapacity(raw); err == nil {
|
|
judged[node+"."+lang] = c
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
nodes := make([]string, 0, len(seats))
|
|
for n := range seats {
|
|
nodes = append(nodes, n)
|
|
}
|
|
sort.Strings(nodes)
|
|
for _, node := range nodes {
|
|
nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{},
|
|
CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"}
|
|
for _, lang := range delivery.AskLanguages {
|
|
s := seats[node][lang]
|
|
if s == nil {
|
|
continue
|
|
}
|
|
setting, ok := resolved[node+"."+s.Module]
|
|
from := "mesh-config, resolved for " + s.Module + " on " + node
|
|
if _, has := named[lang]; !ok && has {
|
|
from = "mesh-config, resolved for " + s.Module + " on " + namedOn[lang] + " (" + s.Module + " is not assigned to " + node + ")"
|
|
// mesh-config resolves the module's value only where it is assigned; elsewhere its capacity is what that
|
|
// value names for the node.
|
|
setting, ok = named[lang], true
|
|
}
|
|
bad, isBad := unreadable[node+"."+s.Module]
|
|
if unknown != "" {
|
|
s.CapacityFrom = "capacity unknown to this tool: " + unknown
|
|
} else if isBad {
|
|
s.CapacityFrom = "mesh-config's value for " + s.Module + " on " + node + " cannot be read (" + bad + "), so its puller uses 0"
|
|
} else if ok {
|
|
s.CapacityFrom = from
|
|
if caps, err := buildrun.ParseCapacity(setting); err != nil {
|
|
s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err)
|
|
} else {
|
|
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
|
}
|
|
} else {
|
|
caps, _ := buildrun.ParseCapacity(buildstep.DefaultCapacities)
|
|
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
|
s.CapacityFrom = fmt.Sprintf("the modules' default %q: mesh-config holds no resolved value for %s on %s", buildstep.DefaultCapacities, s.Module, node)
|
|
}
|
|
if c, ok := judged[node+"."+lang]; ok && s.Holds && unknown == "" && !isBad && c.Configured == s.Capacity && c.Effective != c.Configured {
|
|
// The module running on the node judges by less than mesh-config gives: its node's build CPU bounds it.
|
|
s.Configured, s.Capacity = s.Capacity, c.Effective
|
|
s.CapacityFrom = fmt.Sprintf("%s, as %s on %s judges it (said %s)", c.Why, s.Module, node, c.At.UTC().Format(time.RFC3339))
|
|
}
|
|
s.Paused = paused[node+"."+lang]
|
|
s.EligibleKnown = true
|
|
switch {
|
|
case !s.Holds && s.Phase == "":
|
|
s.WhyNot = s.Module + " is not on " + node
|
|
case !s.Holds:
|
|
s.WhyNot = s.Module + " on " + node + " is " + s.Phase
|
|
case s.Paused:
|
|
s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)"
|
|
case unknown != "":
|
|
s.EligibleKnown = false
|
|
s.WhyNot = "not known: this tool cannot read its capacity (" + unknown + ")"
|
|
case isBad:
|
|
s.WhyNot = "its capacity on " + node + " cannot be read, so its puller uses 0"
|
|
case s.Capacity == 0:
|
|
s.WhyNot = "its capacity on " + node + " is 0"
|
|
}
|
|
if s.EligibleKnown {
|
|
e := s.WhyNot == ""
|
|
s.Eligible = &e
|
|
}
|
|
switch {
|
|
case s.Eligible != nil && *s.Eligible:
|
|
nu.Eligible = "eligible"
|
|
case !s.EligibleKnown && nu.Eligible != "eligible":
|
|
nu.Eligible = "not known"
|
|
case nu.Eligible == "":
|
|
nu.Eligible = "not eligible"
|
|
}
|
|
s.fill(steps[node][lang], now)
|
|
nu.Said = append(nu.Said, s.said())
|
|
nu.Seats = append(nu.Seats, *s)
|
|
}
|
|
ans.Nodes = append(ans.Nodes, nu)
|
|
}
|
|
|
|
if u.Backlog == nil {
|
|
note("the backlog of the seats' streams is not readable here")
|
|
} else if rows, err := u.Backlog(); err != nil {
|
|
note("the backlog of the seats' streams is read by the view program, which did not answer: %v", err)
|
|
} else if rows != nil {
|
|
ans.Backlog = rows
|
|
}
|
|
return ans
|
|
}
|
|
|
|
// tries reads the checker's tries of the last day: the running ones, and the asks every one put.
|
|
func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, ...any)) map[string]bool {
|
|
asks := map[string]bool{}
|
|
if u.Tries.Keys == nil || u.Tries.Get == nil {
|
|
note("the checker's tries are not readable here, so a try's asks are counted under check")
|
|
return asks
|
|
}
|
|
keys, err := u.Tries.Keys()
|
|
if err != nil {
|
|
note("the checker's tries cannot be listed, so a try's asks are counted under check: %v", err)
|
|
return asks
|
|
}
|
|
var want []string
|
|
for _, k := range keys {
|
|
ns, err := strconv.ParseInt(strings.TrimPrefix(k, "try-"), 10, 64)
|
|
if err == nil && strings.HasPrefix(k, "try-") && now.Sub(time.Unix(0, ns)) <= 48*time.Hour {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
unnamed := 0
|
|
raws, err := u.Tries.values(want)
|
|
if err != nil {
|
|
note("the checker's tries cannot all be read: %v", err)
|
|
}
|
|
for i := range want {
|
|
if i >= len(raws) || len(raws[i]) == 0 {
|
|
continue
|
|
}
|
|
var t struct {
|
|
ID string `json:"id"`
|
|
Repository string `json:"repository"`
|
|
State string `json:"state"`
|
|
Started time.Time `json:"started"`
|
|
Asks []string `json:"asks"`
|
|
Steps json.RawMessage `json:"steps"`
|
|
}
|
|
if json.Unmarshal(raws[i], &t) != nil {
|
|
continue
|
|
}
|
|
for _, a := range t.Asks {
|
|
asks[a] = true
|
|
}
|
|
if t.State == "running" {
|
|
ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started, Asks: t.Asks, Steps: t.Steps})
|
|
if len(t.Asks) == 0 {
|
|
unnamed++
|
|
}
|
|
}
|
|
}
|
|
if unnamed > 0 {
|
|
note("%d running tries name no ask yet (none put yet, or a checker that names them only at the end): their asks are counted under check until then", unnamed)
|
|
}
|
|
return asks
|
|
}
|
|
|
|
// fill counts a seat's steps: running now, left running (started past delivery.StepBound, or its module no longer running on
|
|
// the node: no step behind the key runs any more), and the two windows.
|
|
func (s *SeatUsage) fill(steps []usageStep, now time.Time) {
|
|
sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) })
|
|
var live []usageStep // ended, or running and not left: what the windows count
|
|
for _, us := range steps {
|
|
st := us.step
|
|
if why := endedStale(st); why != "" {
|
|
s.EndedStale = append(s.EndedStale, StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Why: why})
|
|
continue
|
|
}
|
|
if st.State != delivery.StepRunning {
|
|
live = append(live, us)
|
|
continue
|
|
}
|
|
line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())}
|
|
switch {
|
|
case !s.Holds:
|
|
line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs"
|
|
case now.Sub(st.Started) > delivery.StepBound:
|
|
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), delivery.StepBound)
|
|
}
|
|
if line.Why != "" {
|
|
s.LeftRunning = append(s.LeftRunning, line)
|
|
continue
|
|
}
|
|
s.Steps = append(s.Steps, line)
|
|
live = append(live, us)
|
|
if us.kind != KindHand {
|
|
s.Running.Total++
|
|
s.Running.ByKind[us.kind]++
|
|
}
|
|
}
|
|
s.LastHour = window(live, now, time.Hour, s.Capacity)
|
|
s.LastDay = window(live, now, 24*time.Hour, s.Capacity)
|
|
}
|
|
|
|
// said is the seat's use in one line.
|
|
func (s *SeatUsage) said() string {
|
|
head := s.Seat + ": eligible"
|
|
switch {
|
|
case !s.EligibleKnown:
|
|
head = s.Seat + ": eligibility " + s.WhyNot
|
|
case s.Eligible == nil || !*s.Eligible:
|
|
head = s.Seat + ": not eligible, " + s.WhyNot
|
|
}
|
|
kinds := make([]string, 0, len(s.Running.ByKind))
|
|
for k, n := range s.Running.ByKind {
|
|
kinds = append(kinds, fmt.Sprintf("%s %d", k, n))
|
|
}
|
|
sort.Strings(kinds)
|
|
line := fmt.Sprintf("%s; %d of capacity %d running", head, s.Running.Total, s.Capacity)
|
|
if len(kinds) > 0 {
|
|
line += " (" + strings.Join(kinds, ", ") + ")"
|
|
}
|
|
if len(s.LeftRunning) > 0 {
|
|
line += fmt.Sprintf(", %d left running", len(s.LeftRunning))
|
|
}
|
|
return line + fmt.Sprintf("; last hour %d finished, median %d s, at capacity %.0f%% of it; last day %d finished, median %d s, at capacity %.1f%% of it",
|
|
s.LastHour.Finished, s.LastHour.MedianSecs, s.LastHour.AtCapacityShare*100, s.LastDay.Finished, s.LastDay.MedianSecs, s.LastDay.AtCapacityShare*100)
|
|
}
|
|
|
|
// window counts the steps of the last span before now: started in it, finished in it with their total and median time,
|
|
// and the time in it during which capacity asks (not hand steps) ran at once.
|
|
func window(steps []usageStep, now time.Time, span time.Duration, capacity int) WindowUsage {
|
|
from := now.Add(-span)
|
|
w := WindowUsage{ByKind: map[string]KindWindow{}}
|
|
var all []int64
|
|
byKind := map[string][]int64{}
|
|
type edge struct {
|
|
at time.Time
|
|
d int
|
|
}
|
|
var edges []edge
|
|
for _, us := range steps {
|
|
st := us.step
|
|
if !st.Started.Before(from) {
|
|
w.Started++
|
|
}
|
|
if !st.Ended.IsZero() && !st.Ended.Before(from) {
|
|
w.Finished++
|
|
if st.State == delivery.StepFailed || st.State == delivery.StepRefused {
|
|
w.Failed++
|
|
}
|
|
secs := int64(st.Ended.Sub(st.Started).Seconds())
|
|
all = append(all, secs)
|
|
byKind[us.kind] = append(byKind[us.kind], secs)
|
|
}
|
|
if us.kind == KindHand {
|
|
continue
|
|
}
|
|
start, end := st.Started, st.Ended
|
|
if end.IsZero() {
|
|
end = now
|
|
}
|
|
if start.Before(from) {
|
|
start = from
|
|
}
|
|
if end.After(start) {
|
|
edges = append(edges, edge{start, 1}, edge{end, -1})
|
|
}
|
|
}
|
|
w.TotalSecs, w.MedianSecs = sumMedian(all)
|
|
for k, d := range byKind {
|
|
kw := KindWindow{Finished: len(d)}
|
|
kw.TotalSecs, kw.MedianSecs = sumMedian(d)
|
|
w.ByKind[k] = kw
|
|
}
|
|
if capacity > 0 {
|
|
// An end before a start at the same instant: a slot handed from one ask to the next is not a moment at capacity+1.
|
|
sort.Slice(edges, func(i, j int) bool {
|
|
if edges[i].at.Equal(edges[j].at) {
|
|
return edges[i].d < edges[j].d
|
|
}
|
|
return edges[i].at.Before(edges[j].at)
|
|
})
|
|
var at time.Duration
|
|
n := 0
|
|
for i, e := range edges {
|
|
n += e.d
|
|
if n >= capacity && i+1 < len(edges) {
|
|
at += edges[i+1].at.Sub(e.at)
|
|
}
|
|
}
|
|
w.AtCapacitySecs = int64(at.Seconds())
|
|
w.AtCapacityShare = float64(int64(at*1000/span)) / 1000
|
|
}
|
|
return w
|
|
}
|
|
|
|
// sumMedian is the sum and the median of d (the lower middle of an even count); zero for none.
|
|
func sumMedian(d []int64) (sum, median int64) {
|
|
if len(d) == 0 {
|
|
return 0, 0
|
|
}
|
|
s := append([]int64(nil), d...)
|
|
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
|
for _, v := range s {
|
|
sum += v
|
|
}
|
|
return sum, s[(len(s)-1)/2]
|
|
}
|