jschoubben
c47afd8f4b
A build module's capacity.<node> now carries its node's build window (the operator, 2026-10-26: g14 builds only from 18:00 to 06:00); the seat shows it, outside it says it takes nothing new, and a value the module cannot read shows as any hour with the value and why. The checker's and the view's SDK pin moves to mesh-sdk d00dac1.
734 lines
27 KiB
Go
734 lines
27 KiB
Go
package view
|
|
|
|
// The build nodes' use, in one answer (the operator, 2026-10-26: "how much are our build agents being used, and is ace
|
|
// eligible for building?" took six calls over four tools). Read from the bus's state alone: which build seats each node
|
|
// holds (the node-engine's have of mesh-build-<language>), its capacity for each (mesh-config's resolved value of the
|
|
// module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each
|
|
// build module's steps state (ADR 0525 §6: an ask ends on <node>.<ask id>) the asks running now by kind, the ones left
|
|
// running, and the steps started and finished in the last hour and day with their total and median time and how long the
|
|
// node was at its capacity. A try's asks are told from a candidate's by the checker's tries state (each try names the
|
|
// asks it put). The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver),
|
|
// asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the
|
|
// bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
)
|
|
|
|
// BuildUsageTool is the view's tool of the build nodes' use.
|
|
const BuildUsageTool = "build_usage"
|
|
|
|
// endedStale says why a step's end was written by its build module for a key left running (mesh-sdk buildstep's ending
|
|
// of stale steps): a restart's, or the step bound's; "" for a step that ran to its own end. Such an end is the moment the
|
|
// module noticed, not the step's, so it is counted apart and in neither the medians nor the time at capacity.
|
|
func endedStale(st delivery.Step) string {
|
|
if st.State != delivery.StepFailed {
|
|
return ""
|
|
}
|
|
if st.Reason == buildstep.RestartReason || strings.HasPrefix(st.Reason, "past the step bound") {
|
|
return st.Reason
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker
|
|
// put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity).
|
|
const (
|
|
KindTry = "try"
|
|
KindHand = "hand"
|
|
)
|
|
|
|
// UsageReader reads what build_usage answers from.
|
|
type UsageReader struct {
|
|
// Steps and Pauses are each build module's states by language (go, node, image).
|
|
Steps map[string]StateRead
|
|
Pauses map[string]StateRead
|
|
// Resolved is mesh-config's resolved values (<node>.<module>.<key>); Haves the node-engine's have.
|
|
Resolved StateRead
|
|
Haves StateRead
|
|
// Tries is the checker's tries (try-<unix ns>), each naming the asks it put.
|
|
Tries StateRead
|
|
// Backlog is the seats' streams' waiting and held asks, as the view program reads them.
|
|
Backlog func() ([]BacklogRow, error)
|
|
Now func() time.Time
|
|
}
|
|
|
|
// UsageAnswer is build_usage's answer.
|
|
type UsageAnswer struct {
|
|
At time.Time `json:"at"`
|
|
Nodes []NodeUsage `json:"nodes"`
|
|
Backlog []BacklogRow `json:"backlog"`
|
|
// TriesRunning are the tries the checker runs now, each with its asks and where each stands (a checker that names its
|
|
// asks as they are put, issue 1890; an older one names them only at the end, and they are counted under check until then).
|
|
TriesRunning []TryRunning `json:"tries_running"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// TryRunning is one try the checker runs now.
|
|
type TryRunning struct {
|
|
ID string `json:"id"`
|
|
Repository string `json:"repository"`
|
|
Started time.Time `json:"started,omitzero"`
|
|
Asks []string `json:"asks,omitempty"`
|
|
// Steps are the try's own words for each ask: asked, or the node that took it, its state and the phase its step says.
|
|
Steps json.RawMessage `json:"steps,omitempty"`
|
|
}
|
|
|
|
// NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU.
|
|
type NodeUsage struct {
|
|
Node string `json:"node"`
|
|
// Eligible is the node's verdict: "eligible" when a seat of it is, "not eligible" when every seat is known not to be,
|
|
// and "not known" when none is known eligible and this tool cannot tell for some seat.
|
|
Eligible string `json:"eligible"`
|
|
// Said is one line per seat, the answer in words.
|
|
Said []string `json:"said"`
|
|
Seats []SeatUsage `json:"seats"`
|
|
CPU string `json:"cpu"`
|
|
}
|
|
|
|
// SeatUsage is one build seat on one node.
|
|
type SeatUsage struct {
|
|
Seat string `json:"seat"`
|
|
Module string `json:"module"`
|
|
// Holds says the node runs the seat's module (its have says running); Phase is what its have says, or absent.
|
|
Holds bool `json:"holds"`
|
|
Phase string `json:"phase,omitempty"`
|
|
// Capacity is the asks it runs at once; CapacityFrom where that came from.
|
|
Capacity int `json:"capacity"`
|
|
CapacityFrom string `json:"capacity_from"`
|
|
// Configured is mesh-config's capacity when the module says it judges by a lower one (capacity.<node> of its pauses
|
|
// state: the configured capacity bounded by its node's build CPU, issue 1806's follow-up); Capacity is then the
|
|
// module's and CapacityFrom its why.
|
|
Configured int `json:"configured,omitempty"`
|
|
// BuildHours is the node's build window as the module says it (HH:MM-HH:MM and its zone), absent when it builds at any
|
|
// hour; OutsideHours says the module judged the time outside it, so it takes nothing new (the operator, 2026-10-26).
|
|
BuildHours string `json:"build_hours,omitempty"`
|
|
OutsideHours bool `json:"outside_hours,omitempty"`
|
|
// BuildHoursFault is why the module cannot read its node's build-hours value, naming it: it then builds at any hour.
|
|
BuildHoursFault string `json:"build_hours_fault,omitempty"`
|
|
Paused bool `json:"paused"`
|
|
// Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not.
|
|
// EligibleKnown is false when this tool cannot tell (it cannot read mesh-config's capacities): Eligible is then
|
|
// absent, never false, and WhyNot starts "not known: ".
|
|
Eligible *bool `json:"eligible,omitempty"`
|
|
EligibleKnown bool `json:"eligible_known"`
|
|
WhyNot string `json:"why_not,omitempty"`
|
|
Running RunningUsage `json:"running"`
|
|
LeftRunning []StepUsage `json:"left_running"`
|
|
// EndedStale are the steps of the last day whose end their build module wrote for a key left running (a restart, or
|
|
// past the step bound): counted here alone, never as finished.
|
|
EndedStale []StepUsage `json:"ended_stale"`
|
|
LastHour WindowUsage `json:"last_hour"`
|
|
LastDay WindowUsage `json:"last_day"`
|
|
Steps []StepUsage `json:"running_steps"`
|
|
}
|
|
|
|
// usageStep is a step read, with the kind it is counted under.
|
|
type usageStep struct {
|
|
step delivery.Step
|
|
kind string
|
|
}
|
|
|
|
// RunningUsage is the asks running now: how many in all and by kind, against the capacity.
|
|
type RunningUsage struct {
|
|
Total int `json:"total"`
|
|
ByKind map[string]int `json:"by_kind"`
|
|
}
|
|
|
|
// StepUsage is one step running now, or one left running and why.
|
|
type StepUsage struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"`
|
|
Tool string `json:"tool"`
|
|
Started time.Time `json:"started"`
|
|
Secs int64 `json:"secs"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// WindowUsage is the steps of one window: started and finished in it, the finished ones' total and median time, by kind,
|
|
// and how long the node ran its capacity of asks (a hand step takes no slot).
|
|
type WindowUsage struct {
|
|
Started int `json:"started"`
|
|
Finished int `json:"finished"`
|
|
Failed int `json:"failed"`
|
|
TotalSecs int64 `json:"total_secs"`
|
|
MedianSecs int64 `json:"median_secs"`
|
|
ByKind map[string]KindWindow `json:"by_kind"`
|
|
AtCapacitySecs int64 `json:"at_capacity_secs"`
|
|
AtCapacityShare float64 `json:"at_capacity_share"`
|
|
}
|
|
|
|
// KindWindow is one kind's finished steps in a window.
|
|
type KindWindow struct {
|
|
Finished int `json:"finished"`
|
|
TotalSecs int64 `json:"total_secs"`
|
|
MedianSecs int64 `json:"median_secs"`
|
|
}
|
|
|
|
// stepIDTime is when a step id was made: its base-36 milliseconds before the six hex digits (buildstep.NewID).
|
|
func stepIDTime(id string) (time.Time, bool) {
|
|
if len(id) <= 6 {
|
|
return time.Time{}, false
|
|
}
|
|
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
return time.UnixMilli(ms).UTC(), true
|
|
}
|
|
|
|
// stepKind is the kind a step is counted under.
|
|
func stepKind(s delivery.Step, tries map[string]bool) string {
|
|
kind, ask := strings.CutSuffix(s.Tool, " ask")
|
|
if !ask {
|
|
return KindHand
|
|
}
|
|
if kind == delivery.AskCheck && tries[s.ID] {
|
|
return KindTry
|
|
}
|
|
return kind
|
|
}
|
|
|
|
// Usage reads every state and answers. A state it cannot read is said in a note, never left out silently.
|
|
func (u UsageReader) Usage() UsageAnswer {
|
|
now := time.Now().UTC()
|
|
if u.Now != nil {
|
|
now = u.Now().UTC()
|
|
}
|
|
ans := UsageAnswer{At: now, Nodes: []NodeUsage{}, Backlog: []BacklogRow{}, TriesRunning: []TryRunning{}}
|
|
note := func(f string, a ...any) { ans.Notes = append(ans.Notes, fmt.Sprintf(f, a...)) }
|
|
|
|
tries := u.tries(now, &ans, note)
|
|
seats := map[string]map[string]*SeatUsage{} // node -> language -> seat
|
|
seat := func(node, lang string) *SeatUsage {
|
|
if seats[node] == nil {
|
|
seats[node] = map[string]*SeatUsage{}
|
|
}
|
|
if s := seats[node][lang]; s != nil {
|
|
return s
|
|
}
|
|
name, _ := delivery.BuildSeat(lang)
|
|
module, _ := delivery.BuildModuleOf(lang)
|
|
s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, EndedStale: []StepUsage{}, Steps: []StepUsage{}}
|
|
seats[node][lang] = s
|
|
return s
|
|
}
|
|
|
|
// Which node runs which build module: the have, <node>.mesh-build-<language>.
|
|
haves := map[string]delivery.Have{}
|
|
if u.Haves.Keys == nil || u.Haves.Get == nil {
|
|
note("the node-engine's have is not readable here, so no seat is said held")
|
|
} else if keys, err := u.Haves.Keys(); err != nil {
|
|
note("the node-engine's have cannot be listed, so no seat is said held: %v", err)
|
|
} else {
|
|
var want []string
|
|
for _, k := range keys {
|
|
for _, lang := range delivery.AskLanguages {
|
|
if m, _ := delivery.BuildModuleOf(lang); strings.HasSuffix(k, "."+m) {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
}
|
|
raws, err := u.Haves.values(want)
|
|
if err != nil {
|
|
note("the node-engine's have cannot be read: %v", err)
|
|
}
|
|
for i, k := range want {
|
|
if i < len(raws) && len(raws[i]) > 0 {
|
|
h, err := delivery.DecodeHave(raws[i])
|
|
if err != nil {
|
|
note("the have %s cannot be read, so its seat is not said held: %v", k, err)
|
|
continue
|
|
}
|
|
haves[k] = h
|
|
}
|
|
}
|
|
}
|
|
for k, h := range haves {
|
|
node, module, _ := strings.Cut(k, ".")
|
|
lang := strings.TrimPrefix(module, "mesh-build-")
|
|
s := seat(node, lang)
|
|
s.Phase, s.Holds = string(h.Phase), h.Phase == delivery.PhaseRunning
|
|
}
|
|
|
|
// Capacity: mesh-config's resolved value of the module's `capacity` on each node.
|
|
resolved := map[string]string{} // <node>.<module> -> setting
|
|
unknown := "" // why this tool cannot say a capacity: mesh-config's values cannot be read here
|
|
unreadable := map[string]string{} // <node>.<module> -> why its resolved key holds no readable value: the puller uses 0
|
|
if u.Resolved.Keys == nil || u.Resolved.Get == nil {
|
|
unknown = "mesh-config's resolved values are not readable here"
|
|
} else if keys, err := u.Resolved.Keys(); err != nil {
|
|
unknown = "mesh-config's resolved values cannot be listed: " + err.Error()
|
|
} else {
|
|
var want []string
|
|
for _, k := range keys {
|
|
if strings.HasSuffix(k, "."+buildstep.CapacitySetting) && strings.Contains(k, ".mesh-build-") {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
raws, err := u.Resolved.values(want)
|
|
if err != nil {
|
|
unknown = "mesh-config's resolved capacities cannot be read: " + err.Error()
|
|
}
|
|
for i, k := range want {
|
|
if err != nil {
|
|
break // the read failed: the values are unknown to this tool, not unreadable in mesh-config
|
|
}
|
|
var r struct {
|
|
Value json.RawMessage `json:"value"`
|
|
Source string `json:"source"`
|
|
}
|
|
key := strings.TrimSuffix(k, "."+buildstep.CapacitySetting)
|
|
setting := ""
|
|
switch {
|
|
case i >= len(raws) || len(raws[i]) == 0:
|
|
unreadable[key] = "its resolved key holds no value"
|
|
case json.Unmarshal(raws[i], &r) != nil:
|
|
unreadable[key] = "its resolved key is not a resolved value"
|
|
case r.Source == "unreadable":
|
|
unreadable[key] = "mesh-config resolves it as unreadable"
|
|
case len(r.Value) == 0 || string(r.Value) == "null" || json.Unmarshal(r.Value, &setting) != nil || strings.TrimSpace(setting) == "":
|
|
unreadable[key] = fmt.Sprintf("its resolved value %s is empty or not a capacity", strings.TrimSpace(string(r.Value)))
|
|
default:
|
|
resolved[key] = setting
|
|
}
|
|
}
|
|
}
|
|
if unknown != "" {
|
|
note("capacity unknown to this tool: %s; eligibility is not answered for a seat that holds its module", unknown)
|
|
}
|
|
for key := range unreadable {
|
|
node, module, _ := strings.Cut(key, ".")
|
|
seat(node, strings.TrimPrefix(module, "mesh-build-"))
|
|
}
|
|
// Every node a capacity names is listed, so "is this node eligible" is answered for any node: the nodes each module's
|
|
// resolved value names (as resolved on the first node in name order), or the modules' default when mesh-config
|
|
// resolves none for the module.
|
|
keys := make([]string, 0, len(resolved))
|
|
for key := range resolved {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
named := map[string]string{} // language -> the module's setting, as resolved on some node
|
|
namedOn := map[string]string{} // language -> that node
|
|
for _, key := range keys {
|
|
node, module, _ := strings.Cut(key, ".")
|
|
lang := strings.TrimPrefix(module, "mesh-build-")
|
|
seat(node, lang)
|
|
if _, ok := named[lang]; !ok {
|
|
named[lang], namedOn[lang] = resolved[key], node
|
|
}
|
|
}
|
|
for _, lang := range delivery.AskLanguages {
|
|
setting, ok := named[lang]
|
|
if !ok {
|
|
setting = buildstep.DefaultCapacities
|
|
}
|
|
if caps, err := buildrun.ParseCapacity(setting); err == nil {
|
|
for node := range caps {
|
|
seat(node, lang)
|
|
}
|
|
}
|
|
}
|
|
// The steps of each build module.
|
|
steps := map[string]map[string][]usageStep{} // node -> language -> steps
|
|
for _, lang := range delivery.AskLanguages {
|
|
module, _ := delivery.BuildModuleOf(lang)
|
|
sr, ok := u.Steps[lang]
|
|
if !ok || sr.Keys == nil || sr.Get == nil {
|
|
note("the steps of %s are not readable here", module)
|
|
continue
|
|
}
|
|
keys, err := sr.Keys()
|
|
if err != nil {
|
|
note("the steps of %s cannot be listed: %v", module, err)
|
|
continue
|
|
}
|
|
// A step id is its start in base-36 milliseconds, so the newest keys are the ones of the last day; at most
|
|
// MaxStateReads of them are read, newest first.
|
|
type keyAt struct {
|
|
key string
|
|
at time.Time
|
|
}
|
|
var recent []keyAt
|
|
for _, k := range keys {
|
|
_, id, err := delivery.ParseStepKey(k)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+delivery.StepBound {
|
|
recent = append(recent, keyAt{k, at})
|
|
}
|
|
}
|
|
sort.Slice(recent, func(i, j int) bool { return recent[i].at.After(recent[j].at) })
|
|
if len(recent) > MaxStateReads {
|
|
note("the steps of %s: read the newest %d of %d keys of the last day; the older ones are not counted", module, MaxStateReads, len(recent))
|
|
recent = recent[:MaxStateReads]
|
|
}
|
|
want := make([]string, len(recent))
|
|
for i, r := range recent {
|
|
want[i] = r.key
|
|
}
|
|
raws, err := sr.values(want)
|
|
if err != nil {
|
|
note("the steps of %s cannot all be read: %v", module, err)
|
|
}
|
|
for i := range want {
|
|
if i >= len(raws) || len(raws[i]) == 0 {
|
|
continue
|
|
}
|
|
s, err := delivery.DecodeStep(raws[i])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if steps[s.Node] == nil {
|
|
steps[s.Node] = map[string][]usageStep{}
|
|
}
|
|
steps[s.Node][lang] = append(steps[s.Node][lang], usageStep{s, stepKind(s, tries)})
|
|
seat(s.Node, lang)
|
|
}
|
|
}
|
|
|
|
// Pauses: paused.<node> of each module; and capacity.<node>, the capacity the module says it judges by.
|
|
paused := map[string]bool{} // <node>.<language>
|
|
judged := map[string]delivery.Capacity{}
|
|
for _, lang := range delivery.AskLanguages {
|
|
module, _ := delivery.BuildModuleOf(lang)
|
|
pr, ok := u.Pauses[lang]
|
|
if !ok || pr.Get == nil {
|
|
note("the pauses of %s are not readable here, so none is said paused", module)
|
|
continue
|
|
}
|
|
for node := range seats {
|
|
if seats[node][lang] == nil {
|
|
continue
|
|
}
|
|
key, err := delivery.PauseKey(node)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
raw, err := pr.Get(key)
|
|
if err != nil {
|
|
note("the pause of %s on %s cannot be read: %v", module, node, err)
|
|
continue
|
|
}
|
|
if len(raw) > 0 {
|
|
if p, err := delivery.DecodePause(raw); err == nil {
|
|
paused[node+"."+lang] = p.Paused
|
|
}
|
|
}
|
|
if key, err := delivery.CapacityKey(node); err == nil {
|
|
if raw, err := pr.Get(key); err == nil && len(raw) > 0 {
|
|
if c, err := delivery.DecodeCapacity(raw); err == nil {
|
|
judged[node+"."+lang] = c
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
nodes := make([]string, 0, len(seats))
|
|
for n := range seats {
|
|
nodes = append(nodes, n)
|
|
}
|
|
sort.Strings(nodes)
|
|
for _, node := range nodes {
|
|
nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{},
|
|
CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"}
|
|
for _, lang := range delivery.AskLanguages {
|
|
s := seats[node][lang]
|
|
if s == nil {
|
|
continue
|
|
}
|
|
setting, ok := resolved[node+"."+s.Module]
|
|
from := "mesh-config, resolved for " + s.Module + " on " + node
|
|
if _, has := named[lang]; !ok && has {
|
|
from = "mesh-config, resolved for " + s.Module + " on " + namedOn[lang] + " (" + s.Module + " is not assigned to " + node + ")"
|
|
// mesh-config resolves the module's value only where it is assigned; elsewhere its capacity is what that
|
|
// value names for the node.
|
|
setting, ok = named[lang], true
|
|
}
|
|
bad, isBad := unreadable[node+"."+s.Module]
|
|
if unknown != "" {
|
|
s.CapacityFrom = "capacity unknown to this tool: " + unknown
|
|
} else if isBad {
|
|
s.CapacityFrom = "mesh-config's value for " + s.Module + " on " + node + " cannot be read (" + bad + "), so its puller uses 0"
|
|
} else if ok {
|
|
s.CapacityFrom = from
|
|
if caps, err := buildrun.ParseCapacity(setting); err != nil {
|
|
s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err)
|
|
} else {
|
|
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
|
}
|
|
} else {
|
|
caps, _ := buildrun.ParseCapacity(buildstep.DefaultCapacities)
|
|
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
|
s.CapacityFrom = fmt.Sprintf("the modules' default %q: mesh-config holds no resolved value for %s on %s", buildstep.DefaultCapacities, s.Module, node)
|
|
}
|
|
if c, ok := judged[node+"."+lang]; ok && s.Holds && unknown == "" && !isBad && c.Configured == s.Capacity {
|
|
// The module running on the node says its build hours, and judges by less than mesh-config gives when its
|
|
// node's build CPU bounds it or the time is outside its build hours.
|
|
s.BuildHours, s.OutsideHours = c.BuildHours, c.OutsideHours
|
|
if c.BuildHoursFault != "" {
|
|
s.BuildHours, s.BuildHoursFault = "any hour", c.BuildHoursFault
|
|
}
|
|
if c.Effective != c.Configured {
|
|
s.Configured, s.Capacity = s.Capacity, c.Effective
|
|
s.CapacityFrom = fmt.Sprintf("%s, as %s on %s judges it (said %s)", c.Why, s.Module, node, c.At.UTC().Format(time.RFC3339))
|
|
}
|
|
}
|
|
s.Paused = paused[node+"."+lang]
|
|
s.EligibleKnown = true
|
|
switch {
|
|
case !s.Holds && s.Phase == "":
|
|
s.WhyNot = s.Module + " is not on " + node
|
|
case !s.Holds:
|
|
s.WhyNot = s.Module + " on " + node + " is " + s.Phase
|
|
case s.Paused:
|
|
s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)"
|
|
case unknown != "":
|
|
s.EligibleKnown = false
|
|
s.WhyNot = "not known: this tool cannot read its capacity (" + unknown + ")"
|
|
case isBad:
|
|
s.WhyNot = "its capacity on " + node + " cannot be read, so its puller uses 0"
|
|
case s.OutsideHours:
|
|
s.WhyNot = "outside its build hours " + s.BuildHours + " on " + node + ": it takes nothing new until they open"
|
|
case s.Capacity == 0:
|
|
s.WhyNot = "its capacity on " + node + " is 0"
|
|
}
|
|
if s.EligibleKnown {
|
|
e := s.WhyNot == ""
|
|
s.Eligible = &e
|
|
}
|
|
switch {
|
|
case s.Eligible != nil && *s.Eligible:
|
|
nu.Eligible = "eligible"
|
|
case !s.EligibleKnown && nu.Eligible != "eligible":
|
|
nu.Eligible = "not known"
|
|
case nu.Eligible == "":
|
|
nu.Eligible = "not eligible"
|
|
}
|
|
s.fill(steps[node][lang], now)
|
|
nu.Said = append(nu.Said, s.said())
|
|
nu.Seats = append(nu.Seats, *s)
|
|
}
|
|
ans.Nodes = append(ans.Nodes, nu)
|
|
}
|
|
|
|
if u.Backlog == nil {
|
|
note("the backlog of the seats' streams is not readable here")
|
|
} else if rows, err := u.Backlog(); err != nil {
|
|
note("the backlog of the seats' streams is read by the view program, which did not answer: %v", err)
|
|
} else if rows != nil {
|
|
ans.Backlog = rows
|
|
}
|
|
return ans
|
|
}
|
|
|
|
// tries reads the checker's tries of the last day: the running ones, and the asks every one put.
|
|
func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, ...any)) map[string]bool {
|
|
asks := map[string]bool{}
|
|
if u.Tries.Keys == nil || u.Tries.Get == nil {
|
|
note("the checker's tries are not readable here, so a try's asks are counted under check")
|
|
return asks
|
|
}
|
|
keys, err := u.Tries.Keys()
|
|
if err != nil {
|
|
note("the checker's tries cannot be listed, so a try's asks are counted under check: %v", err)
|
|
return asks
|
|
}
|
|
var want []string
|
|
for _, k := range keys {
|
|
ns, err := strconv.ParseInt(strings.TrimPrefix(k, "try-"), 10, 64)
|
|
if err == nil && strings.HasPrefix(k, "try-") && now.Sub(time.Unix(0, ns)) <= 48*time.Hour {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
unnamed := 0
|
|
raws, err := u.Tries.values(want)
|
|
if err != nil {
|
|
note("the checker's tries cannot all be read: %v", err)
|
|
}
|
|
for i := range want {
|
|
if i >= len(raws) || len(raws[i]) == 0 {
|
|
continue
|
|
}
|
|
var t struct {
|
|
ID string `json:"id"`
|
|
Repository string `json:"repository"`
|
|
State string `json:"state"`
|
|
Started time.Time `json:"started"`
|
|
Asks []string `json:"asks"`
|
|
Steps json.RawMessage `json:"steps"`
|
|
}
|
|
if json.Unmarshal(raws[i], &t) != nil {
|
|
continue
|
|
}
|
|
for _, a := range t.Asks {
|
|
asks[a] = true
|
|
}
|
|
if t.State == "running" {
|
|
ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started, Asks: t.Asks, Steps: t.Steps})
|
|
if len(t.Asks) == 0 {
|
|
unnamed++
|
|
}
|
|
}
|
|
}
|
|
if unnamed > 0 {
|
|
note("%d running tries name no ask yet (none put yet, or a checker that names them only at the end): their asks are counted under check until then", unnamed)
|
|
}
|
|
return asks
|
|
}
|
|
|
|
// fill counts a seat's steps: running now, left running (started past delivery.StepBound, or its module no longer running on
|
|
// the node: no step behind the key runs any more), and the two windows.
|
|
func (s *SeatUsage) fill(steps []usageStep, now time.Time) {
|
|
sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) })
|
|
var live []usageStep // ended, or running and not left: what the windows count
|
|
for _, us := range steps {
|
|
st := us.step
|
|
if why := endedStale(st); why != "" {
|
|
s.EndedStale = append(s.EndedStale, StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Why: why})
|
|
continue
|
|
}
|
|
if st.State != delivery.StepRunning {
|
|
live = append(live, us)
|
|
continue
|
|
}
|
|
line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())}
|
|
switch {
|
|
case !s.Holds:
|
|
line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs"
|
|
case now.Sub(st.Started) > delivery.StepBound:
|
|
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), delivery.StepBound)
|
|
}
|
|
if line.Why != "" {
|
|
s.LeftRunning = append(s.LeftRunning, line)
|
|
continue
|
|
}
|
|
s.Steps = append(s.Steps, line)
|
|
live = append(live, us)
|
|
if us.kind != KindHand {
|
|
s.Running.Total++
|
|
s.Running.ByKind[us.kind]++
|
|
}
|
|
}
|
|
s.LastHour = window(live, now, time.Hour, s.Capacity)
|
|
s.LastDay = window(live, now, 24*time.Hour, s.Capacity)
|
|
}
|
|
|
|
// said is the seat's use in one line.
|
|
func (s *SeatUsage) said() string {
|
|
head := s.Seat + ": eligible"
|
|
switch {
|
|
case !s.EligibleKnown:
|
|
head = s.Seat + ": eligibility " + s.WhyNot
|
|
case s.Eligible == nil || !*s.Eligible:
|
|
head = s.Seat + ": not eligible, " + s.WhyNot
|
|
}
|
|
kinds := make([]string, 0, len(s.Running.ByKind))
|
|
for k, n := range s.Running.ByKind {
|
|
kinds = append(kinds, fmt.Sprintf("%s %d", k, n))
|
|
}
|
|
sort.Strings(kinds)
|
|
line := fmt.Sprintf("%s; %d of capacity %d running", head, s.Running.Total, s.Capacity)
|
|
if len(kinds) > 0 {
|
|
line += " (" + strings.Join(kinds, ", ") + ")"
|
|
}
|
|
if len(s.LeftRunning) > 0 {
|
|
line += fmt.Sprintf(", %d left running", len(s.LeftRunning))
|
|
}
|
|
return line + fmt.Sprintf("; last hour %d finished, median %d s, at capacity %.0f%% of it; last day %d finished, median %d s, at capacity %.1f%% of it",
|
|
s.LastHour.Finished, s.LastHour.MedianSecs, s.LastHour.AtCapacityShare*100, s.LastDay.Finished, s.LastDay.MedianSecs, s.LastDay.AtCapacityShare*100)
|
|
}
|
|
|
|
// window counts the steps of the last span before now: started in it, finished in it with their total and median time,
|
|
// and the time in it during which capacity asks (not hand steps) ran at once.
|
|
func window(steps []usageStep, now time.Time, span time.Duration, capacity int) WindowUsage {
|
|
from := now.Add(-span)
|
|
w := WindowUsage{ByKind: map[string]KindWindow{}}
|
|
var all []int64
|
|
byKind := map[string][]int64{}
|
|
type edge struct {
|
|
at time.Time
|
|
d int
|
|
}
|
|
var edges []edge
|
|
for _, us := range steps {
|
|
st := us.step
|
|
if !st.Started.Before(from) {
|
|
w.Started++
|
|
}
|
|
if !st.Ended.IsZero() && !st.Ended.Before(from) {
|
|
w.Finished++
|
|
if st.State == delivery.StepFailed || st.State == delivery.StepRefused {
|
|
w.Failed++
|
|
}
|
|
secs := int64(st.Ended.Sub(st.Started).Seconds())
|
|
all = append(all, secs)
|
|
byKind[us.kind] = append(byKind[us.kind], secs)
|
|
}
|
|
if us.kind == KindHand {
|
|
continue
|
|
}
|
|
start, end := st.Started, st.Ended
|
|
if end.IsZero() {
|
|
end = now
|
|
}
|
|
if start.Before(from) {
|
|
start = from
|
|
}
|
|
if end.After(start) {
|
|
edges = append(edges, edge{start, 1}, edge{end, -1})
|
|
}
|
|
}
|
|
w.TotalSecs, w.MedianSecs = sumMedian(all)
|
|
for k, d := range byKind {
|
|
kw := KindWindow{Finished: len(d)}
|
|
kw.TotalSecs, kw.MedianSecs = sumMedian(d)
|
|
w.ByKind[k] = kw
|
|
}
|
|
if capacity > 0 {
|
|
// An end before a start at the same instant: a slot handed from one ask to the next is not a moment at capacity+1.
|
|
sort.Slice(edges, func(i, j int) bool {
|
|
if edges[i].at.Equal(edges[j].at) {
|
|
return edges[i].d < edges[j].d
|
|
}
|
|
return edges[i].at.Before(edges[j].at)
|
|
})
|
|
var at time.Duration
|
|
n := 0
|
|
for i, e := range edges {
|
|
n += e.d
|
|
if n >= capacity && i+1 < len(edges) {
|
|
at += edges[i+1].at.Sub(e.at)
|
|
}
|
|
}
|
|
w.AtCapacitySecs = int64(at.Seconds())
|
|
w.AtCapacityShare = float64(int64(at*1000/span)) / 1000
|
|
}
|
|
return w
|
|
}
|
|
|
|
// sumMedian is the sum and the median of d (the lower middle of an even count); zero for none.
|
|
func sumMedian(d []int64) (sum, median int64) {
|
|
if len(d) == 0 {
|
|
return 0, 0
|
|
}
|
|
s := append([]int64(nil), d...)
|
|
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
|
for _, v := range s {
|
|
sum += v
|
|
}
|
|
return sum, s[(len(s)-1)/2]
|
|
}
|