View: build_usage answers how the build nodes are used in one call; the checker's tries name their asks
The operator's question (how much are the build nodes used, is ace eligible) took six calls over four tools. build_usage reads it from the bus's states: seats held (have), capacity (mesh-config resolved), pauses, the build modules' steps by kind with left-running keys, the last hour and day with total and median time and time at capacity, the running tries, and the backlog the view program reads from the seats' streams. A try now names the asks it put, and the checker announces its tries readable, so the view tells a try's asks from a check's.
This commit is contained in:
@@ -0,0 +1,592 @@
|
||||
package view
|
||||
|
||||
// The build nodes' use, in one answer (the operator, 2026-10-26: "how much are our build agents being used, and is ace
|
||||
// eligible for building?" took six calls over four tools). Read from the bus's state alone: which build seats each node
|
||||
// holds (the node-engine's have of mesh-build-<language>), its capacity for each (mesh-config's resolved value of the
|
||||
// module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each
|
||||
// build module's steps state (ADR 0525 §6: an ask ends on <node>.<ask id>) the asks running now by kind, the ones left
|
||||
// running, and the steps started and finished in the last hour and day with their total and median time and how long the
|
||||
// node was at its capacity. A try's asks are told from a candidate's by the checker's tries state (each try names the
|
||||
// asks it put). The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver),
|
||||
// asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the
|
||||
// bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// BuildUsageTool is the view's tool of the build nodes' use.
|
||||
const BuildUsageTool = "build_usage"
|
||||
|
||||
// UsageStepBound is how long an ask step may say running before it is taken as left running: the checker's step bound
|
||||
// (mesh-delivery-checker check.DefaultStepBound), after which no asker waits for it.
|
||||
const UsageStepBound = 30 * time.Minute
|
||||
|
||||
// The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker
|
||||
// put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity).
|
||||
const (
|
||||
KindTry = "try"
|
||||
KindHand = "hand"
|
||||
)
|
||||
|
||||
// UsageReader reads what build_usage answers from.
|
||||
type UsageReader struct {
|
||||
// Steps and Pauses are each build module's states by language (go, node, image).
|
||||
Steps map[string]StateRead
|
||||
Pauses map[string]StateRead
|
||||
// Resolved is mesh-config's resolved values (<node>.<module>.<key>); Haves the node-engine's have.
|
||||
Resolved StateRead
|
||||
Haves StateRead
|
||||
// Tries is the checker's tries (try-<unix ns>), each naming the asks it put.
|
||||
Tries StateRead
|
||||
// Backlog is the seats' streams' waiting and held asks, as the view program reads them.
|
||||
Backlog func() ([]BacklogRow, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// UsageAnswer is build_usage's answer.
|
||||
type UsageAnswer struct {
|
||||
At time.Time `json:"at"`
|
||||
Nodes []NodeUsage `json:"nodes"`
|
||||
Backlog []BacklogRow `json:"backlog"`
|
||||
// TriesRunning are the tries the checker runs now; their asks are counted under check until the try ends and names them.
|
||||
TriesRunning []TryRunning `json:"tries_running"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// TryRunning is one try the checker runs now.
|
||||
type TryRunning struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
}
|
||||
|
||||
// NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU.
|
||||
type NodeUsage struct {
|
||||
Node string `json:"node"`
|
||||
Eligible bool `json:"eligible"`
|
||||
// Said is one line per seat, the answer in words.
|
||||
Said []string `json:"said"`
|
||||
Seats []SeatUsage `json:"seats"`
|
||||
CPU string `json:"cpu"`
|
||||
}
|
||||
|
||||
// SeatUsage is one build seat on one node.
|
||||
type SeatUsage struct {
|
||||
Seat string `json:"seat"`
|
||||
Module string `json:"module"`
|
||||
// Holds says the node runs the seat's module (its have says running); Phase is what its have says, or absent.
|
||||
Holds bool `json:"holds"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
// Capacity is the asks it runs at once; CapacityFrom where that came from.
|
||||
Capacity int `json:"capacity"`
|
||||
CapacityFrom string `json:"capacity_from"`
|
||||
Paused bool `json:"paused"`
|
||||
// Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not.
|
||||
Eligible bool `json:"eligible"`
|
||||
WhyNot string `json:"why_not,omitempty"`
|
||||
Running RunningUsage `json:"running"`
|
||||
LeftRunning []StepUsage `json:"left_running"`
|
||||
LastHour WindowUsage `json:"last_hour"`
|
||||
LastDay WindowUsage `json:"last_day"`
|
||||
Steps []StepUsage `json:"running_steps"`
|
||||
}
|
||||
|
||||
// usageStep is a step read, with the kind it is counted under.
|
||||
type usageStep struct {
|
||||
step delivery.Step
|
||||
kind string
|
||||
}
|
||||
|
||||
// RunningUsage is the asks running now: how many in all and by kind, against the capacity.
|
||||
type RunningUsage struct {
|
||||
Total int `json:"total"`
|
||||
ByKind map[string]int `json:"by_kind"`
|
||||
}
|
||||
|
||||
// StepUsage is one step running now, or one left running and why.
|
||||
type StepUsage struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Tool string `json:"tool"`
|
||||
Started time.Time `json:"started"`
|
||||
Secs int64 `json:"secs"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// WindowUsage is the steps of one window: started and finished in it, the finished ones' total and median time, by kind,
|
||||
// and how long the node ran its capacity of asks (a hand step takes no slot).
|
||||
type WindowUsage struct {
|
||||
Started int `json:"started"`
|
||||
Finished int `json:"finished"`
|
||||
Failed int `json:"failed"`
|
||||
TotalSecs int64 `json:"total_secs"`
|
||||
MedianSecs int64 `json:"median_secs"`
|
||||
ByKind map[string]KindWindow `json:"by_kind"`
|
||||
AtCapacitySecs int64 `json:"at_capacity_secs"`
|
||||
AtCapacityShare float64 `json:"at_capacity_share"`
|
||||
}
|
||||
|
||||
// KindWindow is one kind's finished steps in a window.
|
||||
type KindWindow struct {
|
||||
Finished int `json:"finished"`
|
||||
TotalSecs int64 `json:"total_secs"`
|
||||
MedianSecs int64 `json:"median_secs"`
|
||||
}
|
||||
|
||||
// stepIDTime is when a step id was made: its base-36 milliseconds before the six hex digits (buildstep.NewID).
|
||||
func stepIDTime(id string) (time.Time, bool) {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), true
|
||||
}
|
||||
|
||||
// stepKind is the kind a step is counted under.
|
||||
func stepKind(s delivery.Step, tries map[string]bool) string {
|
||||
kind, ask := strings.CutSuffix(s.Tool, " ask")
|
||||
if !ask {
|
||||
return KindHand
|
||||
}
|
||||
if kind == delivery.AskCheck && tries[s.ID] {
|
||||
return KindTry
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
// Usage reads every state and answers. A state it cannot read is said in a note, never left out silently.
|
||||
func (u UsageReader) Usage() UsageAnswer {
|
||||
now := time.Now().UTC()
|
||||
if u.Now != nil {
|
||||
now = u.Now().UTC()
|
||||
}
|
||||
ans := UsageAnswer{At: now, Nodes: []NodeUsage{}, Backlog: []BacklogRow{}, TriesRunning: []TryRunning{}}
|
||||
note := func(f string, a ...any) { ans.Notes = append(ans.Notes, fmt.Sprintf(f, a...)) }
|
||||
|
||||
tries := u.tries(now, &ans, note)
|
||||
seats := map[string]map[string]*SeatUsage{} // node -> language -> seat
|
||||
seat := func(node, lang string) *SeatUsage {
|
||||
if seats[node] == nil {
|
||||
seats[node] = map[string]*SeatUsage{}
|
||||
}
|
||||
if s := seats[node][lang]; s != nil {
|
||||
return s
|
||||
}
|
||||
name, _ := delivery.BuildSeat(lang)
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, Steps: []StepUsage{}}
|
||||
seats[node][lang] = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Which node runs which build module: the have, <node>.mesh-build-<language>.
|
||||
haves := map[string]delivery.Have{}
|
||||
if u.Haves.Keys == nil || u.Haves.Get == nil {
|
||||
note("the node-engine's have is not readable here, so no seat is said held")
|
||||
} else if keys, err := u.Haves.Keys(); err != nil {
|
||||
note("the node-engine's have cannot be listed, so no seat is said held: %v", err)
|
||||
} else {
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
if m, _ := delivery.BuildModuleOf(lang); strings.HasSuffix(k, "."+m) {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
raws, err := u.Haves.values(want)
|
||||
if err != nil {
|
||||
note("the node-engine's have cannot be read: %v", err)
|
||||
}
|
||||
for i, k := range want {
|
||||
if i < len(raws) && len(raws[i]) > 0 {
|
||||
h, err := delivery.DecodeHave(raws[i])
|
||||
if err != nil {
|
||||
note("the have %s cannot be read, so its seat is not said held: %v", k, err)
|
||||
continue
|
||||
}
|
||||
haves[k] = h
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, h := range haves {
|
||||
node, module, _ := strings.Cut(k, ".")
|
||||
lang := strings.TrimPrefix(module, "mesh-build-")
|
||||
s := seat(node, lang)
|
||||
s.Phase, s.Holds = string(h.Phase), h.Phase == delivery.PhaseRunning
|
||||
}
|
||||
|
||||
// Capacity: mesh-config's resolved value of the module's `capacity` on each node.
|
||||
resolved := map[string]string{} // <node>.<module> -> setting
|
||||
if u.Resolved.Keys == nil || u.Resolved.Get == nil {
|
||||
note("mesh-config's resolved values are not readable here, so each capacity is the modules' default %q", buildstep.DefaultCapacities)
|
||||
} else if keys, err := u.Resolved.Keys(); err != nil {
|
||||
note("mesh-config's resolved values cannot be listed, so each capacity is the modules' default %q: %v", buildstep.DefaultCapacities, err)
|
||||
} else {
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
if strings.HasSuffix(k, "."+buildstep.CapacitySetting) && strings.Contains(k, ".mesh-build-") {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
raws, err := u.Resolved.values(want)
|
||||
if err != nil {
|
||||
note("mesh-config's resolved capacities cannot be read: %v", err)
|
||||
}
|
||||
for i, k := range want {
|
||||
var r struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
setting := ""
|
||||
if i < len(raws) && json.Unmarshal(raws[i], &r) == nil && json.Unmarshal(r.Value, &setting) == nil {
|
||||
resolved[strings.TrimSuffix(k, "."+buildstep.CapacitySetting)] = setting
|
||||
}
|
||||
}
|
||||
}
|
||||
for key := range resolved {
|
||||
node, module, _ := strings.Cut(key, ".")
|
||||
seat(node, strings.TrimPrefix(module, "mesh-build-"))
|
||||
}
|
||||
// The steps of each build module.
|
||||
steps := map[string]map[string][]usageStep{} // node -> language -> steps
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
sr, ok := u.Steps[lang]
|
||||
if !ok || sr.Keys == nil || sr.Get == nil {
|
||||
note("the steps of %s are not readable here", module)
|
||||
continue
|
||||
}
|
||||
keys, err := sr.Keys()
|
||||
if err != nil {
|
||||
note("the steps of %s cannot be listed: %v", module, err)
|
||||
continue
|
||||
}
|
||||
// A step id is its start in base-36 milliseconds, so the newest keys are the ones of the last day; at most
|
||||
// MaxStateReads of them are read, newest first.
|
||||
type keyAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var recent []keyAt
|
||||
for _, k := range keys {
|
||||
_, id, err := delivery.ParseStepKey(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+UsageStepBound {
|
||||
recent = append(recent, keyAt{k, at})
|
||||
}
|
||||
}
|
||||
sort.Slice(recent, func(i, j int) bool { return recent[i].at.After(recent[j].at) })
|
||||
if len(recent) > MaxStateReads {
|
||||
note("the steps of %s: read the newest %d of %d keys of the last day; the older ones are not counted", module, MaxStateReads, len(recent))
|
||||
recent = recent[:MaxStateReads]
|
||||
}
|
||||
want := make([]string, len(recent))
|
||||
for i, r := range recent {
|
||||
want[i] = r.key
|
||||
}
|
||||
raws, err := sr.values(want)
|
||||
if err != nil {
|
||||
note("the steps of %s cannot all be read: %v", module, err)
|
||||
}
|
||||
for i := range want {
|
||||
if i >= len(raws) || len(raws[i]) == 0 {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(raws[i])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if steps[s.Node] == nil {
|
||||
steps[s.Node] = map[string][]usageStep{}
|
||||
}
|
||||
steps[s.Node][lang] = append(steps[s.Node][lang], usageStep{s, stepKind(s, tries)})
|
||||
seat(s.Node, lang)
|
||||
}
|
||||
}
|
||||
|
||||
// Pauses: paused.<node> of each module.
|
||||
paused := map[string]bool{} // <node>.<language>
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
pr, ok := u.Pauses[lang]
|
||||
if !ok || pr.Get == nil {
|
||||
note("the pauses of %s are not readable here, so none is said paused", module)
|
||||
continue
|
||||
}
|
||||
for node := range seats {
|
||||
if seats[node][lang] == nil {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PauseKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, err := pr.Get(key)
|
||||
if err != nil {
|
||||
note("the pause of %s on %s cannot be read: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
if p, err := delivery.DecodePause(raw); err == nil {
|
||||
paused[node+"."+lang] = p.Paused
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nodes := make([]string, 0, len(seats))
|
||||
for n := range seats {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, node := range nodes {
|
||||
nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{},
|
||||
CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
s := seats[node][lang]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
if setting, ok := resolved[node+"."+s.Module]; ok {
|
||||
s.CapacityFrom = "mesh-config, resolved for " + s.Module + " on " + node
|
||||
if caps, err := buildrun.ParseCapacity(setting); err != nil {
|
||||
s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err)
|
||||
} else {
|
||||
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
||||
}
|
||||
} else {
|
||||
caps, _ := buildrun.ParseCapacity(buildstep.DefaultCapacities)
|
||||
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
||||
s.CapacityFrom = fmt.Sprintf("the modules' default %q: mesh-config holds no resolved value for %s on %s", buildstep.DefaultCapacities, s.Module, node)
|
||||
}
|
||||
s.Paused = paused[node+"."+lang]
|
||||
switch {
|
||||
case !s.Holds && s.Phase == "":
|
||||
s.WhyNot = s.Module + " is not on " + node
|
||||
case !s.Holds:
|
||||
s.WhyNot = s.Module + " on " + node + " is " + s.Phase
|
||||
case s.Capacity == 0:
|
||||
s.WhyNot = "its capacity on " + node + " is 0"
|
||||
case s.Paused:
|
||||
s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)"
|
||||
default:
|
||||
s.Eligible = true
|
||||
}
|
||||
nu.Eligible = nu.Eligible || s.Eligible
|
||||
s.fill(steps[node][lang], now)
|
||||
nu.Said = append(nu.Said, s.said())
|
||||
nu.Seats = append(nu.Seats, *s)
|
||||
}
|
||||
ans.Nodes = append(ans.Nodes, nu)
|
||||
}
|
||||
|
||||
if u.Backlog == nil {
|
||||
note("the backlog of the seats' streams is not readable here")
|
||||
} else if rows, err := u.Backlog(); err != nil {
|
||||
note("the backlog of the seats' streams is read by the view program, which did not answer: %v", err)
|
||||
} else if rows != nil {
|
||||
ans.Backlog = rows
|
||||
}
|
||||
return ans
|
||||
}
|
||||
|
||||
// tries reads the checker's tries of the last day: the running ones, and the asks every one put.
|
||||
func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, ...any)) map[string]bool {
|
||||
asks := map[string]bool{}
|
||||
if u.Tries.Keys == nil || u.Tries.Get == nil {
|
||||
note("the checker's tries are not readable here, so a try's asks are counted under check")
|
||||
return asks
|
||||
}
|
||||
keys, err := u.Tries.Keys()
|
||||
if err != nil {
|
||||
note("the checker's tries cannot be listed, so a try's asks are counted under check: %v", err)
|
||||
return asks
|
||||
}
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
ns, err := strconv.ParseInt(strings.TrimPrefix(k, "try-"), 10, 64)
|
||||
if err == nil && strings.HasPrefix(k, "try-") && now.Sub(time.Unix(0, ns)) <= 48*time.Hour {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
raws, err := u.Tries.values(want)
|
||||
if err != nil {
|
||||
note("the checker's tries cannot all be read: %v", err)
|
||||
}
|
||||
for i := range want {
|
||||
if i >= len(raws) || len(raws[i]) == 0 {
|
||||
continue
|
||||
}
|
||||
var t struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
State string `json:"state"`
|
||||
Started time.Time `json:"started"`
|
||||
Asks []string `json:"asks"`
|
||||
}
|
||||
if json.Unmarshal(raws[i], &t) != nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range t.Asks {
|
||||
asks[a] = true
|
||||
}
|
||||
if t.State == "running" {
|
||||
ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started})
|
||||
}
|
||||
}
|
||||
if len(ans.TriesRunning) > 0 {
|
||||
note("a running try's asks are counted under check until the try ends and names them")
|
||||
}
|
||||
return asks
|
||||
}
|
||||
|
||||
// fill counts a seat's steps: running now, left running (started past UsageStepBound, or its module no longer running on
|
||||
// the node: no step behind the key runs any more), and the two windows.
|
||||
func (s *SeatUsage) fill(steps []usageStep, now time.Time) {
|
||||
sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) })
|
||||
var live []usageStep // ended, or running and not left: what the windows count
|
||||
for _, us := range steps {
|
||||
st := us.step
|
||||
if st.State != delivery.StepRunning {
|
||||
live = append(live, us)
|
||||
continue
|
||||
}
|
||||
line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())}
|
||||
switch {
|
||||
case !s.Holds:
|
||||
line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs"
|
||||
case now.Sub(st.Started) > UsageStepBound:
|
||||
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), UsageStepBound)
|
||||
}
|
||||
if line.Why != "" {
|
||||
s.LeftRunning = append(s.LeftRunning, line)
|
||||
continue
|
||||
}
|
||||
s.Steps = append(s.Steps, line)
|
||||
live = append(live, us)
|
||||
if us.kind != KindHand {
|
||||
s.Running.Total++
|
||||
s.Running.ByKind[us.kind]++
|
||||
}
|
||||
}
|
||||
s.LastHour = window(live, now, time.Hour, s.Capacity)
|
||||
s.LastDay = window(live, now, 24*time.Hour, s.Capacity)
|
||||
}
|
||||
|
||||
// said is the seat's use in one line.
|
||||
func (s *SeatUsage) said() string {
|
||||
head := s.Seat + ": eligible"
|
||||
if !s.Eligible {
|
||||
head = s.Seat + ": not eligible, " + s.WhyNot
|
||||
}
|
||||
kinds := make([]string, 0, len(s.Running.ByKind))
|
||||
for k, n := range s.Running.ByKind {
|
||||
kinds = append(kinds, fmt.Sprintf("%s %d", k, n))
|
||||
}
|
||||
sort.Strings(kinds)
|
||||
line := fmt.Sprintf("%s; %d of capacity %d running", head, s.Running.Total, s.Capacity)
|
||||
if len(kinds) > 0 {
|
||||
line += " (" + strings.Join(kinds, ", ") + ")"
|
||||
}
|
||||
if len(s.LeftRunning) > 0 {
|
||||
line += fmt.Sprintf(", %d left running", len(s.LeftRunning))
|
||||
}
|
||||
return line + fmt.Sprintf("; last hour %d finished, median %d s, at capacity %.0f%% of it; last day %d finished, median %d s, at capacity %.1f%% of it",
|
||||
s.LastHour.Finished, s.LastHour.MedianSecs, s.LastHour.AtCapacityShare*100, s.LastDay.Finished, s.LastDay.MedianSecs, s.LastDay.AtCapacityShare*100)
|
||||
}
|
||||
|
||||
// window counts the steps of the last span before now: started in it, finished in it with their total and median time,
|
||||
// and the time in it during which capacity asks (not hand steps) ran at once.
|
||||
func window(steps []usageStep, now time.Time, span time.Duration, capacity int) WindowUsage {
|
||||
from := now.Add(-span)
|
||||
w := WindowUsage{ByKind: map[string]KindWindow{}}
|
||||
var all []int64
|
||||
byKind := map[string][]int64{}
|
||||
type edge struct {
|
||||
at time.Time
|
||||
d int
|
||||
}
|
||||
var edges []edge
|
||||
for _, us := range steps {
|
||||
st := us.step
|
||||
if !st.Started.Before(from) {
|
||||
w.Started++
|
||||
}
|
||||
if !st.Ended.IsZero() && !st.Ended.Before(from) {
|
||||
w.Finished++
|
||||
if st.State == delivery.StepFailed || st.State == delivery.StepRefused {
|
||||
w.Failed++
|
||||
}
|
||||
secs := int64(st.Ended.Sub(st.Started).Seconds())
|
||||
all = append(all, secs)
|
||||
byKind[us.kind] = append(byKind[us.kind], secs)
|
||||
}
|
||||
if us.kind == KindHand {
|
||||
continue
|
||||
}
|
||||
start, end := st.Started, st.Ended
|
||||
if end.IsZero() {
|
||||
end = now
|
||||
}
|
||||
if start.Before(from) {
|
||||
start = from
|
||||
}
|
||||
if end.After(start) {
|
||||
edges = append(edges, edge{start, 1}, edge{end, -1})
|
||||
}
|
||||
}
|
||||
w.TotalSecs, w.MedianSecs = sumMedian(all)
|
||||
for k, d := range byKind {
|
||||
kw := KindWindow{Finished: len(d)}
|
||||
kw.TotalSecs, kw.MedianSecs = sumMedian(d)
|
||||
w.ByKind[k] = kw
|
||||
}
|
||||
if capacity > 0 {
|
||||
// An end before a start at the same instant: a slot handed from one ask to the next is not a moment at capacity+1.
|
||||
sort.Slice(edges, func(i, j int) bool {
|
||||
if edges[i].at.Equal(edges[j].at) {
|
||||
return edges[i].d < edges[j].d
|
||||
}
|
||||
return edges[i].at.Before(edges[j].at)
|
||||
})
|
||||
var at time.Duration
|
||||
n := 0
|
||||
for i, e := range edges {
|
||||
n += e.d
|
||||
if n >= capacity && i+1 < len(edges) {
|
||||
at += edges[i+1].at.Sub(e.at)
|
||||
}
|
||||
}
|
||||
w.AtCapacitySecs = int64(at.Seconds())
|
||||
w.AtCapacityShare = float64(int64(at*1000/span)) / 1000
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// sumMedian is the sum and the median of d (the lower middle of an even count); zero for none.
|
||||
func sumMedian(d []int64) (sum, median int64) {
|
||||
if len(d) == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
s := append([]int64(nil), d...)
|
||||
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
||||
for _, v := range s {
|
||||
sum += v
|
||||
}
|
||||
return sum, s[(len(s)-1)/2]
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// usageNow is the fixture's clock.
|
||||
var usageNow = time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// at is the fixture's time hh:mm on its day.
|
||||
func at(hm string) time.Time {
|
||||
t, _ := time.Parse("2006-01-02 15:04", "2026-10-26 "+hm)
|
||||
return t.UTC()
|
||||
}
|
||||
|
||||
// stepID is a step id made at t, as buildstep.NewID makes it, with suffix as its six hex digits.
|
||||
func stepID(t time.Time, suffix string) string { return strconv.FormatInt(t.UnixMilli(), 36) + suffix }
|
||||
|
||||
func states(m map[string]string) StateRead {
|
||||
return StateRead{
|
||||
Keys: func() ([]string, error) {
|
||||
var k []string
|
||||
for key := range m {
|
||||
k = append(k, key)
|
||||
}
|
||||
return k, nil
|
||||
},
|
||||
Get: func(key string) (json.RawMessage, error) {
|
||||
if v, ok := m[key]; ok {
|
||||
return json.RawMessage(v), nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func step(node, id, tool, state string, started, ended time.Time) string {
|
||||
e := ""
|
||||
if !ended.IsZero() {
|
||||
e = `,"ended":"` + ended.Format(time.RFC3339) + `"`
|
||||
}
|
||||
return fmt.Sprintf(`{"v":1,"id":%q,"tool":%q,"node":%q,"state":%q,"started":%q%s}`, id, tool, node, state, started.Format(time.RFC3339), e)
|
||||
}
|
||||
|
||||
// usageFixture is four nodes: shanks builds Go at capacity 2 (two asks running, one left running, a try's ask, a check,
|
||||
// a recheck of the morning and a hand step ended); ace holds the Go seat at capacity 2 and is paused; g14 holds it at
|
||||
// capacity 0; the Node seat on shanks has no resolved capacity and takes the modules' default.
|
||||
func usageFixture() UsageReader {
|
||||
running := `{"v":1,"phase":"running","health":"ok"}`
|
||||
tryAsk := stepID(at("11:30"), "a00003")
|
||||
goSteps := map[string]string{}
|
||||
add := func(node, id, tool, state string, started, ended time.Time) {
|
||||
goSteps[node+"."+id] = step(node, id, tool, state, started, ended)
|
||||
}
|
||||
add("shanks", stepID(at("11:50"), "a00001"), "delivery ask", "running", at("11:50"), time.Time{})
|
||||
add("shanks", stepID(at("11:55"), "a00002"), "check ask", "running", at("11:55"), time.Time{})
|
||||
add("shanks", tryAsk, "check ask", "passed", at("11:30"), at("11:40"))
|
||||
add("shanks", stepID(at("11:20"), "a00004"), "check ask", "failed", at("11:20"), at("11:26"))
|
||||
add("shanks", stepID(at("06:00"), "a00005"), "recheck ask", "passed", at("06:00"), at("06:10"))
|
||||
add("shanks", stepID(at("10:00"), "a00006"), "check ask", "running", at("10:00"), time.Time{})
|
||||
add("shanks", stepID(at("11:45"), "a00007"), "build_go_test", "passed", at("11:45"), at("11:46"))
|
||||
goSteps["shanks.notastepkey"] = `{}` // a key that is no step id is passed over
|
||||
return UsageReader{
|
||||
Now: func() time.Time { return usageNow },
|
||||
Haves: states(map[string]string{"shanks.mesh-build-go": running, "shanks.mesh-build-node": running, "ace.mesh-build-go": running,
|
||||
"g14.mesh-build-go": running, "shanks.node": `{}`, "shanks.mesh-delivery-view": running}),
|
||||
Resolved: states(map[string]string{
|
||||
"shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
|
||||
"ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
|
||||
"g14.mesh-build-go.capacity": `{"node":"g14","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
|
||||
"shanks.mesh-config.other": `{"value":"x"}`,
|
||||
}),
|
||||
Steps: map[string]StateRead{"go": states(goSteps), "node": states(map[string]string{}), "image": states(map[string]string{})},
|
||||
Pauses: map[string]StateRead{"go": states(map[string]string{"paused.ace": `{"v":1,"paused":true,"by":"lead@shanks","at":"2026-10-26T09:00:00Z"}`}), "node": states(nil), "image": states(nil)},
|
||||
Tries: states(map[string]string{
|
||||
"try-" + strconv.FormatInt(at("11:29").UnixNano(), 10): `{"id":"try-1","repository":"novox/mesh-sdk","head":"h","trunk":"t","state":"pass","asks":["` + tryAsk + `"]}`,
|
||||
"try-" + strconv.FormatInt(at("11:58").UnixNano(), 10): `{"id":"try-2","repository":"novox/mesh-delivery","head":"h","trunk":"t","state":"running","started":"2026-10-26T11:58:00Z"}`,
|
||||
}),
|
||||
Backlog: func() ([]BacklogRow, error) {
|
||||
return []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func seatOf(t *testing.T, a UsageAnswer, node, seat string) SeatUsage {
|
||||
t.Helper()
|
||||
for _, n := range a.Nodes {
|
||||
for _, s := range n.Seats {
|
||||
if n.Node == node && s.Seat == seat {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("no seat %s on %s in %+v", seat, node, a.Nodes)
|
||||
return SeatUsage{}
|
||||
}
|
||||
|
||||
// build_usage answers, from the states alone, what the operator asked in six calls: per node its seats, capacity,
|
||||
// eligibility, the asks running by kind and the ones left running, the last hour and day, and mesh-wide the backlog.
|
||||
func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) {
|
||||
a := usageFixture().Usage()
|
||||
if testing.Verbose() {
|
||||
out, _ := json.MarshalIndent(a, "", " ")
|
||||
t.Logf("build_usage answers:\n%s", out)
|
||||
}
|
||||
sg := seatOf(t, a, "shanks", "node-build-go")
|
||||
eligible := map[string]bool{}
|
||||
for _, n := range a.Nodes {
|
||||
eligible[n.Node] = n.Eligible
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
got, want any
|
||||
}{
|
||||
{"nodes", len(a.Nodes), 3},
|
||||
{"eligible", eligible, map[string]bool{"shanks": true, "ace": false, "g14": false}},
|
||||
{"ace why not", seatOf(t, a, "ace", "node-build-go").WhyNot, "paused (node-build-go.resume takes asks again)"},
|
||||
{"g14 why not", seatOf(t, a, "g14", "node-build-go").WhyNot, "its capacity on g14 is 0"},
|
||||
{"shanks capacity", sg.Capacity, 2},
|
||||
{"shanks node seat takes the default", seatOf(t, a, "shanks", "node-build-node").Capacity, 3},
|
||||
{"running", sg.Running.Total, 2},
|
||||
{"running by kind", sg.Running.ByKind, map[string]int{"delivery": 1, "check": 1}},
|
||||
{"running steps, the hand step's ended", len(sg.Steps), 2},
|
||||
{"left running", len(sg.LeftRunning), 1},
|
||||
{"hour started", sg.LastHour.Started, 5}, // delivery, check, try, check, hand
|
||||
{"hour finished", sg.LastHour.Finished, 3}, // try 600 s, check 360 s, hand 60 s
|
||||
{"hour failed", sg.LastHour.Failed, 1},
|
||||
{"hour total", sg.LastHour.TotalSecs, int64(1020)},
|
||||
{"hour median", sg.LastHour.MedianSecs, int64(360)},
|
||||
{"hour try", sg.LastHour.ByKind["try"], KindWindow{Finished: 1, TotalSecs: 600, MedianSecs: 600}},
|
||||
{"hour hand", sg.LastHour.ByKind["hand"], KindWindow{Finished: 1, TotalSecs: 60, MedianSecs: 60}},
|
||||
{"hour at capacity", sg.LastHour.AtCapacitySecs, int64(300)}, // 11:55-12:00, two asks of two
|
||||
{"hour at capacity share", sg.LastHour.AtCapacityShare, 0.083},
|
||||
{"day finished", sg.LastDay.Finished, 4},
|
||||
{"day recheck", sg.LastDay.ByKind["recheck"].Finished, 1},
|
||||
{"day at capacity", sg.LastDay.AtCapacitySecs, int64(300)},
|
||||
{"backlog", a.Backlog, []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}},
|
||||
{"tries running", len(a.TriesRunning), 1},
|
||||
{"said", a.Nodes[0].Said, []string{"node-build-go: not eligible, paused (node-build-go.resume takes asks again); 0 of capacity 2 running; last hour 0 finished, median 0 s, at capacity 0% of it; last day 0 finished, median 0 s, at capacity 0.0% of it"}},
|
||||
{"said shanks", a.Nodes[2].Said[0], "node-build-go: eligible; 2 of capacity 2 running (check 1, delivery 1), 1 left running; last hour 3 finished, median 360 s, at capacity 8% of it; last day 4 finished, median 360 s, at capacity 0.3% of it"},
|
||||
{"cpu says where it is", strings.Contains(a.Nodes[0].CPU, "/disk-load.cpu_top"), true},
|
||||
} {
|
||||
if fmt.Sprint(c.got) != fmt.Sprint(c.want) {
|
||||
t.Errorf("%s = %v, want %v", c.what, c.got, c.want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(sg.LeftRunning[0].Why, "past the step bound") {
|
||||
t.Errorf("left running says %q", sg.LeftRunning[0].Why)
|
||||
}
|
||||
|
||||
// Nothing readable: each state is named in a note, the backlog's refusal too, and nothing is said held.
|
||||
none := UsageReader{Now: func() time.Time { return usageNow }, Backlog: func() ([]BacklogRow, error) { return nil, errors.New("no socket") }}.Usage()
|
||||
notes := strings.Join(none.Notes, " | ")
|
||||
for _, want := range []string{"have is not readable", "resolved values are not readable", "steps of mesh-build-go are not readable", "tries are not readable", "did not answer: no socket"} {
|
||||
if !strings.Contains(notes, want) {
|
||||
t.Errorf("the notes %q do not say %q", notes, want)
|
||||
}
|
||||
}
|
||||
if len(none.Nodes) != 0 || none.Backlog == nil {
|
||||
t.Errorf("nothing readable answered nodes %v, backlog %v", none.Nodes, none.Backlog)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user