A held module still runs its asks, and a have that says running proves nothing about the holder: the usage view asks each build seat's verb current on each node at once, says the seat held where its holder answers, counts the asks it lists as running, and says a step key it does not list left running. The node-engine's phase is shown and judges nothing. The manifest invokes the three build seats' current. Builds: design 47 H2 §813 (9), §801
744 lines
27 KiB
Go
744 lines
27 KiB
Go
package view
|
|
|
|
// The build nodes' use, in one answer (the operator, 2026-10-26: "how much are our build agents being used, and is ace
|
|
// eligible for building?" took six calls over four tools). Read from the bus's state alone: which build seats each node
|
|
// holds (the node-engine's have of mesh-build-<language>), its capacity for each (mesh-config's resolved value of the
|
|
// module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each
|
|
// build module's steps state (ADR 0525 §6: an ask ends on <node>.<ask id>) the asks running now by kind, the ones left
|
|
// running, and the steps started and finished in the last hour and day with their total and median time and how long the
|
|
// node was at its capacity. A build module's work is read from its seat's verb current on each node (design 47 H2 §813 (9),
|
|
// §801): the seat is held on a node whose holder answers it, whatever the node-engine's have says of its phase, and the asks
|
|
// running are the ones it lists. The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver),
|
|
// asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the
|
|
// bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
buildseats "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
)
|
|
|
|
// BuildUsageTool is the view's tool of the build nodes' use.
|
|
const BuildUsageTool = "build_usage"
|
|
|
|
// endedStale says why a step's end was written by its build module for a key left running (mesh-sdk buildstep's ending
|
|
// of stale steps): a restart's, or the step bound's; "" for a step that ran to its own end. Such an end is the moment the
|
|
// module noticed, not the step's, so it is counted apart and in neither the medians nor the time at capacity.
|
|
func endedStale(st buildstep.Step) string {
|
|
if st.State != buildstep.StepFailed {
|
|
return ""
|
|
}
|
|
if st.Reason == buildrun.RestartReason || strings.HasPrefix(st.Reason, "past the step bound") {
|
|
return st.Reason
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// The kinds a step is counted under: an ask's kind (delivery, check, recheck), or hand (a build tool called directly, outside
|
|
// the seat's asks, which takes no slot of the capacity). The checker and its tries are gone (design 47 H §764).
|
|
const (
|
|
KindHand = "hand"
|
|
)
|
|
|
|
// UsageReader reads what build_usage answers from.
|
|
type UsageReader struct {
|
|
// Seats are the build seats, each by the acting rule that builds, with the module holding it on the seat list (design 47 G
|
|
// A-82): no build module is named here. StepsOf and PausesOf read a build seat's steps and pauses through the seat.
|
|
Seats *buildseats.BuildSeats
|
|
StepsOf func(seat string) StateRead
|
|
PausesOf func(seat string) StateRead
|
|
// Resolved is mesh-config's resolved values (<node>.<module>.<key>); Haves the node-engine's have.
|
|
Resolved StateRead
|
|
Haves StateRead
|
|
// Current asks a build seat's verb current of one node's holder (design 47 H2 §801), its answer as the part says it.
|
|
Current func(seat, node string) (json.RawMessage, error)
|
|
// Backlog is the seats' streams' waiting and held asks, as the view program reads them.
|
|
Backlog func() ([]BacklogRow, error)
|
|
Now func() time.Time
|
|
}
|
|
|
|
// UsageAnswer is build_usage's answer.
|
|
type UsageAnswer struct {
|
|
At time.Time `json:"at"`
|
|
Nodes []NodeUsage `json:"nodes"`
|
|
Backlog []BacklogRow `json:"backlog"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU.
|
|
type NodeUsage struct {
|
|
Node string `json:"node"`
|
|
// Eligible is the node's verdict: "eligible" when a seat of it is, "not eligible" when every seat is known not to be,
|
|
// and "not known" when none is known eligible and this tool cannot tell for some seat.
|
|
Eligible string `json:"eligible"`
|
|
// Said is one line per seat, the answer in words.
|
|
Said []string `json:"said"`
|
|
Seats []SeatUsage `json:"seats"`
|
|
CPU string `json:"cpu"`
|
|
}
|
|
|
|
// SeatUsage is one build seat on one node.
|
|
type SeatUsage struct {
|
|
Seat string `json:"seat"`
|
|
Module string `json:"module"`
|
|
// Holds says the seat's holder on the node answers its verb current (design 47 H2 §813 (9)); Phase is what the
|
|
// node-engine's have says, or absent, and judges nothing.
|
|
Holds bool `json:"holds"`
|
|
Phase string `json:"phase,omitempty"`
|
|
// Capacity is the asks it runs at once; CapacityFrom where that came from.
|
|
Capacity int `json:"capacity"`
|
|
CapacityFrom string `json:"capacity_from"`
|
|
// Configured is mesh-config's capacity when the module says it judges by a lower one (capacity.<node> of its pauses
|
|
// state: the configured capacity bounded by its node's build CPU, issue 1806's follow-up); Capacity is then the
|
|
// module's and CapacityFrom its why.
|
|
Configured int `json:"configured,omitempty"`
|
|
// BuildHours is the node's build window as the module says it (HH:MM-HH:MM and its zone), absent when it builds at any
|
|
// hour; OutsideHours says the module judged the time outside it, so it takes nothing new (the operator, 2026-10-26).
|
|
BuildHours string `json:"build_hours,omitempty"`
|
|
OutsideHours bool `json:"outside_hours,omitempty"`
|
|
// BuildHoursFault is why the module cannot read its node's build-hours value, naming it: it then builds at any hour.
|
|
BuildHoursFault string `json:"build_hours_fault,omitempty"`
|
|
Paused bool `json:"paused"`
|
|
// Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not.
|
|
// EligibleKnown is false when this tool cannot tell (it cannot read mesh-config's capacities): Eligible is then
|
|
// absent, never false, and WhyNot starts "not known: ".
|
|
Eligible *bool `json:"eligible,omitempty"`
|
|
EligibleKnown bool `json:"eligible_known"`
|
|
WhyNot string `json:"why_not,omitempty"`
|
|
Running RunningUsage `json:"running"`
|
|
LeftRunning []StepUsage `json:"left_running"`
|
|
// EndedStale are the steps of the last day whose end their build module wrote for a key left running (a restart, or
|
|
// past the step bound): counted here alone, never as finished.
|
|
EndedStale []StepUsage `json:"ended_stale"`
|
|
LastHour WindowUsage `json:"last_hour"`
|
|
LastDay WindowUsage `json:"last_day"`
|
|
Steps []StepUsage `json:"running_steps"`
|
|
}
|
|
|
|
// currentAnswer is what a build seat's verb current answers (the build-step part, design 47 H2 §801), as far as the usage reads it.
|
|
type currentAnswer struct {
|
|
Node string `json:"node"`
|
|
Asks []currentAsk `json:"asks"`
|
|
}
|
|
|
|
// currentAsk is one ask a holder runs now, whoever asked it.
|
|
type currentAsk struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"`
|
|
Started time.Time `json:"started"`
|
|
}
|
|
|
|
// usageStep is a step read, with the kind it is counted under.
|
|
type usageStep struct {
|
|
step buildstep.Step
|
|
kind string
|
|
}
|
|
|
|
// RunningUsage is the asks running now: how many in all and by kind, against the capacity.
|
|
type RunningUsage struct {
|
|
Total int `json:"total"`
|
|
ByKind map[string]int `json:"by_kind"`
|
|
}
|
|
|
|
// StepUsage is one step running now, or one left running and why.
|
|
type StepUsage struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"`
|
|
Tool string `json:"tool"`
|
|
Started time.Time `json:"started"`
|
|
Secs int64 `json:"secs"`
|
|
Why string `json:"why,omitempty"`
|
|
}
|
|
|
|
// WindowUsage is the steps of one window: started and finished in it, the finished ones' total and median time, by kind,
|
|
// and how long the node ran its capacity of asks (a hand step takes no slot).
|
|
type WindowUsage struct {
|
|
Started int `json:"started"`
|
|
Finished int `json:"finished"`
|
|
Failed int `json:"failed"`
|
|
TotalSecs int64 `json:"total_secs"`
|
|
MedianSecs int64 `json:"median_secs"`
|
|
ByKind map[string]KindWindow `json:"by_kind"`
|
|
AtCapacitySecs int64 `json:"at_capacity_secs"`
|
|
AtCapacityShare float64 `json:"at_capacity_share"`
|
|
}
|
|
|
|
// KindWindow is one kind's finished steps in a window.
|
|
type KindWindow struct {
|
|
Finished int `json:"finished"`
|
|
TotalSecs int64 `json:"total_secs"`
|
|
MedianSecs int64 `json:"median_secs"`
|
|
}
|
|
|
|
// stepIDTime is when a step id was made: its base-36 milliseconds before the six hex digits (buildstep.NewID).
|
|
func stepIDTime(id string) (time.Time, bool) {
|
|
if len(id) <= 6 {
|
|
return time.Time{}, false
|
|
}
|
|
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
|
if err != nil {
|
|
return time.Time{}, false
|
|
}
|
|
return time.UnixMilli(ms).UTC(), true
|
|
}
|
|
|
|
// stepKind is the kind a step is counted under.
|
|
func stepKind(s buildstep.Step) string {
|
|
kind, ask := strings.CutSuffix(s.Tool, " ask")
|
|
if !ask {
|
|
return KindHand
|
|
}
|
|
return kind
|
|
}
|
|
|
|
// Usage reads every state and answers. A state it cannot read is said in a note, never left out silently.
|
|
func (u UsageReader) Usage() UsageAnswer {
|
|
now := time.Now().UTC()
|
|
if u.Now != nil {
|
|
now = u.Now().UTC()
|
|
}
|
|
ans := UsageAnswer{At: now, Nodes: []NodeUsage{}, Backlog: []BacklogRow{}}
|
|
note := func(f string, a ...any) { ans.Notes = append(ans.Notes, fmt.Sprintf(f, a...)) }
|
|
|
|
// The build seats once for this answer: each asks the seat list for its holder (design 47 G A-82).
|
|
buildSeats := u.Seats.All()
|
|
buildSeatOfModule := func(module string) buildseats.BuildModule {
|
|
for _, m := range buildSeats {
|
|
if m.Module == module && module != "" {
|
|
return m
|
|
}
|
|
}
|
|
return buildseats.BuildModule{}
|
|
}
|
|
seats := map[string]map[string]*SeatUsage{} // node -> language -> seat
|
|
seat := func(node, lang string) *SeatUsage {
|
|
if seats[node] == nil {
|
|
seats[node] = map[string]*SeatUsage{}
|
|
}
|
|
if s := seats[node][lang]; s != nil {
|
|
return s
|
|
}
|
|
var bm buildseats.BuildModule
|
|
for _, m := range buildSeats {
|
|
if m.AskLanguage() == lang {
|
|
bm = m
|
|
}
|
|
}
|
|
s := &SeatUsage{Seat: bm.Seat, Module: bm.Module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, EndedStale: []StepUsage{}, Steps: []StepUsage{}}
|
|
seats[node][lang] = s
|
|
return s
|
|
}
|
|
|
|
// Which node runs which build module: the have, <node>.mesh-build-<language>.
|
|
haves := map[string]nodeengine.HaveByNodeModuleValue{}
|
|
if u.Haves.Keys == nil || u.Haves.Get == nil {
|
|
note("the node-engine's have is not readable here, so no seat is said held")
|
|
} else if keys, err := u.Haves.Keys(); err != nil {
|
|
note("the node-engine's have cannot be listed, so no seat is said held: %v", err)
|
|
} else {
|
|
var want []string
|
|
for _, k := range keys {
|
|
if _, module, _ := strings.Cut(k, "."); buildSeatOfModule(module).Seat != "" {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
raws, err := u.Haves.values(want)
|
|
if err != nil {
|
|
note("the node-engine's have cannot be read: %v", err)
|
|
}
|
|
for i, k := range want {
|
|
if i < len(raws) && len(raws[i]) > 0 {
|
|
h, err := nodeengineclient.DecodeHave(raws[i])
|
|
if err != nil {
|
|
note("the have %s cannot be read, so its seat is not said held: %v", k, err)
|
|
continue
|
|
}
|
|
haves[k] = h
|
|
}
|
|
}
|
|
}
|
|
for k, h := range haves {
|
|
node, module, _ := strings.Cut(k, ".")
|
|
bm := buildSeatOfModule(module)
|
|
s := seat(node, bm.AskLanguage())
|
|
s.Phase = string(h.Phase)
|
|
}
|
|
|
|
// Capacity: mesh-config's resolved value of the module's `capacity` on each node.
|
|
resolved := map[string]int{} // <node>.<module> -> its capacity
|
|
unknown := "" // why this tool cannot say a capacity: mesh-config's values cannot be read here
|
|
unreadable := map[string]string{} // <node>.<module> -> why its resolved key holds no readable value: the puller uses 0
|
|
if u.Resolved.Keys == nil || u.Resolved.Get == nil {
|
|
unknown = "mesh-config's resolved values are not readable here"
|
|
} else if keys, err := u.Resolved.Keys(); err != nil {
|
|
unknown = "mesh-config's resolved values cannot be listed: " + err.Error()
|
|
} else {
|
|
var want []string
|
|
for _, k := range keys {
|
|
if strings.HasSuffix(k, "."+buildstep.CapacitySetting) && strings.Contains(k, ".mesh-build-") {
|
|
want = append(want, k)
|
|
}
|
|
}
|
|
raws, err := u.Resolved.values(want)
|
|
if err != nil {
|
|
unknown = "mesh-config's resolved capacities cannot be read: " + err.Error()
|
|
}
|
|
for i, k := range want {
|
|
if err != nil {
|
|
break // the read failed: the values are unknown to this tool, not unreadable in mesh-config
|
|
}
|
|
var r struct {
|
|
Value json.RawMessage `json:"value"`
|
|
Source string `json:"source"`
|
|
}
|
|
key := strings.TrimSuffix(k, "."+buildstep.CapacitySetting)
|
|
n := 0
|
|
switch {
|
|
case i >= len(raws) || len(raws[i]) == 0:
|
|
unreadable[key] = "its resolved key holds no value"
|
|
case json.Unmarshal(raws[i], &r) != nil:
|
|
unreadable[key] = "its resolved key is not a resolved value"
|
|
case r.Source == "unreadable":
|
|
unreadable[key] = "mesh-config resolves it as unreadable"
|
|
case len(r.Value) == 0 || string(r.Value) == "null" || json.Unmarshal(r.Value, &n) != nil || n < 0:
|
|
// The capacity is the node's integer on its layer of mesh-config (novox/hq ADR 0567 §12).
|
|
unreadable[key] = fmt.Sprintf("its resolved value %s is not a whole number of asks at once", strings.TrimSpace(string(r.Value)))
|
|
default:
|
|
resolved[key] = n
|
|
}
|
|
}
|
|
}
|
|
if unknown != "" {
|
|
note("capacity unknown to this tool: %s; eligibility is not answered for a seat that holds its module", unknown)
|
|
}
|
|
for key := range unreadable {
|
|
node, module, _ := strings.Cut(key, ".")
|
|
bm := buildSeatOfModule(module)
|
|
seat(node, bm.AskLanguage())
|
|
}
|
|
// Every node with a resolved capacity is listed, so "is this node eligible" is answered for it.
|
|
for key := range resolved {
|
|
node, module, _ := strings.Cut(key, ".")
|
|
bm := buildSeatOfModule(module)
|
|
seat(node, bm.AskLanguage())
|
|
}
|
|
if len(buildSeats) == 0 {
|
|
note("no build seat is known from the seat definitions and the seat list, so no steps or pauses are read")
|
|
}
|
|
// The steps of each build module.
|
|
steps := map[string]map[string][]usageStep{} // node -> language -> steps
|
|
for _, bm := range buildSeats {
|
|
lang, module := bm.AskLanguage(), cmpOr(bm.Module, bm.Seat)
|
|
var sr StateRead
|
|
if u.StepsOf != nil {
|
|
sr = u.StepsOf(bm.Seat)
|
|
}
|
|
if sr.Keys == nil || sr.Get == nil {
|
|
note("the steps of %s are not readable here", module)
|
|
continue
|
|
}
|
|
keys, err := sr.Keys()
|
|
if err != nil {
|
|
note("the steps of %s cannot be listed: %v", module, err)
|
|
continue
|
|
}
|
|
// A step id is its start in base-36 milliseconds, so the newest keys are the ones of the last day; at most
|
|
// MaxStateReads of them are read, newest first.
|
|
type keyAt struct {
|
|
key string
|
|
at time.Time
|
|
}
|
|
var recent []keyAt
|
|
for _, k := range keys {
|
|
_, _, id, err := buildstep.ParseStepKey(k)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+buildstep.StepBound {
|
|
recent = append(recent, keyAt{k, at})
|
|
}
|
|
}
|
|
sort.Slice(recent, func(i, j int) bool { return recent[i].at.After(recent[j].at) })
|
|
if len(recent) > MaxStateReads {
|
|
note("the steps of %s: read the newest %d of %d keys of the last day; the older ones are not counted", module, MaxStateReads, len(recent))
|
|
recent = recent[:MaxStateReads]
|
|
}
|
|
want := make([]string, len(recent))
|
|
for i, r := range recent {
|
|
want[i] = r.key
|
|
}
|
|
raws, err := sr.values(want)
|
|
if err != nil {
|
|
note("the steps of %s cannot all be read: %v", module, err)
|
|
}
|
|
for i := range want {
|
|
if i >= len(raws) || len(raws[i]) == 0 {
|
|
continue
|
|
}
|
|
s, err := buildstep.DecodeStep(raws[i])
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if steps[s.Node] == nil {
|
|
steps[s.Node] = map[string][]usageStep{}
|
|
}
|
|
steps[s.Node][lang] = append(steps[s.Node][lang], usageStep{s, stepKind(s)})
|
|
seat(s.Node, lang)
|
|
}
|
|
}
|
|
|
|
// Pauses: paused.<node> of each module; and capacity.<node>, the capacity the module says it judges by.
|
|
paused := map[string]bool{} // <node>.<language>
|
|
judged := map[string]buildstep.Capacity{}
|
|
for _, bm := range buildSeats {
|
|
lang, module := bm.AskLanguage(), cmpOr(bm.Module, bm.Seat)
|
|
var pr StateRead
|
|
if u.PausesOf != nil {
|
|
pr = u.PausesOf(bm.Seat)
|
|
}
|
|
if pr.Get == nil {
|
|
note("the pauses of %s are not readable here, so none is said paused", module)
|
|
continue
|
|
}
|
|
for node := range seats {
|
|
if seats[node][lang] == nil {
|
|
continue
|
|
}
|
|
key, err := buildstep.PauseKey(node)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
raw, err := pr.Get(key)
|
|
if err != nil {
|
|
note("the pause of %s on %s cannot be read: %v", module, node, err)
|
|
continue
|
|
}
|
|
if len(raw) > 0 {
|
|
if p, err := buildstep.DecodePause(raw); err == nil {
|
|
paused[node+"."+lang] = p.Paused
|
|
}
|
|
}
|
|
if key, err := buildstep.CapacityKey(node); err == nil {
|
|
if raw, err := pr.Get(key); err == nil && len(raw) > 0 {
|
|
if c, err := buildstep.DecodeCapacity(raw); err == nil {
|
|
judged[node+"."+lang] = c
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
nodes := make([]string, 0, len(seats))
|
|
for n := range seats {
|
|
nodes = append(nodes, n)
|
|
}
|
|
sort.Strings(nodes)
|
|
|
|
// What each holder runs: its seat's verb current on its node, every node and seat asked at once (design 47 H2 §813 (9)).
|
|
current, unanswered := u.currents(nodes, seats)
|
|
if u.Current == nil {
|
|
note("the build seats' verb current is not asked here, so no seat is said held")
|
|
}
|
|
for _, node := range nodes {
|
|
nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{},
|
|
CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"}
|
|
for _, bm := range buildSeats {
|
|
lang := bm.AskLanguage()
|
|
s := seats[node][lang]
|
|
if s == nil {
|
|
continue
|
|
}
|
|
capacity, ok := resolved[node+"."+s.Module]
|
|
bad, isBad := unreadable[node+"."+s.Module]
|
|
switch {
|
|
case unknown != "":
|
|
s.CapacityFrom = "capacity unknown to this tool: " + unknown
|
|
case isBad:
|
|
s.CapacityFrom = "mesh-config's value for " + s.Module + " on " + node + " cannot be read (" + bad + "), so its puller uses 0"
|
|
case ok:
|
|
s.Capacity, s.CapacityFrom = capacity, "mesh-config, resolved for "+s.Module+" on "+node
|
|
default:
|
|
s.CapacityFrom = fmt.Sprintf("the modules' default 0: mesh-config holds no resolved value for %s on %s", s.Module, node)
|
|
}
|
|
if c, ok := judged[node+"."+lang]; ok && s.Holds && unknown == "" && !isBad && c.Configured == s.Capacity {
|
|
// The module running on the node says its build hours, and judges by less than mesh-config gives when its
|
|
// node's build CPU bounds it or the time is outside its build hours.
|
|
s.BuildHours, s.OutsideHours = c.BuildHours, c.OutsideHours
|
|
if c.BuildHoursFault != "" {
|
|
s.BuildHours, s.BuildHoursFault = "any hour", c.BuildHoursFault
|
|
}
|
|
if c.Effective != c.Configured {
|
|
s.Configured, s.Capacity = s.Capacity, c.Effective
|
|
s.CapacityFrom = fmt.Sprintf("%s, as %s on %s judges it (said %s)", c.Why, s.Module, node, c.At.UTC().Format(time.RFC3339))
|
|
}
|
|
}
|
|
s.Paused = paused[node+"."+lang]
|
|
s.EligibleKnown = true
|
|
switch {
|
|
case !s.Holds && s.Phase == "":
|
|
s.WhyNot = s.Module + " is not on " + node
|
|
case !s.Holds:
|
|
s.WhyNot = s.Seat + "'s current on " + node + " answers nothing (" + unanswered[node+"."+lang] + "); its have says " + s.Phase
|
|
case s.Paused:
|
|
s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)"
|
|
case unknown != "":
|
|
s.EligibleKnown = false
|
|
s.WhyNot = "not known: this tool cannot read its capacity (" + unknown + ")"
|
|
case isBad:
|
|
s.WhyNot = "its capacity on " + node + " cannot be read, so its puller uses 0"
|
|
case s.OutsideHours:
|
|
s.WhyNot = "outside its build hours " + s.BuildHours + " on " + node + ": it takes nothing new until they open"
|
|
case s.Capacity == 0:
|
|
s.WhyNot = "its capacity on " + node + " is 0"
|
|
}
|
|
if s.EligibleKnown {
|
|
e := s.WhyNot == ""
|
|
s.Eligible = &e
|
|
}
|
|
switch {
|
|
case s.Eligible != nil && *s.Eligible:
|
|
nu.Eligible = "eligible"
|
|
case !s.EligibleKnown && nu.Eligible != "eligible":
|
|
nu.Eligible = "not known"
|
|
case nu.Eligible == "":
|
|
nu.Eligible = "not eligible"
|
|
}
|
|
s.fill(steps[node][lang], current[node+"."+lang], now)
|
|
nu.Said = append(nu.Said, s.said())
|
|
nu.Seats = append(nu.Seats, *s)
|
|
}
|
|
ans.Nodes = append(ans.Nodes, nu)
|
|
}
|
|
|
|
if u.Backlog == nil {
|
|
note("the backlog of the seats' streams is not readable here")
|
|
} else if rows, err := u.Backlog(); err != nil {
|
|
note("the backlog of the seats' streams is read by the view program, which did not answer: %v", err)
|
|
} else if rows != nil {
|
|
ans.Backlog = rows
|
|
}
|
|
return ans
|
|
}
|
|
|
|
// currents asks every seat on every node its verb current at once: the answers by <node>.<language>, and for each seat that
|
|
// answered nothing, why.
|
|
func (u UsageReader) currents(nodes []string, seats map[string]map[string]*SeatUsage) (map[string][]currentAsk, map[string]string) {
|
|
answers, unanswered := map[string][]currentAsk{}, map[string]string{}
|
|
if u.Current == nil {
|
|
for _, node := range nodes {
|
|
for lang := range seats[node] {
|
|
unanswered[node+"."+lang] = "not asked here"
|
|
}
|
|
}
|
|
return answers, unanswered
|
|
}
|
|
var mu sync.Mutex
|
|
var wg sync.WaitGroup
|
|
for _, node := range nodes {
|
|
for lang, s := range seats[node] {
|
|
if s.Seat == "" {
|
|
continue
|
|
}
|
|
wg.Add(1)
|
|
go func(node, lang string, s *SeatUsage) {
|
|
defer wg.Done()
|
|
raw, err := u.Current(s.Seat, node)
|
|
var a currentAnswer
|
|
if err == nil {
|
|
err = json.Unmarshal(raw, &a)
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
switch {
|
|
case err != nil:
|
|
unanswered[node+"."+lang] = err.Error()
|
|
case a.Node != "" && a.Node != node:
|
|
unanswered[node+"."+lang] = "it answered for " + a.Node
|
|
default:
|
|
s.Holds = true
|
|
answers[node+"."+lang] = a.Asks
|
|
}
|
|
}(node, lang, s)
|
|
}
|
|
}
|
|
wg.Wait()
|
|
return answers, unanswered
|
|
}
|
|
|
|
// fill counts a seat's work: the asks running now as its seat's current lists them (design 47 H2 §813 (9)), a step's key
|
|
// left running (started past buildstep.StepBound, or an ask its holder's current does not list, or a hand step whose holder
|
|
// answers nothing: no step behind the key runs any more), and the two windows.
|
|
func (s *SeatUsage) fill(steps []usageStep, asks []currentAsk, now time.Time) {
|
|
sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) })
|
|
sort.Slice(asks, func(i, j int) bool { return asks[i].Started.Before(asks[j].Started) })
|
|
runs := map[string]bool{}
|
|
for _, a := range asks {
|
|
runs[a.ID] = true
|
|
tool := a.Kind + " ask"
|
|
for _, us := range steps {
|
|
if us.step.ID == a.ID {
|
|
tool = us.step.Tool
|
|
}
|
|
}
|
|
s.Steps = append(s.Steps, StepUsage{ID: a.ID, Kind: a.Kind, Tool: tool, Started: a.Started, Secs: int64(now.Sub(a.Started).Seconds())})
|
|
s.Running.Total++
|
|
s.Running.ByKind[a.Kind]++
|
|
}
|
|
var live []usageStep // ended, or running and not left: what the windows count
|
|
for _, us := range steps {
|
|
st := us.step
|
|
if why := endedStale(st); why != "" {
|
|
s.EndedStale = append(s.EndedStale, StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Why: why})
|
|
continue
|
|
}
|
|
if st.State != buildstep.StepRunning {
|
|
live = append(live, us)
|
|
continue
|
|
}
|
|
line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())}
|
|
switch {
|
|
case runs[st.ID]:
|
|
case now.Sub(st.Started) > buildstep.StepBound:
|
|
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), buildstep.StepBound)
|
|
case !s.Holds:
|
|
line.Why = "its key says running, and " + s.Seat + "'s current on its node answers nothing, so no step behind it runs"
|
|
case us.kind != KindHand:
|
|
line.Why = "its key says running, and " + s.Seat + "'s current on its node does not list it, so no step behind it runs"
|
|
}
|
|
if line.Why != "" {
|
|
s.LeftRunning = append(s.LeftRunning, line)
|
|
continue
|
|
}
|
|
if us.kind == KindHand {
|
|
s.Steps = append(s.Steps, line)
|
|
}
|
|
live = append(live, us)
|
|
}
|
|
s.LastHour = window(live, now, time.Hour, s.Capacity)
|
|
s.LastDay = window(live, now, 24*time.Hour, s.Capacity)
|
|
}
|
|
|
|
// said is the seat's use in one line.
|
|
func (s *SeatUsage) said() string {
|
|
head := s.Seat + ": eligible"
|
|
switch {
|
|
case !s.EligibleKnown:
|
|
head = s.Seat + ": eligibility " + s.WhyNot
|
|
case s.Eligible == nil || !*s.Eligible:
|
|
head = s.Seat + ": not eligible, " + s.WhyNot
|
|
}
|
|
kinds := make([]string, 0, len(s.Running.ByKind))
|
|
for k, n := range s.Running.ByKind {
|
|
kinds = append(kinds, fmt.Sprintf("%s %d", k, n))
|
|
}
|
|
sort.Strings(kinds)
|
|
line := fmt.Sprintf("%s; %d of capacity %d running", head, s.Running.Total, s.Capacity)
|
|
if len(kinds) > 0 {
|
|
line += " (" + strings.Join(kinds, ", ") + ")"
|
|
}
|
|
if len(s.LeftRunning) > 0 {
|
|
line += fmt.Sprintf(", %d left running", len(s.LeftRunning))
|
|
}
|
|
return line + fmt.Sprintf("; last hour %d finished, median %d s, at capacity %.0f%% of it; last day %d finished, median %d s, at capacity %.1f%% of it",
|
|
s.LastHour.Finished, s.LastHour.MedianSecs, s.LastHour.AtCapacityShare*100, s.LastDay.Finished, s.LastDay.MedianSecs, s.LastDay.AtCapacityShare*100)
|
|
}
|
|
|
|
// window counts the steps of the last span before now: started in it, finished in it with their total and median time,
|
|
// and the time in it during which capacity asks (not hand steps) ran at once.
|
|
func window(steps []usageStep, now time.Time, span time.Duration, capacity int) WindowUsage {
|
|
from := now.Add(-span)
|
|
w := WindowUsage{ByKind: map[string]KindWindow{}}
|
|
var all []int64
|
|
byKind := map[string][]int64{}
|
|
type edge struct {
|
|
at time.Time
|
|
d int
|
|
}
|
|
var edges []edge
|
|
for _, us := range steps {
|
|
st := us.step
|
|
if !st.Started.Before(from) {
|
|
w.Started++
|
|
}
|
|
if !st.Ended.IsZero() && !st.Ended.Before(from) {
|
|
w.Finished++
|
|
if st.State == buildstep.StepFailed || st.State == buildstep.StepRefused {
|
|
w.Failed++
|
|
}
|
|
secs := int64(st.Ended.Sub(st.Started).Seconds())
|
|
all = append(all, secs)
|
|
byKind[us.kind] = append(byKind[us.kind], secs)
|
|
}
|
|
if us.kind == KindHand {
|
|
continue
|
|
}
|
|
start, end := st.Started, st.Ended
|
|
if end.IsZero() {
|
|
end = now
|
|
}
|
|
if start.Before(from) {
|
|
start = from
|
|
}
|
|
if end.After(start) {
|
|
edges = append(edges, edge{start, 1}, edge{end, -1})
|
|
}
|
|
}
|
|
w.TotalSecs, w.MedianSecs = sumMedian(all)
|
|
for k, d := range byKind {
|
|
kw := KindWindow{Finished: len(d)}
|
|
kw.TotalSecs, kw.MedianSecs = sumMedian(d)
|
|
w.ByKind[k] = kw
|
|
}
|
|
if capacity > 0 {
|
|
// An end before a start at the same instant: a slot handed from one ask to the next is not a moment at capacity+1.
|
|
sort.Slice(edges, func(i, j int) bool {
|
|
if edges[i].at.Equal(edges[j].at) {
|
|
return edges[i].d < edges[j].d
|
|
}
|
|
return edges[i].at.Before(edges[j].at)
|
|
})
|
|
var at time.Duration
|
|
n := 0
|
|
for i, e := range edges {
|
|
n += e.d
|
|
if n >= capacity && i+1 < len(edges) {
|
|
at += edges[i+1].at.Sub(e.at)
|
|
}
|
|
}
|
|
w.AtCapacitySecs = int64(at.Seconds())
|
|
w.AtCapacityShare = float64(int64(at*1000/span)) / 1000
|
|
}
|
|
return w
|
|
}
|
|
|
|
// sumMedian is the sum and the median of d (the lower middle of an even count); zero for none.
|
|
func sumMedian(d []int64) (sum, median int64) {
|
|
if len(d) == 0 {
|
|
return 0, 0
|
|
}
|
|
s := append([]int64(nil), d...)
|
|
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
|
for _, v := range s {
|
|
sum += v
|
|
}
|
|
return sum, s[(len(s)-1)/2]
|
|
}
|
|
|
|
// cmpOr is the first of two names that is not empty.
|
|
func cmpOr(a, b string) string {
|
|
if a != "" {
|
|
return a
|
|
}
|
|
return b
|
|
}
|