package view // The build nodes' use, in one answer (the operator, 2026-10-26: "how much are our build agents being used, and is ace // eligible for building?" took six calls over four tools). Read from the bus's state alone: which build seats each node // holds (the node-engine's have of mesh-build-), its capacity for each (mesh-config's resolved value of the // module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each // build module's steps state (ADR 0525 §6: an ask ends on .) the asks running now by kind, the ones left // running, and the steps started and finished in the last hour and day with their total and median time and how long the // node was at its capacity. A try's asks are told from a candidate's by the checker's tries state (each try names the // asks it put). The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver), // asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the // bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it. import ( "encoding/json" "fmt" "sort" "strconv" "strings" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // BuildUsageTool is the view's tool of the build nodes' use. const BuildUsageTool = "build_usage" // endedStale says why a step's end was written by its build module for a key left running (mesh-sdk buildstep's ending // of stale steps): a restart's, or the step bound's; "" for a step that ran to its own end. Such an end is the moment the // module noticed, not the step's, so it is counted apart and in neither the medians nor the time at capacity. func endedStale(st delivery.Step) string { if st.State != delivery.StepFailed { return "" } if st.Reason == buildstep.RestartReason || strings.HasPrefix(st.Reason, "past the step bound") { return st.Reason } return "" } // The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker // put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity). const ( KindTry = "try" KindHand = "hand" ) // UsageReader reads what build_usage answers from. type UsageReader struct { // Steps and Pauses are each build module's states by language (go, node, image). Steps map[string]StateRead Pauses map[string]StateRead // Resolved is mesh-config's resolved values (..); Haves the node-engine's have. Resolved StateRead Haves StateRead // Tries is the checker's tries (try-), each naming the asks it put. Tries StateRead // Backlog is the seats' streams' waiting and held asks, as the view program reads them. Backlog func() ([]BacklogRow, error) Now func() time.Time } // UsageAnswer is build_usage's answer. type UsageAnswer struct { At time.Time `json:"at"` Nodes []NodeUsage `json:"nodes"` Backlog []BacklogRow `json:"backlog"` // TriesRunning are the tries the checker runs now; their asks are counted under check until the try ends and names them. TriesRunning []TryRunning `json:"tries_running"` Notes []string `json:"notes,omitempty"` } // TryRunning is one try the checker runs now. type TryRunning struct { ID string `json:"id"` Repository string `json:"repository"` Started time.Time `json:"started,omitzero"` } // NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU. type NodeUsage struct { Node string `json:"node"` // Eligible is the node's verdict: "eligible" when a seat of it is, "not eligible" when every seat is known not to be, // and "not known" when none is known eligible and this tool cannot tell for some seat. Eligible string `json:"eligible"` // Said is one line per seat, the answer in words. Said []string `json:"said"` Seats []SeatUsage `json:"seats"` CPU string `json:"cpu"` } // SeatUsage is one build seat on one node. type SeatUsage struct { Seat string `json:"seat"` Module string `json:"module"` // Holds says the node runs the seat's module (its have says running); Phase is what its have says, or absent. Holds bool `json:"holds"` Phase string `json:"phase,omitempty"` // Capacity is the asks it runs at once; CapacityFrom where that came from. Capacity int `json:"capacity"` CapacityFrom string `json:"capacity_from"` Paused bool `json:"paused"` // Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not. // EligibleKnown is false when this tool cannot tell (it cannot read mesh-config's capacities): Eligible is then // absent, never false, and WhyNot starts "not known: ". Eligible *bool `json:"eligible,omitempty"` EligibleKnown bool `json:"eligible_known"` WhyNot string `json:"why_not,omitempty"` Running RunningUsage `json:"running"` LeftRunning []StepUsage `json:"left_running"` // EndedStale are the steps of the last day whose end their build module wrote for a key left running (a restart, or // past the step bound): counted here alone, never as finished. EndedStale []StepUsage `json:"ended_stale"` LastHour WindowUsage `json:"last_hour"` LastDay WindowUsage `json:"last_day"` Steps []StepUsage `json:"running_steps"` } // usageStep is a step read, with the kind it is counted under. type usageStep struct { step delivery.Step kind string } // RunningUsage is the asks running now: how many in all and by kind, against the capacity. type RunningUsage struct { Total int `json:"total"` ByKind map[string]int `json:"by_kind"` } // StepUsage is one step running now, or one left running and why. type StepUsage struct { ID string `json:"id"` Kind string `json:"kind"` Tool string `json:"tool"` Started time.Time `json:"started"` Secs int64 `json:"secs"` Why string `json:"why,omitempty"` } // WindowUsage is the steps of one window: started and finished in it, the finished ones' total and median time, by kind, // and how long the node ran its capacity of asks (a hand step takes no slot). type WindowUsage struct { Started int `json:"started"` Finished int `json:"finished"` Failed int `json:"failed"` TotalSecs int64 `json:"total_secs"` MedianSecs int64 `json:"median_secs"` ByKind map[string]KindWindow `json:"by_kind"` AtCapacitySecs int64 `json:"at_capacity_secs"` AtCapacityShare float64 `json:"at_capacity_share"` } // KindWindow is one kind's finished steps in a window. type KindWindow struct { Finished int `json:"finished"` TotalSecs int64 `json:"total_secs"` MedianSecs int64 `json:"median_secs"` } // stepIDTime is when a step id was made: its base-36 milliseconds before the six hex digits (buildstep.NewID). func stepIDTime(id string) (time.Time, bool) { if len(id) <= 6 { return time.Time{}, false } ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64) if err != nil { return time.Time{}, false } return time.UnixMilli(ms).UTC(), true } // stepKind is the kind a step is counted under. func stepKind(s delivery.Step, tries map[string]bool) string { kind, ask := strings.CutSuffix(s.Tool, " ask") if !ask { return KindHand } if kind == delivery.AskCheck && tries[s.ID] { return KindTry } return kind } // Usage reads every state and answers. A state it cannot read is said in a note, never left out silently. func (u UsageReader) Usage() UsageAnswer { now := time.Now().UTC() if u.Now != nil { now = u.Now().UTC() } ans := UsageAnswer{At: now, Nodes: []NodeUsage{}, Backlog: []BacklogRow{}, TriesRunning: []TryRunning{}} note := func(f string, a ...any) { ans.Notes = append(ans.Notes, fmt.Sprintf(f, a...)) } tries := u.tries(now, &ans, note) seats := map[string]map[string]*SeatUsage{} // node -> language -> seat seat := func(node, lang string) *SeatUsage { if seats[node] == nil { seats[node] = map[string]*SeatUsage{} } if s := seats[node][lang]; s != nil { return s } name, _ := delivery.BuildSeat(lang) module, _ := delivery.BuildModuleOf(lang) s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, EndedStale: []StepUsage{}, Steps: []StepUsage{}} seats[node][lang] = s return s } // Which node runs which build module: the have, .mesh-build-. haves := map[string]delivery.Have{} if u.Haves.Keys == nil || u.Haves.Get == nil { note("the node-engine's have is not readable here, so no seat is said held") } else if keys, err := u.Haves.Keys(); err != nil { note("the node-engine's have cannot be listed, so no seat is said held: %v", err) } else { var want []string for _, k := range keys { for _, lang := range delivery.AskLanguages { if m, _ := delivery.BuildModuleOf(lang); strings.HasSuffix(k, "."+m) { want = append(want, k) } } } raws, err := u.Haves.values(want) if err != nil { note("the node-engine's have cannot be read: %v", err) } for i, k := range want { if i < len(raws) && len(raws[i]) > 0 { h, err := delivery.DecodeHave(raws[i]) if err != nil { note("the have %s cannot be read, so its seat is not said held: %v", k, err) continue } haves[k] = h } } } for k, h := range haves { node, module, _ := strings.Cut(k, ".") lang := strings.TrimPrefix(module, "mesh-build-") s := seat(node, lang) s.Phase, s.Holds = string(h.Phase), h.Phase == delivery.PhaseRunning } // Capacity: mesh-config's resolved value of the module's `capacity` on each node. resolved := map[string]string{} // . -> setting unknown := "" // why this tool cannot say a capacity: mesh-config's values cannot be read here unreadable := map[string]string{} // . -> why its resolved key holds no readable value: the puller uses 0 if u.Resolved.Keys == nil || u.Resolved.Get == nil { unknown = "mesh-config's resolved values are not readable here" } else if keys, err := u.Resolved.Keys(); err != nil { unknown = "mesh-config's resolved values cannot be listed: " + err.Error() } else { var want []string for _, k := range keys { if strings.HasSuffix(k, "."+buildstep.CapacitySetting) && strings.Contains(k, ".mesh-build-") { want = append(want, k) } } raws, err := u.Resolved.values(want) if err != nil { unknown = "mesh-config's resolved capacities cannot be read: " + err.Error() } for i, k := range want { if err != nil { break // the read failed: the values are unknown to this tool, not unreadable in mesh-config } var r struct { Value json.RawMessage `json:"value"` Source string `json:"source"` } key := strings.TrimSuffix(k, "."+buildstep.CapacitySetting) setting := "" switch { case i >= len(raws) || len(raws[i]) == 0: unreadable[key] = "its resolved key holds no value" case json.Unmarshal(raws[i], &r) != nil: unreadable[key] = "its resolved key is not a resolved value" case r.Source == "unreadable": unreadable[key] = "mesh-config resolves it as unreadable" case len(r.Value) == 0 || string(r.Value) == "null" || json.Unmarshal(r.Value, &setting) != nil || strings.TrimSpace(setting) == "": unreadable[key] = fmt.Sprintf("its resolved value %s is empty or not a capacity", strings.TrimSpace(string(r.Value))) default: resolved[key] = setting } } } if unknown != "" { note("capacity unknown to this tool: %s; eligibility is not answered for a seat that holds its module", unknown) } for key := range unreadable { node, module, _ := strings.Cut(key, ".") seat(node, strings.TrimPrefix(module, "mesh-build-")) } // Every node a capacity names is listed, so "is this node eligible" is answered for any node: the nodes each module's // resolved value names (as resolved on the first node in name order), or the modules' default when mesh-config // resolves none for the module. keys := make([]string, 0, len(resolved)) for key := range resolved { keys = append(keys, key) } sort.Strings(keys) named := map[string]string{} // language -> the module's setting, as resolved on some node namedOn := map[string]string{} // language -> that node for _, key := range keys { node, module, _ := strings.Cut(key, ".") lang := strings.TrimPrefix(module, "mesh-build-") seat(node, lang) if _, ok := named[lang]; !ok { named[lang], namedOn[lang] = resolved[key], node } } for _, lang := range delivery.AskLanguages { setting, ok := named[lang] if !ok { setting = buildstep.DefaultCapacities } if caps, err := buildrun.ParseCapacity(setting); err == nil { for node := range caps { seat(node, lang) } } } // The steps of each build module. steps := map[string]map[string][]usageStep{} // node -> language -> steps for _, lang := range delivery.AskLanguages { module, _ := delivery.BuildModuleOf(lang) sr, ok := u.Steps[lang] if !ok || sr.Keys == nil || sr.Get == nil { note("the steps of %s are not readable here", module) continue } keys, err := sr.Keys() if err != nil { note("the steps of %s cannot be listed: %v", module, err) continue } // A step id is its start in base-36 milliseconds, so the newest keys are the ones of the last day; at most // MaxStateReads of them are read, newest first. type keyAt struct { key string at time.Time } var recent []keyAt for _, k := range keys { _, id, err := delivery.ParseStepKey(k) if err != nil { continue } if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+delivery.StepBound { recent = append(recent, keyAt{k, at}) } } sort.Slice(recent, func(i, j int) bool { return recent[i].at.After(recent[j].at) }) if len(recent) > MaxStateReads { note("the steps of %s: read the newest %d of %d keys of the last day; the older ones are not counted", module, MaxStateReads, len(recent)) recent = recent[:MaxStateReads] } want := make([]string, len(recent)) for i, r := range recent { want[i] = r.key } raws, err := sr.values(want) if err != nil { note("the steps of %s cannot all be read: %v", module, err) } for i := range want { if i >= len(raws) || len(raws[i]) == 0 { continue } s, err := delivery.DecodeStep(raws[i]) if err != nil { continue } if steps[s.Node] == nil { steps[s.Node] = map[string][]usageStep{} } steps[s.Node][lang] = append(steps[s.Node][lang], usageStep{s, stepKind(s, tries)}) seat(s.Node, lang) } } // Pauses: paused. of each module. paused := map[string]bool{} // . for _, lang := range delivery.AskLanguages { module, _ := delivery.BuildModuleOf(lang) pr, ok := u.Pauses[lang] if !ok || pr.Get == nil { note("the pauses of %s are not readable here, so none is said paused", module) continue } for node := range seats { if seats[node][lang] == nil { continue } key, err := delivery.PauseKey(node) if err != nil { continue } raw, err := pr.Get(key) if err != nil { note("the pause of %s on %s cannot be read: %v", module, node, err) continue } if len(raw) > 0 { if p, err := delivery.DecodePause(raw); err == nil { paused[node+"."+lang] = p.Paused } } } } nodes := make([]string, 0, len(seats)) for n := range seats { nodes = append(nodes, n) } sort.Strings(nodes) for _, node := range nodes { nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{}, CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"} for _, lang := range delivery.AskLanguages { s := seats[node][lang] if s == nil { continue } setting, ok := resolved[node+"."+s.Module] from := "mesh-config, resolved for " + s.Module + " on " + node if _, has := named[lang]; !ok && has { from = "mesh-config, resolved for " + s.Module + " on " + namedOn[lang] + " (" + s.Module + " is not assigned to " + node + ")" // mesh-config resolves the module's value only where it is assigned; elsewhere its capacity is what that // value names for the node. setting, ok = named[lang], true } bad, isBad := unreadable[node+"."+s.Module] if unknown != "" { s.CapacityFrom = "capacity unknown to this tool: " + unknown } else if isBad { s.CapacityFrom = "mesh-config's value for " + s.Module + " on " + node + " cannot be read (" + bad + "), so its puller uses 0" } else if ok { s.CapacityFrom = from if caps, err := buildrun.ParseCapacity(setting); err != nil { s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err) } else { s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node) } } else { caps, _ := buildrun.ParseCapacity(buildstep.DefaultCapacities) s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node) s.CapacityFrom = fmt.Sprintf("the modules' default %q: mesh-config holds no resolved value for %s on %s", buildstep.DefaultCapacities, s.Module, node) } s.Paused = paused[node+"."+lang] s.EligibleKnown = true switch { case !s.Holds && s.Phase == "": s.WhyNot = s.Module + " is not on " + node case !s.Holds: s.WhyNot = s.Module + " on " + node + " is " + s.Phase case s.Paused: s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)" case unknown != "": s.EligibleKnown = false s.WhyNot = "not known: this tool cannot read its capacity (" + unknown + ")" case isBad: s.WhyNot = "its capacity on " + node + " cannot be read, so its puller uses 0" case s.Capacity == 0: s.WhyNot = "its capacity on " + node + " is 0" } if s.EligibleKnown { e := s.WhyNot == "" s.Eligible = &e } switch { case s.Eligible != nil && *s.Eligible: nu.Eligible = "eligible" case !s.EligibleKnown && nu.Eligible != "eligible": nu.Eligible = "not known" case nu.Eligible == "": nu.Eligible = "not eligible" } s.fill(steps[node][lang], now) nu.Said = append(nu.Said, s.said()) nu.Seats = append(nu.Seats, *s) } ans.Nodes = append(ans.Nodes, nu) } if u.Backlog == nil { note("the backlog of the seats' streams is not readable here") } else if rows, err := u.Backlog(); err != nil { note("the backlog of the seats' streams is read by the view program, which did not answer: %v", err) } else if rows != nil { ans.Backlog = rows } return ans } // tries reads the checker's tries of the last day: the running ones, and the asks every one put. func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, ...any)) map[string]bool { asks := map[string]bool{} if u.Tries.Keys == nil || u.Tries.Get == nil { note("the checker's tries are not readable here, so a try's asks are counted under check") return asks } keys, err := u.Tries.Keys() if err != nil { note("the checker's tries cannot be listed, so a try's asks are counted under check: %v", err) return asks } var want []string for _, k := range keys { ns, err := strconv.ParseInt(strings.TrimPrefix(k, "try-"), 10, 64) if err == nil && strings.HasPrefix(k, "try-") && now.Sub(time.Unix(0, ns)) <= 48*time.Hour { want = append(want, k) } } raws, err := u.Tries.values(want) if err != nil { note("the checker's tries cannot all be read: %v", err) } for i := range want { if i >= len(raws) || len(raws[i]) == 0 { continue } var t struct { ID string `json:"id"` Repository string `json:"repository"` State string `json:"state"` Started time.Time `json:"started"` Asks []string `json:"asks"` } if json.Unmarshal(raws[i], &t) != nil { continue } for _, a := range t.Asks { asks[a] = true } if t.State == "running" { ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started}) } } if len(ans.TriesRunning) > 0 { note("a running try's asks are counted under check until the try ends and names them") } return asks } // fill counts a seat's steps: running now, left running (started past delivery.StepBound, or its module no longer running on // the node: no step behind the key runs any more), and the two windows. func (s *SeatUsage) fill(steps []usageStep, now time.Time) { sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) }) var live []usageStep // ended, or running and not left: what the windows count for _, us := range steps { st := us.step if why := endedStale(st); why != "" { s.EndedStale = append(s.EndedStale, StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Why: why}) continue } if st.State != delivery.StepRunning { live = append(live, us) continue } line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())} switch { case !s.Holds: line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs" case now.Sub(st.Started) > delivery.StepBound: line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), delivery.StepBound) } if line.Why != "" { s.LeftRunning = append(s.LeftRunning, line) continue } s.Steps = append(s.Steps, line) live = append(live, us) if us.kind != KindHand { s.Running.Total++ s.Running.ByKind[us.kind]++ } } s.LastHour = window(live, now, time.Hour, s.Capacity) s.LastDay = window(live, now, 24*time.Hour, s.Capacity) } // said is the seat's use in one line. func (s *SeatUsage) said() string { head := s.Seat + ": eligible" switch { case !s.EligibleKnown: head = s.Seat + ": eligibility " + s.WhyNot case s.Eligible == nil || !*s.Eligible: head = s.Seat + ": not eligible, " + s.WhyNot } kinds := make([]string, 0, len(s.Running.ByKind)) for k, n := range s.Running.ByKind { kinds = append(kinds, fmt.Sprintf("%s %d", k, n)) } sort.Strings(kinds) line := fmt.Sprintf("%s; %d of capacity %d running", head, s.Running.Total, s.Capacity) if len(kinds) > 0 { line += " (" + strings.Join(kinds, ", ") + ")" } if len(s.LeftRunning) > 0 { line += fmt.Sprintf(", %d left running", len(s.LeftRunning)) } return line + fmt.Sprintf("; last hour %d finished, median %d s, at capacity %.0f%% of it; last day %d finished, median %d s, at capacity %.1f%% of it", s.LastHour.Finished, s.LastHour.MedianSecs, s.LastHour.AtCapacityShare*100, s.LastDay.Finished, s.LastDay.MedianSecs, s.LastDay.AtCapacityShare*100) } // window counts the steps of the last span before now: started in it, finished in it with their total and median time, // and the time in it during which capacity asks (not hand steps) ran at once. func window(steps []usageStep, now time.Time, span time.Duration, capacity int) WindowUsage { from := now.Add(-span) w := WindowUsage{ByKind: map[string]KindWindow{}} var all []int64 byKind := map[string][]int64{} type edge struct { at time.Time d int } var edges []edge for _, us := range steps { st := us.step if !st.Started.Before(from) { w.Started++ } if !st.Ended.IsZero() && !st.Ended.Before(from) { w.Finished++ if st.State == delivery.StepFailed || st.State == delivery.StepRefused { w.Failed++ } secs := int64(st.Ended.Sub(st.Started).Seconds()) all = append(all, secs) byKind[us.kind] = append(byKind[us.kind], secs) } if us.kind == KindHand { continue } start, end := st.Started, st.Ended if end.IsZero() { end = now } if start.Before(from) { start = from } if end.After(start) { edges = append(edges, edge{start, 1}, edge{end, -1}) } } w.TotalSecs, w.MedianSecs = sumMedian(all) for k, d := range byKind { kw := KindWindow{Finished: len(d)} kw.TotalSecs, kw.MedianSecs = sumMedian(d) w.ByKind[k] = kw } if capacity > 0 { // An end before a start at the same instant: a slot handed from one ask to the next is not a moment at capacity+1. sort.Slice(edges, func(i, j int) bool { if edges[i].at.Equal(edges[j].at) { return edges[i].d < edges[j].d } return edges[i].at.Before(edges[j].at) }) var at time.Duration n := 0 for i, e := range edges { n += e.d if n >= capacity && i+1 < len(edges) { at += edges[i+1].at.Sub(e.at) } } w.AtCapacitySecs = int64(at.Seconds()) w.AtCapacityShare = float64(int64(at*1000/span)) / 1000 } return w } // sumMedian is the sum and the median of d (the lower middle of an even count); zero for none. func sumMedian(d []int64) (sum, median int64) { if len(d) == 0 { return 0, 0 } s := append([]int64(nil), d...) sort.Slice(s, func(i, j int) bool { return s[i] < s[j] }) for _, v := range s { sum += v } return sum, s[(len(s)-1)/2] }