novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

View: build_usage counts keys a build module ended for being left running apart, lists every node a capacity names, and says capacity unknown when mesh-config cannot be read

Browse Source 
Since mesh-sdk #90 a build module ends a left-running key as failed at the moment it notices; counted as finished,
such a step put hours into the medians and the time at capacity. The view takes mesh-sdk's step bound and restart
reason from the SDK it now pins.
This commit is contained in:
jschoubben
2026-10-26 14:58:47 +01:00
parent 1b4c860f63
commit 166a918b43
5 changed files with 103 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go
+3 -2
View File
@@ -20,9 +20,10 @@ import (
func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
return []stdio.Tool{
{Name: view.BuildUsageTool,
Description: "How the build nodes are used, per node: each build seat it holds (node-build-go, -node, -image) with its capacity " +
Description: "How the build nodes are used, and whether a node is eligible to build, per node (every node holding a build module or named by a capacity): each build seat it holds (node-build-go, -node, -image) with its capacity " +
"(mesh-config's resolved value, or the modules' default) and pause, and whether it is eligible to build and why not; the asks " +
"running now by kind (delivery, check, recheck, try) and the keys left running (past the step bound, or their module not running); " +
"running now by kind (delivery, check, recheck, try), the keys left running (past the step bound, or their module not running) and the ones " +
"their module ended for that (a restart, or past the step bound), counted apart and in no median or occupancy; " +
"the steps started and finished in the last hour and the last day, failed ones, total and median time, by kind, and how long and " +
"what share of the window the node ran its capacity of asks. Mesh-wide: the backlog of asks waiting and held on each build seat's " +
"stream, and the tries running. Read from the bus's states (have, mesh-config, the build modules' steps and pauses, the checker's tries); " +
modules/mesh-delivery-view/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-view/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81 h1:LhljEtt7FbNpVn67eOQbdsFDS8Qj6WPUF6l9IXr9mo4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 h1:s1WcG7Ll1OkEwwvMKEabdWvPvTiFCHghlXiY1LoIJDA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-view/internal/view/usage.go
+74 -17
View File
@@ -27,9 +27,18 @@ import (
// BuildUsageTool is the view's tool of the build nodes' use.
const BuildUsageTool = "build_usage"
// UsageStepBound is how long an ask step may say running before it is taken as left running: the checker's step bound
// (mesh-delivery-checker check.DefaultStepBound), after which no asker waits for it.
const UsageStepBound = 30 * time.Minute
// endedStale says why a step's end was written by its build module for a key left running (mesh-sdk buildstep's ending
// of stale steps): a restart's, or the step bound's; "" for a step that ran to its own end. Such an end is the moment the
// module noticed, not the step's, so it is counted apart and in neither the medians nor the time at capacity.
func endedStale(st delivery.Step) string {
if st.State != delivery.StepFailed {
return ""
}
if st.Reason == buildstep.RestartReason || strings.HasPrefix(st.Reason, "past the step bound") {
return st.Reason
}
return ""
}
// The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker
// put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity).
@@ -96,9 +105,12 @@ type SeatUsage struct {
WhyNot string `json:"why_not,omitempty"`
Running RunningUsage `json:"running"`
LeftRunning []StepUsage `json:"left_running"`
LastHour WindowUsage `json:"last_hour"`
LastDay WindowUsage `json:"last_day"`
Steps []StepUsage `json:"running_steps"`
// EndedStale are the steps of the last day whose end their build module wrote for a key left running (a restart, or
// past the step bound): counted here alone, never as finished.
EndedStale []StepUsage `json:"ended_stale"`
LastHour WindowUsage `json:"last_hour"`
LastDay WindowUsage `json:"last_day"`
Steps []StepUsage `json:"running_steps"`
}
// usageStep is a step read, with the kind it is counted under.
@@ -187,7 +199,7 @@ func (u UsageReader) Usage() UsageAnswer {
}
name, _ := delivery.BuildSeat(lang)
module, _ := delivery.BuildModuleOf(lang)
s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, Steps: []StepUsage{}}
s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, EndedStale: []StepUsage{}, Steps: []StepUsage{}}
seats[node][lang] = s
return s
}
@@ -231,10 +243,11 @@ func (u UsageReader) Usage() UsageAnswer {
// Capacity: mesh-config's resolved value of the module's `capacity` on each node.
resolved := map[string]string{} // <node>.<module> -> setting
unknown := "" // why no capacity can be said, when mesh-config's values cannot be read
if u.Resolved.Keys == nil || u.Resolved.Get == nil {
note("mesh-config's resolved values are not readable here, so each capacity is the modules' default %q", buildstep.DefaultCapacities)
unknown = "mesh-config's resolved values are not readable here"
} else if keys, err := u.Resolved.Keys(); err != nil {
note("mesh-config's resolved values cannot be listed, so each capacity is the modules' default %q: %v", buildstep.DefaultCapacities, err)
unknown = "mesh-config's resolved values cannot be listed: " + err.Error()
} else {
var want []string
for _, k := range keys {
@@ -244,7 +257,7 @@ func (u UsageReader) Usage() UsageAnswer {
}
raws, err := u.Resolved.values(want)
if err != nil {
note("mesh-config's resolved capacities cannot be read: %v", err)
unknown = "mesh-config's resolved capacities cannot be read: " + err.Error()
}
for i, k := range want {
var r struct {
@@ -256,9 +269,37 @@ func (u UsageReader) Usage() UsageAnswer {
}
}
}
if unknown != "" {
note("capacity unknown: %s, and a puller whose capacity cannot be read uses 0", unknown)
}
// Every node a capacity names is listed, so "is this node eligible" is answered for any node: the nodes each module's
// resolved value names (as resolved on the first node in name order), or the modules' default when mesh-config
// resolves none for the module.
keys := make([]string, 0, len(resolved))
for key := range resolved {
keys = append(keys, key)
}
sort.Strings(keys)
named := map[string]string{} // language -> the module's setting, as resolved on some node
namedOn := map[string]string{} // language -> that node
for _, key := range keys {
node, module, _ := strings.Cut(key, ".")
seat(node, strings.TrimPrefix(module, "mesh-build-"))
lang := strings.TrimPrefix(module, "mesh-build-")
seat(node, lang)
if _, ok := named[lang]; !ok {
named[lang], namedOn[lang] = resolved[key], node
}
}
for _, lang := range delivery.AskLanguages {
setting, ok := named[lang]
if !ok {
setting = buildstep.DefaultCapacities
}
if caps, err := buildrun.ParseCapacity(setting); err == nil {
for node := range caps {
seat(node, lang)
}
}
}
// The steps of each build module.
steps := map[string]map[string][]usageStep{} // node -> language -> steps
@@ -286,7 +327,7 @@ func (u UsageReader) Usage() UsageAnswer {
if err != nil {
continue
}
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+UsageStepBound {
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+delivery.StepBound {
recent = append(recent, keyAt{k, at})
}
}
@@ -362,8 +403,18 @@ func (u UsageReader) Usage() UsageAnswer {
if s == nil {
continue
}
if setting, ok := resolved[node+"."+s.Module]; ok {
s.CapacityFrom = "mesh-config, resolved for " + s.Module + " on " + node
setting, ok := resolved[node+"."+s.Module]
from := "mesh-config, resolved for " + s.Module + " on " + node
if _, has := named[lang]; !ok && has {
from = "mesh-config, resolved for " + s.Module + " on " + namedOn[lang] + " (" + s.Module + " is not assigned to " + node + ")"
// mesh-config resolves the module's value only where it is assigned; elsewhere its capacity is what that
// value names for the node.
setting, ok = named[lang], true
}
if unknown != "" {
s.CapacityFrom = "capacity unknown: the puller uses 0 (" + unknown + ")"
} else if ok {
s.CapacityFrom = from
if caps, err := buildrun.ParseCapacity(setting); err != nil {
s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err)
} else {
@@ -380,6 +431,8 @@ func (u UsageReader) Usage() UsageAnswer {
s.WhyNot = s.Module + " is not on " + node
case !s.Holds:
s.WhyNot = s.Module + " on " + node + " is " + s.Phase
case unknown != "":
s.WhyNot = "capacity unknown: the puller uses 0"
case s.Capacity == 0:
s.WhyNot = "its capacity on " + node + " is 0"
case s.Paused:
@@ -455,13 +508,17 @@ func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, ..
return asks
}
// fill counts a seat's steps: running now, left running (started past UsageStepBound, or its module no longer running on
// fill counts a seat's steps: running now, left running (started past delivery.StepBound, or its module no longer running on
// the node: no step behind the key runs any more), and the two windows.
func (s *SeatUsage) fill(steps []usageStep, now time.Time) {
sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) })
var live []usageStep // ended, or running and not left: what the windows count
for _, us := range steps {
st := us.step
if why := endedStale(st); why != "" {
s.EndedStale = append(s.EndedStale, StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Why: why})
continue
}
if st.State != delivery.StepRunning {
live = append(live, us)
continue
@@ -470,8 +527,8 @@ func (s *SeatUsage) fill(steps []usageStep, now time.Time) {
switch {
case !s.Holds:
line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs"
case now.Sub(st.Started) > UsageStepBound:
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), UsageStepBound)
case now.Sub(st.Started) > delivery.StepBound:
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), delivery.StepBound)
}
if line.Why != "" {
s.LeftRunning = append(s.LeftRunning, line)
modules/mesh-delivery-view/internal/view/usage_test.go
+23 -10
View File
@@ -65,15 +65,18 @@ func usageFixture() UsageReader {
add("shanks", stepID(at("06:00"), "a00005"), "recheck ask", "passed", at("06:00"), at("06:10"))
add("shanks", stepID(at("10:00"), "a00006"), "check ask", "running", at("10:00"), time.Time{})
add("shanks", stepID(at("11:45"), "a00007"), "build_go_test", "passed", at("11:45"), at("11:46"))
// Two keys a build module ended for being left running: their end is when it noticed, so neither is finished work.
goSteps["shanks."+stepID(at("08:00"), "a00008")] = `{"v":1,"id":"` + stepID(at("08:00"), "a00008") + `","tool":"check ask","node":"shanks","state":"failed","started":"2026-10-26T08:00:00Z","ended":"2026-10-26T11:59:00Z","reason":"the build module restarted while it ran"}`
goSteps["shanks."+stepID(at("09:00"), "a00009")] = `{"v":1,"id":"` + stepID(at("09:00"), "a00009") + `","tool":"delivery ask","node":"shanks","state":"failed","started":"2026-10-26T09:00:00Z","ended":"2026-10-26T11:58:00Z","reason":"past the step bound: its asker stopped waiting at 2026-10-26T09:30:00Z"}`
goSteps["shanks.notastepkey"] = `{}` // a key that is no step id is passed over
return UsageReader{
Now: func() time.Time { return usageNow },
Haves: states(map[string]string{"shanks.mesh-build-go": running, "shanks.mesh-build-node": running, "ace.mesh-build-go": running,
"g14.mesh-build-go": running, "shanks.node": `{}`, "shanks.mesh-delivery-view": running}),
Resolved: states(map[string]string{
"shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
"ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
"g14.mesh-build-go.capacity": `{"node":"g14","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
"shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`,
"ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`,
"g14.mesh-build-go.capacity": `{"node":"g14","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`,
"shanks.mesh-config.other": `{"value":"x"}`,
}),
Steps: map[string]StateRead{"go": states(goSteps), "node": states(map[string]string{}), "image": states(map[string]string{})},
@@ -118,8 +121,11 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) {
what string
got, want any
}{
{"nodes", len(a.Nodes), 3},
{"eligible", eligible, map[string]bool{"shanks": true, "ace": false, "g14": false}},
{"nodes, novox named by the default capacity", len(a.Nodes), 4},
{"eligible", eligible, map[string]bool{"shanks": true, "ace": false, "g14": false, "novox": false}},
{"novox why not", seatOf(t, a, "novox", "node-build-go").WhyNot, "mesh-build-go is not on novox"},
{"novox capacity, from the module's value resolved elsewhere", seatOf(t, a, "novox", "node-build-go").CapacityFrom, "mesh-config, resolved for mesh-build-go on ace (mesh-build-go is not assigned to novox)"},
{"ended by restart or bound, apart", len(sg.EndedStale), 2},
{"ace why not", seatOf(t, a, "ace", "node-build-go").WhyNot, "paused (node-build-go.resume takes asks again)"},
{"g14 why not", seatOf(t, a, "g14", "node-build-go").WhyNot, "its capacity on g14 is 0"},
{"shanks capacity", sg.Capacity, 2},
@@ -142,8 +148,8 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) {
{"day at capacity", sg.LastDay.AtCapacitySecs, int64(300)},
{"backlog", a.Backlog, []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}},
{"tries running", len(a.TriesRunning), 1},
{"said", a.Nodes[0].Said, []string{"node-build-go: not eligible, paused (node-build-go.resume takes asks again); 0 of capacity 2 running; last hour 0 finished, median 0 s, at capacity 0% of it; last day 0 finished, median 0 s, at capacity 0.0% of it"}},
{"said shanks", a.Nodes[2].Said[0], "node-build-go: eligible; 2 of capacity 2 running (check 1, delivery 1), 1 left running; last hour 3 finished, median 360 s, at capacity 8% of it; last day 4 finished, median 360 s, at capacity 0.3% of it"},
{"said", a.Nodes[0].Said[:1], []string{"node-build-go: not eligible, paused (node-build-go.resume takes asks again); 0 of capacity 2 running; last hour 0 finished, median 0 s, at capacity 0% of it; last day 0 finished, median 0 s, at capacity 0.0% of it"}},
{"said shanks", a.Nodes[3].Said[0], "node-build-go: eligible; 2 of capacity 2 running (check 1, delivery 1), 1 left running; last hour 3 finished, median 360 s, at capacity 8% of it; last day 4 finished, median 360 s, at capacity 0.3% of it"},
{"cpu says where it is", strings.Contains(a.Nodes[0].CPU, "/disk-load.cpu_top"), true},
} {
if fmt.Sprint(c.got) != fmt.Sprint(c.want) {
@@ -157,12 +163,19 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) {
// Nothing readable: each state is named in a note, the backlog's refusal too, and nothing is said held.
none := UsageReader{Now: func() time.Time { return usageNow }, Backlog: func() ([]BacklogRow, error) { return nil, errors.New("no socket") }}.Usage()
notes := strings.Join(none.Notes, " | ")
for _, want := range []string{"have is not readable", "resolved values are not readable", "steps of mesh-build-go are not readable", "tries are not readable", "did not answer: no socket"} {
for _, want := range []string{"have is not readable", "capacity unknown: mesh-config's resolved values are not readable", "steps of mesh-build-go are not readable", "tries are not readable", "did not answer: no socket"} {
if !strings.Contains(notes, want) {
t.Errorf("the notes %q do not say %q", notes, want)
}
}
if len(none.Nodes) != 0 || none.Backlog == nil {
t.Errorf("nothing readable answered nodes %v, backlog %v", none.Nodes, none.Backlog)
if none.Backlog == nil {
t.Errorf("nothing readable answered no backlog list")
}
for _, n := range none.Nodes { // the default's nodes, none said eligible on a capacity nobody can read
for _, st := range n.Seats {
if st.Eligible || !strings.HasPrefix(st.CapacityFrom, "capacity unknown: the puller uses 0") {
t.Errorf("with nothing readable %s on %s says eligible %v, capacity from %q", st.Seat, n.Node, st.Eligible, st.CapacityFrom)
}
}
}
}