From 166a918b43b173a2b7e6b1127dd0724e0f47debe Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 14:27:25 +0100 Subject: [PATCH] View: build_usage counts keys a build module ended for being left running apart, lists every node a capacity names, and says capacity unknown when mesh-config cannot be read Since mesh-sdk #90 a build module ends a left-running key as failed at the moment it notices; counted as finished, such a step put hours into the medians and the time at capacity. The view takes mesh-sdk's step bound and restart reason from the SDK it now pins. --- .../cmd/mesh-delivery-view-tools/main.go | 5 +- modules/mesh-delivery-view/go.mod | 2 +- modules/mesh-delivery-view/go.sum | 4 +- .../mesh-delivery-view/internal/view/usage.go | 91 +++++++++++++++---- .../internal/view/usage_test.go | 33 +++++-- 5 files changed, 103 insertions(+), 32 deletions(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 77d172e..c48e580 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -20,9 +20,10 @@ import ( func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { return []stdio.Tool{ {Name: view.BuildUsageTool, - Description: "How the build nodes are used, per node: each build seat it holds (node-build-go, -node, -image) with its capacity " + + Description: "How the build nodes are used, and whether a node is eligible to build, per node (every node holding a build module or named by a capacity): each build seat it holds (node-build-go, -node, -image) with its capacity " + "(mesh-config's resolved value, or the modules' default) and pause, and whether it is eligible to build and why not; the asks " + - "running now by kind (delivery, check, recheck, try) and the keys left running (past the step bound, or their module not running); " + + "running now by kind (delivery, check, recheck, try), the keys left running (past the step bound, or their module not running) and the ones " + + "their module ended for that (a restart, or past the step bound), counted apart and in no median or occupancy; " + "the steps started and finished in the last hour and the last day, failed ones, total and median time, by kind, and how long and " + "what share of the window the node ran its capacity of asks. Mesh-wide: the backlog of asks waiting and held on each build seat's " + "stream, and the tries running. Read from the bus's states (have, mesh-config, the build modules' steps and pauses, the checker's tries); " + diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 2aba3f5..ae85fd6 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 85d92c8..f754077 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81 h1:LhljEtt7FbNpVn67eOQbdsFDS8Qj6WPUF6l9IXr9mo4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025224357-095d41fa3e81/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2 h1:s1WcG7Ll1OkEwwvMKEabdWvPvTiFCHghlXiY1LoIJDA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125731-2aae6f5580f2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/usage.go b/modules/mesh-delivery-view/internal/view/usage.go index be23451..3216d28 100644 --- a/modules/mesh-delivery-view/internal/view/usage.go +++ b/modules/mesh-delivery-view/internal/view/usage.go @@ -27,9 +27,18 @@ import ( // BuildUsageTool is the view's tool of the build nodes' use. const BuildUsageTool = "build_usage" -// UsageStepBound is how long an ask step may say running before it is taken as left running: the checker's step bound -// (mesh-delivery-checker check.DefaultStepBound), after which no asker waits for it. -const UsageStepBound = 30 * time.Minute +// endedStale says why a step's end was written by its build module for a key left running (mesh-sdk buildstep's ending +// of stale steps): a restart's, or the step bound's; "" for a step that ran to its own end. Such an end is the moment the +// module noticed, not the step's, so it is counted apart and in neither the medians nor the time at capacity. +func endedStale(st delivery.Step) string { + if st.State != delivery.StepFailed { + return "" + } + if st.Reason == buildstep.RestartReason || strings.HasPrefix(st.Reason, "past the step bound") { + return st.Reason + } + return "" +} // The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker // put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity). @@ -96,9 +105,12 @@ type SeatUsage struct { WhyNot string `json:"why_not,omitempty"` Running RunningUsage `json:"running"` LeftRunning []StepUsage `json:"left_running"` - LastHour WindowUsage `json:"last_hour"` - LastDay WindowUsage `json:"last_day"` - Steps []StepUsage `json:"running_steps"` + // EndedStale are the steps of the last day whose end their build module wrote for a key left running (a restart, or + // past the step bound): counted here alone, never as finished. + EndedStale []StepUsage `json:"ended_stale"` + LastHour WindowUsage `json:"last_hour"` + LastDay WindowUsage `json:"last_day"` + Steps []StepUsage `json:"running_steps"` } // usageStep is a step read, with the kind it is counted under. @@ -187,7 +199,7 @@ func (u UsageReader) Usage() UsageAnswer { } name, _ := delivery.BuildSeat(lang) module, _ := delivery.BuildModuleOf(lang) - s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, Steps: []StepUsage{}} + s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, EndedStale: []StepUsage{}, Steps: []StepUsage{}} seats[node][lang] = s return s } @@ -231,10 +243,11 @@ func (u UsageReader) Usage() UsageAnswer { // Capacity: mesh-config's resolved value of the module's `capacity` on each node. resolved := map[string]string{} // . -> setting + unknown := "" // why no capacity can be said, when mesh-config's values cannot be read if u.Resolved.Keys == nil || u.Resolved.Get == nil { - note("mesh-config's resolved values are not readable here, so each capacity is the modules' default %q", buildstep.DefaultCapacities) + unknown = "mesh-config's resolved values are not readable here" } else if keys, err := u.Resolved.Keys(); err != nil { - note("mesh-config's resolved values cannot be listed, so each capacity is the modules' default %q: %v", buildstep.DefaultCapacities, err) + unknown = "mesh-config's resolved values cannot be listed: " + err.Error() } else { var want []string for _, k := range keys { @@ -244,7 +257,7 @@ func (u UsageReader) Usage() UsageAnswer { } raws, err := u.Resolved.values(want) if err != nil { - note("mesh-config's resolved capacities cannot be read: %v", err) + unknown = "mesh-config's resolved capacities cannot be read: " + err.Error() } for i, k := range want { var r struct { @@ -256,9 +269,37 @@ func (u UsageReader) Usage() UsageAnswer { } } } + if unknown != "" { + note("capacity unknown: %s, and a puller whose capacity cannot be read uses 0", unknown) + } + // Every node a capacity names is listed, so "is this node eligible" is answered for any node: the nodes each module's + // resolved value names (as resolved on the first node in name order), or the modules' default when mesh-config + // resolves none for the module. + keys := make([]string, 0, len(resolved)) for key := range resolved { + keys = append(keys, key) + } + sort.Strings(keys) + named := map[string]string{} // language -> the module's setting, as resolved on some node + namedOn := map[string]string{} // language -> that node + for _, key := range keys { node, module, _ := strings.Cut(key, ".") - seat(node, strings.TrimPrefix(module, "mesh-build-")) + lang := strings.TrimPrefix(module, "mesh-build-") + seat(node, lang) + if _, ok := named[lang]; !ok { + named[lang], namedOn[lang] = resolved[key], node + } + } + for _, lang := range delivery.AskLanguages { + setting, ok := named[lang] + if !ok { + setting = buildstep.DefaultCapacities + } + if caps, err := buildrun.ParseCapacity(setting); err == nil { + for node := range caps { + seat(node, lang) + } + } } // The steps of each build module. steps := map[string]map[string][]usageStep{} // node -> language -> steps @@ -286,7 +327,7 @@ func (u UsageReader) Usage() UsageAnswer { if err != nil { continue } - if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+UsageStepBound { + if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+delivery.StepBound { recent = append(recent, keyAt{k, at}) } } @@ -362,8 +403,18 @@ func (u UsageReader) Usage() UsageAnswer { if s == nil { continue } - if setting, ok := resolved[node+"."+s.Module]; ok { - s.CapacityFrom = "mesh-config, resolved for " + s.Module + " on " + node + setting, ok := resolved[node+"."+s.Module] + from := "mesh-config, resolved for " + s.Module + " on " + node + if _, has := named[lang]; !ok && has { + from = "mesh-config, resolved for " + s.Module + " on " + namedOn[lang] + " (" + s.Module + " is not assigned to " + node + ")" + // mesh-config resolves the module's value only where it is assigned; elsewhere its capacity is what that + // value names for the node. + setting, ok = named[lang], true + } + if unknown != "" { + s.CapacityFrom = "capacity unknown: the puller uses 0 (" + unknown + ")" + } else if ok { + s.CapacityFrom = from if caps, err := buildrun.ParseCapacity(setting); err != nil { s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err) } else { @@ -380,6 +431,8 @@ func (u UsageReader) Usage() UsageAnswer { s.WhyNot = s.Module + " is not on " + node case !s.Holds: s.WhyNot = s.Module + " on " + node + " is " + s.Phase + case unknown != "": + s.WhyNot = "capacity unknown: the puller uses 0" case s.Capacity == 0: s.WhyNot = "its capacity on " + node + " is 0" case s.Paused: @@ -455,13 +508,17 @@ func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, .. return asks } -// fill counts a seat's steps: running now, left running (started past UsageStepBound, or its module no longer running on +// fill counts a seat's steps: running now, left running (started past delivery.StepBound, or its module no longer running on // the node: no step behind the key runs any more), and the two windows. func (s *SeatUsage) fill(steps []usageStep, now time.Time) { sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) }) var live []usageStep // ended, or running and not left: what the windows count for _, us := range steps { st := us.step + if why := endedStale(st); why != "" { + s.EndedStale = append(s.EndedStale, StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Why: why}) + continue + } if st.State != delivery.StepRunning { live = append(live, us) continue @@ -470,8 +527,8 @@ func (s *SeatUsage) fill(steps []usageStep, now time.Time) { switch { case !s.Holds: line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs" - case now.Sub(st.Started) > UsageStepBound: - line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), UsageStepBound) + case now.Sub(st.Started) > delivery.StepBound: + line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), delivery.StepBound) } if line.Why != "" { s.LeftRunning = append(s.LeftRunning, line) diff --git a/modules/mesh-delivery-view/internal/view/usage_test.go b/modules/mesh-delivery-view/internal/view/usage_test.go index e740880..24d43d2 100644 --- a/modules/mesh-delivery-view/internal/view/usage_test.go +++ b/modules/mesh-delivery-view/internal/view/usage_test.go @@ -65,15 +65,18 @@ func usageFixture() UsageReader { add("shanks", stepID(at("06:00"), "a00005"), "recheck ask", "passed", at("06:00"), at("06:10")) add("shanks", stepID(at("10:00"), "a00006"), "check ask", "running", at("10:00"), time.Time{}) add("shanks", stepID(at("11:45"), "a00007"), "build_go_test", "passed", at("11:45"), at("11:46")) + // Two keys a build module ended for being left running: their end is when it noticed, so neither is finished work. + goSteps["shanks."+stepID(at("08:00"), "a00008")] = `{"v":1,"id":"` + stepID(at("08:00"), "a00008") + `","tool":"check ask","node":"shanks","state":"failed","started":"2026-10-26T08:00:00Z","ended":"2026-10-26T11:59:00Z","reason":"the build module restarted while it ran"}` + goSteps["shanks."+stepID(at("09:00"), "a00009")] = `{"v":1,"id":"` + stepID(at("09:00"), "a00009") + `","tool":"delivery ask","node":"shanks","state":"failed","started":"2026-10-26T09:00:00Z","ended":"2026-10-26T11:58:00Z","reason":"past the step bound: its asker stopped waiting at 2026-10-26T09:30:00Z"}` goSteps["shanks.notastepkey"] = `{}` // a key that is no step id is passed over return UsageReader{ Now: func() time.Time { return usageNow }, Haves: states(map[string]string{"shanks.mesh-build-go": running, "shanks.mesh-build-node": running, "ace.mesh-build-go": running, "g14.mesh-build-go": running, "shanks.node": `{}`, "shanks.mesh-delivery-view": running}), Resolved: states(map[string]string{ - "shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`, - "ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`, - "g14.mesh-build-go.capacity": `{"node":"g14","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`, + "shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`, + "ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`, + "g14.mesh-build-go.capacity": `{"node":"g14","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0,novox=0","source":"mesh"}`, "shanks.mesh-config.other": `{"value":"x"}`, }), Steps: map[string]StateRead{"go": states(goSteps), "node": states(map[string]string{}), "image": states(map[string]string{})}, @@ -118,8 +121,11 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) { what string got, want any }{ - {"nodes", len(a.Nodes), 3}, - {"eligible", eligible, map[string]bool{"shanks": true, "ace": false, "g14": false}}, + {"nodes, novox named by the default capacity", len(a.Nodes), 4}, + {"eligible", eligible, map[string]bool{"shanks": true, "ace": false, "g14": false, "novox": false}}, + {"novox why not", seatOf(t, a, "novox", "node-build-go").WhyNot, "mesh-build-go is not on novox"}, + {"novox capacity, from the module's value resolved elsewhere", seatOf(t, a, "novox", "node-build-go").CapacityFrom, "mesh-config, resolved for mesh-build-go on ace (mesh-build-go is not assigned to novox)"}, + {"ended by restart or bound, apart", len(sg.EndedStale), 2}, {"ace why not", seatOf(t, a, "ace", "node-build-go").WhyNot, "paused (node-build-go.resume takes asks again)"}, {"g14 why not", seatOf(t, a, "g14", "node-build-go").WhyNot, "its capacity on g14 is 0"}, {"shanks capacity", sg.Capacity, 2}, @@ -142,8 +148,8 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) { {"day at capacity", sg.LastDay.AtCapacitySecs, int64(300)}, {"backlog", a.Backlog, []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}}, {"tries running", len(a.TriesRunning), 1}, - {"said", a.Nodes[0].Said, []string{"node-build-go: not eligible, paused (node-build-go.resume takes asks again); 0 of capacity 2 running; last hour 0 finished, median 0 s, at capacity 0% of it; last day 0 finished, median 0 s, at capacity 0.0% of it"}}, - {"said shanks", a.Nodes[2].Said[0], "node-build-go: eligible; 2 of capacity 2 running (check 1, delivery 1), 1 left running; last hour 3 finished, median 360 s, at capacity 8% of it; last day 4 finished, median 360 s, at capacity 0.3% of it"}, + {"said", a.Nodes[0].Said[:1], []string{"node-build-go: not eligible, paused (node-build-go.resume takes asks again); 0 of capacity 2 running; last hour 0 finished, median 0 s, at capacity 0% of it; last day 0 finished, median 0 s, at capacity 0.0% of it"}}, + {"said shanks", a.Nodes[3].Said[0], "node-build-go: eligible; 2 of capacity 2 running (check 1, delivery 1), 1 left running; last hour 3 finished, median 360 s, at capacity 8% of it; last day 4 finished, median 360 s, at capacity 0.3% of it"}, {"cpu says where it is", strings.Contains(a.Nodes[0].CPU, "/disk-load.cpu_top"), true}, } { if fmt.Sprint(c.got) != fmt.Sprint(c.want) { @@ -157,12 +163,19 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) { // Nothing readable: each state is named in a note, the backlog's refusal too, and nothing is said held. none := UsageReader{Now: func() time.Time { return usageNow }, Backlog: func() ([]BacklogRow, error) { return nil, errors.New("no socket") }}.Usage() notes := strings.Join(none.Notes, " | ") - for _, want := range []string{"have is not readable", "resolved values are not readable", "steps of mesh-build-go are not readable", "tries are not readable", "did not answer: no socket"} { + for _, want := range []string{"have is not readable", "capacity unknown: mesh-config's resolved values are not readable", "steps of mesh-build-go are not readable", "tries are not readable", "did not answer: no socket"} { if !strings.Contains(notes, want) { t.Errorf("the notes %q do not say %q", notes, want) } } - if len(none.Nodes) != 0 || none.Backlog == nil { - t.Errorf("nothing readable answered nodes %v, backlog %v", none.Nodes, none.Backlog) + if none.Backlog == nil { + t.Errorf("nothing readable answered no backlog list") + } + for _, n := range none.Nodes { // the default's nodes, none said eligible on a capacity nobody can read + for _, st := range n.Seats { + if st.Eligible || !strings.HasPrefix(st.CapacityFrom, "capacity unknown: the puller uses 0") { + t.Errorf("with nothing readable %s on %s says eligible %v, capacity from %q", st.Seat, n.Node, st.Eligible, st.CapacityFrom) + } + } } }