diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index edcd8c615..d75e70a13 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -449,15 +449,22 @@ func seatStateRead(holder func(seat string) (string, bool), read func(name strin Get: func(key string) (json.RawMessage, error) { return at().Get(key) }} } +// currentVerb is the build-step part's verb current, which every build seat serves (design 47 H2 §801). +const currentVerb = "current" + // usageReaderOf is the usage tools' reader: the haves and the config's resolved values read as they are, a -// build seat's steps and pauses in the state of the seat's holder (ofHolder), and the backlog the view program's socket answers; -// main and its test wire it the one way. -func usageReaderOf(haves view.StateRead, seats *buildrun.BuildSeats, holder func(seat string) (string, bool), state func(name string) view.StateRead, backlog func() ([]view.BacklogRow, error)) view.UsageReader { +// build seat's steps and pauses in the state of the seat's holder (ofHolder), what each holder runs by its seat's verb current +// asked of its node (design 47 H2 §813 (9)), and the backlog the view program's socket answers; main and its test wire it the +// one way. +func usageReaderOf(haves view.StateRead, seats *buildrun.BuildSeats, holder func(seat string) (string, bool), state func(name string) view.StateRead, ask func(key string, body any) (json.RawMessage, error), backlog func() ([]view.BacklogRow, error)) view.UsageReader { return view.UsageReader{Haves: haves, Resolved: state("mesh-config.resolved"), Seats: seats, StepsOf: ofHolder(holder, state, view.StepsState), PausesOf: ofHolder(holder, state, buildstep.PausesState), - Backlog: backlog} + Current: func(seat, node string) (json.RawMessage, error) { + return ask("seat:"+seat+"."+currentVerb+"@"+node, map[string]any{}) + }, + Backlog: backlog} } func main() { @@ -498,7 +505,7 @@ func main() { } return nil }, holderOf(stdio.Ask)) - u := usageReaderOf(r.HavesState, seats, holderOf(stdio.Ask), func(name string) view.StateRead { return read(stdio.State(name)) }, backlogOf(socket())) + u := usageReaderOf(r.HavesState, seats, holderOf(stdio.Ask), func(name string) view.StateRead { return read(stdio.State(name)) }, stdio.Ask, backlogOf(socket())) // view_first_views and view_first_view_again: where the view's process stands with its first views of the indexer's and // the assigner's states (ADR 0615 §6), and a re-read by hand. tools := append(Tools(r, u), gate.Tools()...) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index 27041ab15..0eb83ccc7 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -680,7 +680,13 @@ func TestBuildSeatStatesAreReadInTheHoldersState(t *testing.T) { // values by their names (no checker's tries, design 47 H §764), the haves and the backlog as given. named = nil seats := &buildrun.BuildSeats{} - u := usageReaderOf(view.StateRead{}, seats, holder, read, func() ([]view.BacklogRow, error) { return []view.BacklogRow{{}}, nil }) + var asked string + ask := func(key string, body any) (json.RawMessage, error) { asked = key; return json.RawMessage(`{}`), nil } + u := usageReaderOf(view.StateRead{}, seats, holder, read, ask, func() ([]view.BacklogRow, error) { return []view.BacklogRow{{}}, nil }) + // What a holder runs is its seat's verb current asked of its node (design 47 H2 §813 (9)). + if _, err := u.Current("node-build-go", "g14"); err != nil || asked != "seat:node-build-go.current@g14" { + t.Errorf("the usage reader asks %q, %v; want seat:node-build-go.current@g14", asked, err) + } u.StepsOf("node-build-go") u.PausesOf("node-build-go") if want := []string{"mesh-config.resolved", "acme-node-build-go.steps", "acme-node-build-go.pauses"}; !slices.Equal(named, want) || u.Seats != seats { diff --git a/modules/mesh-delivery-view/internal/view/usage.go b/modules/mesh-delivery-view/internal/view/usage.go index 511cb779c..8f827dae0 100644 --- a/modules/mesh-delivery-view/internal/view/usage.go +++ b/modules/mesh-delivery-view/internal/view/usage.go @@ -6,7 +6,9 @@ package view // module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each // build module's steps state (ADR 0525 §6: an ask ends on .) the asks running now by kind, the ones left // running, and the steps started and finished in the last hour and day with their total and median time and how long the -// node was at its capacity. The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver), +// node was at its capacity. A build module's work is read from its seat's verb current on each node (design 47 H2 §813 (9), +// §801): the seat is held on a node whose holder answers it, whatever the node-engine's have says of its phase, and the asks +// running are the ones it lists. The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver), // asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the // bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it. @@ -16,6 +18,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -57,6 +60,8 @@ type UsageReader struct { // Resolved is mesh-config's resolved values (..); Haves the node-engine's have. Resolved StateRead Haves StateRead + // Current asks a build seat's verb current of one node's holder (design 47 H2 §801), its answer as the part says it. + Current func(seat, node string) (json.RawMessage, error) // Backlog is the seats' streams' waiting and held asks, as the view program reads them. Backlog func() ([]BacklogRow, error) Now func() time.Time @@ -86,7 +91,8 @@ type NodeUsage struct { type SeatUsage struct { Seat string `json:"seat"` Module string `json:"module"` - // Holds says the node runs the seat's module (its have says running); Phase is what its have says, or absent. + // Holds says the seat's holder on the node answers its verb current (design 47 H2 §813 (9)); Phase is what the + // node-engine's have says, or absent, and judges nothing. Holds bool `json:"holds"` Phase string `json:"phase,omitempty"` // Capacity is the asks it runs at once; CapacityFrom where that came from. @@ -119,6 +125,19 @@ type SeatUsage struct { Steps []StepUsage `json:"running_steps"` } +// currentAnswer is what a build seat's verb current answers (the build-step part, design 47 H2 §801), as far as the usage reads it. +type currentAnswer struct { + Node string `json:"node"` + Asks []currentAsk `json:"asks"` +} + +// currentAsk is one ask a holder runs now, whoever asked it. +type currentAsk struct { + ID string `json:"id"` + Kind string `json:"kind"` + Started time.Time `json:"started"` +} + // usageStep is a step read, with the kind it is counted under. type usageStep struct { step buildstep.Step @@ -252,7 +271,7 @@ func (u UsageReader) Usage() UsageAnswer { node, module, _ := strings.Cut(k, ".") bm := buildSeatOfModule(module) s := seat(node, bm.AskLanguage()) - s.Phase, s.Holds = string(h.Phase), h.Phase == nodeengineclient.PhaseRunning + s.Phase = string(h.Phase) } // Capacity: mesh-config's resolved value of the module's `capacity` on each node. @@ -424,6 +443,12 @@ func (u UsageReader) Usage() UsageAnswer { nodes = append(nodes, n) } sort.Strings(nodes) + + // What each holder runs: its seat's verb current on its node, every node and seat asked at once (design 47 H2 §813 (9)). + current, unanswered := u.currents(nodes, seats) + if u.Current == nil { + note("the build seats' verb current is not asked here, so no seat is said held") + } for _, node := range nodes { nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{}, CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"} @@ -463,7 +488,7 @@ func (u UsageReader) Usage() UsageAnswer { case !s.Holds && s.Phase == "": s.WhyNot = s.Module + " is not on " + node case !s.Holds: - s.WhyNot = s.Module + " on " + node + " is " + s.Phase + s.WhyNot = s.Seat + "'s current on " + node + " answers nothing (" + unanswered[node+"."+lang] + "); its have says " + s.Phase case s.Paused: s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)" case unknown != "": @@ -488,7 +513,7 @@ func (u UsageReader) Usage() UsageAnswer { case nu.Eligible == "": nu.Eligible = "not eligible" } - s.fill(steps[node][lang], now) + s.fill(steps[node][lang], current[node+"."+lang], now) nu.Said = append(nu.Said, s.said()) nu.Seats = append(nu.Seats, *s) } @@ -505,10 +530,70 @@ func (u UsageReader) Usage() UsageAnswer { return ans } -// fill counts a seat's steps: running now, left running (started past buildstep.StepBound, or its module no longer running on -// the node: no step behind the key runs any more), and the two windows. -func (s *SeatUsage) fill(steps []usageStep, now time.Time) { +// currents asks every seat on every node its verb current at once: the answers by ., and for each seat that +// answered nothing, why. +func (u UsageReader) currents(nodes []string, seats map[string]map[string]*SeatUsage) (map[string][]currentAsk, map[string]string) { + answers, unanswered := map[string][]currentAsk{}, map[string]string{} + if u.Current == nil { + for _, node := range nodes { + for lang := range seats[node] { + unanswered[node+"."+lang] = "not asked here" + } + } + return answers, unanswered + } + var mu sync.Mutex + var wg sync.WaitGroup + for _, node := range nodes { + for lang, s := range seats[node] { + if s.Seat == "" { + continue + } + wg.Add(1) + go func(node, lang string, s *SeatUsage) { + defer wg.Done() + raw, err := u.Current(s.Seat, node) + var a currentAnswer + if err == nil { + err = json.Unmarshal(raw, &a) + } + mu.Lock() + defer mu.Unlock() + switch { + case err != nil: + unanswered[node+"."+lang] = err.Error() + case a.Node != "" && a.Node != node: + unanswered[node+"."+lang] = "it answered for " + a.Node + default: + s.Holds = true + answers[node+"."+lang] = a.Asks + } + }(node, lang, s) + } + } + wg.Wait() + return answers, unanswered +} + +// fill counts a seat's work: the asks running now as its seat's current lists them (design 47 H2 §813 (9)), a step's key +// left running (started past buildstep.StepBound, or an ask its holder's current does not list, or a hand step whose holder +// answers nothing: no step behind the key runs any more), and the two windows. +func (s *SeatUsage) fill(steps []usageStep, asks []currentAsk, now time.Time) { sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) }) + sort.Slice(asks, func(i, j int) bool { return asks[i].Started.Before(asks[j].Started) }) + runs := map[string]bool{} + for _, a := range asks { + runs[a.ID] = true + tool := a.Kind + " ask" + for _, us := range steps { + if us.step.ID == a.ID { + tool = us.step.Tool + } + } + s.Steps = append(s.Steps, StepUsage{ID: a.ID, Kind: a.Kind, Tool: tool, Started: a.Started, Secs: int64(now.Sub(a.Started).Seconds())}) + s.Running.Total++ + s.Running.ByKind[a.Kind]++ + } var live []usageStep // ended, or running and not left: what the windows count for _, us := range steps { st := us.step @@ -522,21 +607,22 @@ func (s *SeatUsage) fill(steps []usageStep, now time.Time) { } line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())} switch { - case !s.Holds: - line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs" + case runs[st.ID]: case now.Sub(st.Started) > buildstep.StepBound: line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), buildstep.StepBound) + case !s.Holds: + line.Why = "its key says running, and " + s.Seat + "'s current on its node answers nothing, so no step behind it runs" + case us.kind != KindHand: + line.Why = "its key says running, and " + s.Seat + "'s current on its node does not list it, so no step behind it runs" } if line.Why != "" { s.LeftRunning = append(s.LeftRunning, line) continue } - s.Steps = append(s.Steps, line) - live = append(live, us) - if us.kind != KindHand { - s.Running.Total++ - s.Running.ByKind[us.kind]++ + if us.kind == KindHand { + s.Steps = append(s.Steps, line) } + live = append(live, us) } s.LastHour = window(live, now, time.Hour, s.Capacity) s.LastDay = window(live, now, 24*time.Hour, s.Capacity) diff --git a/modules/mesh-delivery-view/internal/view/usage_test.go b/modules/mesh-delivery-view/internal/view/usage_test.go index a82594271..cadaeb4c8 100644 --- a/modules/mesh-delivery-view/internal/view/usage_test.go +++ b/modules/mesh-delivery-view/internal/view/usage_test.go @@ -53,9 +53,12 @@ func step(node, id, tool, state string, started, ended time.Time) string { // usageFixture is three nodes, each capacity the node's integer on its layer of mesh-config (ADR 0567 §12): shanks builds // Go at capacity 2 (two asks running, one left running, a try's ask, a check, a recheck of the morning and a hand step // ended); ace holds the Go seat at capacity 2 and is paused, and its Node capacity is still of the old form; g14 holds the -// Go seat at capacity 0; the Node seat on shanks has no resolved capacity and takes the modules' default 0. +// Go seat at capacity 0, its have held; the Node seat on shanks has no resolved capacity and takes the modules' default 0, +// and its holder answers no current though its have says running. Each seat is judged held by its verb current (design 47 +// H2 §813 (9)). func usageFixture() UsageReader { running := `{"v":1,"phase":"running","health":"ok"}` + held := `{"v":1,"phase":"held","health":"ok"}` checkAsk := stepID(at("11:30"), "a00003") goSteps := map[string]string{} add := func(node, id, tool, state string, started, ended time.Time) { @@ -75,7 +78,7 @@ func usageFixture() UsageReader { return UsageReader{ Now: func() time.Time { return usageNow }, Haves: states(map[string]string{"shanks.mesh-build-go": running, "shanks.mesh-build-node": running, "ace.mesh-build-go": running, - "g14.mesh-build-go": running, "g14.mesh-build-node": running, "shanks.node": `{}`, "shanks.mesh-delivery-view": running}), + "g14.mesh-build-go": held, "g14.mesh-build-node": running, "shanks.node": `{}`, "shanks.mesh-delivery-view": running}), Resolved: states(map[string]string{ "shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":2,"source":"node"}`, "ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":2,"source":"node"}`, @@ -97,6 +100,19 @@ func usageFixture() UsageReader { } return states(nil) }, + Current: func(seat, node string) (json.RawMessage, error) { + answers := map[string]string{ + "node-build-go.shanks": `{"node":"shanks","takes":true,"capacity":2,"asks":[{"id":"` + stepID(at("11:55"), "a00002") + `","kind":"check","started":"2026-10-26T11:55:00Z","asker":"mesh-delivery-builder"},` + + `{"id":"` + stepID(at("11:50"), "a00001") + `","kind":"delivery","started":"2026-10-26T11:50:00Z","asker":"mesh-delivery-builder"}]}`, + "node-build-go.ace": `{"node":"ace","takes":false,"why-not":"paused","capacity":2,"asks":[]}`, + "node-build-go.g14": `{"node":"g14","takes":false,"why-not":"capacity 0","capacity":0,"asks":[]}`, + "node-build-node.g14": `{"node":"g14","takes":true,"capacity":0,"asks":[]}`, + } + if a, ok := answers[seat+"."+node]; ok { + return json.RawMessage(a), nil + } + return nil, errors.New("no responders") + }, Backlog: func() ([]BacklogRow, error) { return []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}, nil }, @@ -140,7 +156,8 @@ func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) { {"a capacity of the old form naming nodes is unreadable", seatOf(t, a, "ace", "node-build-node").CapacityFrom, `mesh-config's value for mesh-build-node on ace cannot be read (its resolved value "shanks=2,ace=2" is not a whole number of asks at once), so its puller uses 0`}, {"ended by restart or bound, apart", len(sg.EndedStale), 2}, {"ace why not", seatOf(t, a, "ace", "node-build-go").WhyNot, "paused (node-build-go.resume takes asks again)"}, - {"g14 why not", seatOf(t, a, "g14", "node-build-go").WhyNot, "its capacity on g14 is 0"}, + {"g14 why not: its have is held, its holder answers current (§813 (9))", seatOf(t, a, "g14", "node-build-go").WhyNot, "its capacity on g14 is 0"}, + {"a have running whose holder answers no current holds nothing (§813 (9))", seatOf(t, a, "shanks", "node-build-node").WhyNot, "node-build-node's current on shanks answers nothing (no responders); its have says running"}, {"shanks capacity", sg.Capacity, 2}, {"shanks node seat takes the default", seatOf(t, a, "shanks", "node-build-node").Capacity, 0}, {"running", sg.Running.Total, 2}, diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index e473d9a00..17e3f7a24 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -174,16 +174,17 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 {Verb: "mesh-delivery-indexer.indexer_read_packages"}, {Verb: "mesh-delivery-indexer.indexer_read_predefined"}, {Verb: "mesh-delivery-indexer.indexer_read_taken"}, {Verb: "mesh-delivery-teller.teller_read_answers"}, // A-175: the teller's answers by their views family {Verb: "seat:mesh-catalog.revision", Fields: []string{"none-yet"}}, - {Verb: "seat:mesh-catalog.registrations"}, // §614: the watched branches of the registrations, then registration-changed - {Verb: "seat:mesh-catalog.entry", Fields: []string{"manifest-digest"}}, // §558: the module's value is the entry's manifest-digest (§506) - {Verb: "seat:mesh-delivery.source", Fields: []string{"branches", "call", "repositories", "revision"}}, // §507, §614, A-173: the branches before the first registrations, and each watched branch's source with its call, no group (design 47 H §737, §764) // ADR 0663: the modules are the catalog half's, read through the seat's verb revision, never the indexer's - {Verb: "seat:mesh-catalog.seat-definitions"}, // design 47 G A-82, §204: the build seats are read from the seat list - {Verb: "seat:mesh-broker.show"}, // A-274: the binding of its finding-events - {Verb: "seat:mesh-hand-acts.hand-act"}, // the join: its first view asked again by hand is a hand act (views.StatusTools) - {Verb: "seat:mesh-module-assigner.assignments"}, // A-82: the holder of each build seat - {Verb: "seat:mesh-wants.revision"}, // §512, A-242: view_tiers compares kept assignment revisions - {Verb: "seat:mesh-wants.difference"}, // §512, A-242: the tiers and each change's after are the assigner's - {Verb: "seat:node-engine.manifest-reads"}}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2) + {Verb: "seat:mesh-catalog.registrations"}, // §614: the watched branches of the registrations, then registration-changed + {Verb: "seat:mesh-catalog.entry", Fields: []string{"manifest-digest"}}, // §558: the module's value is the entry's manifest-digest (§506) + {Verb: "seat:mesh-delivery.source", Fields: []string{"branches", "call", "repositories", "revision"}}, // §507, §614, A-173: the branches before the first registrations, and each watched branch's source with its call, no group (design 47 H §737, §764) // ADR 0663: the modules are the catalog half's, read through the seat's verb revision, never the indexer's + {Verb: "seat:mesh-catalog.seat-definitions"}, // design 47 G A-82, §204: the build seats are read from the seat list + {Verb: "seat:mesh-broker.show"}, // A-274: the binding of its finding-events + {Verb: "seat:mesh-hand-acts.hand-act"}, // the join: its first view asked again by hand is a hand act (views.StatusTools) + {Verb: "seat:mesh-module-assigner.assignments"}, // A-82: the holder of each build seat + {Verb: "seat:mesh-wants.revision"}, // §512, A-242: view_tiers compares kept assignment revisions + {Verb: "seat:mesh-wants.difference"}, // §512, A-242: the tiers and each change's after are the assigner's + {Verb: "seat:node-build-go.current"}, {Verb: "seat:node-build-image.current"}, {Verb: "seat:node-build-node.current"}, // design 47 H2 §813 (9): build_usage reads each holder's work by its seat's verb current + {Verb: "seat:node-engine.manifest-reads"}}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2) t.Errorf("invokes = %v, want %v", m.Invokes, want) } // design 47 G §387: the keeper's ask for the findings is heard, the finding stream declared (§383), and §368: version is MAJOR.MINOR. diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index b2deb1163..c8b859e87 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -119,6 +119,9 @@ "seat:mesh-module-assigner.assignments", "seat:mesh-wants.revision", "seat:mesh-wants.difference", + "seat:node-build-go.current", + "seat:node-build-image.current", + "seat:node-build-node.current", "seat:node-engine.manifest-reads" ], "reads": [