View: build_usage answers how the build nodes are used in one call; the checker's tries name their asks
The operator's question (how much are the build nodes used, is ace eligible) took six calls over four tools. build_usage reads it from the bus's states: seats held (have), capacity (mesh-config resolved), pauses, the build modules' steps by kind with left-running keys, the last hour and day with total and median time and time at capacity, the running tries, and the backlog the view program reads from the seats' streams. A try now names the asks it put, and the checker announces its tries readable, so the view tells a try's asks from a check's.
This commit is contained in:
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
|
||||
if !equal(manifest, announced) {
|
||||
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
||||
}
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
|
||||
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage).
|
||||
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) {
|
||||
t.Errorf("readable %v, want %v", readable, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,14 +112,15 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view.
|
||||
var keptStates = []announce.State{
|
||||
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
||||
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||
{Name: "kept", Readable: true},
|
||||
{Name: "owed", Readable: true},
|
||||
{Name: "previews", Readable: true},
|
||||
{Name: "tries"},
|
||||
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage).
|
||||
{Name: "tries", Readable: true},
|
||||
{Name: "verdicts", Readable: true},
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,8 @@ type Case struct {
|
||||
|
||||
notesMu sync.Mutex
|
||||
notes []string
|
||||
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
|
||||
asks []string
|
||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||
Progress *Progress
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
|
||||
@@ -59,6 +61,20 @@ type Case struct {
|
||||
Composition *delivery.Composition
|
||||
}
|
||||
|
||||
// Asked records the id of an ask the check put.
|
||||
func (c *Case) Asked(id string) {
|
||||
c.notesMu.Lock()
|
||||
c.asks = append(c.asks, id)
|
||||
c.notesMu.Unlock()
|
||||
}
|
||||
|
||||
// Asks are the ids of the asks the check put, in order.
|
||||
func (c *Case) Asks() []string {
|
||||
c.notesMu.Lock()
|
||||
defer c.notesMu.Unlock()
|
||||
return append([]string(nil), c.asks...)
|
||||
}
|
||||
|
||||
// Progress is the step a check is at, written by the check and read when its bound passes.
|
||||
type Progress struct {
|
||||
mu sync.Mutex
|
||||
|
||||
@@ -423,6 +423,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
what += " for a node with " + strings.Join(needs, ", ")
|
||||
}
|
||||
st.c.Progress.Set(what, "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
|
||||
@@ -75,9 +75,12 @@ type Try struct {
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
|
||||
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
|
||||
Asks []string `json:"asks,omitempty"`
|
||||
Asked time.Time `json:"asked"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// Over says whether the try came to its end.
|
||||
@@ -349,7 +352,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
v := out.Verdict
|
||||
t.Ended = tr.now().UTC()
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
t.Levels, t.Machines = v.Levels, v.Machines
|
||||
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
|
||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||
for _, n := range cs.Notes() {
|
||||
t.Said += " | " + n
|
||||
|
||||
@@ -97,6 +97,12 @@ func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { /
|
||||
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
|
||||
t.Fatalf("try = %+v", got)
|
||||
}
|
||||
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
|
||||
if len(got.Asks) != 1 {
|
||||
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
|
||||
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
|
||||
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
|
||||
}
|
||||
fetch := b.steps.tool("fetch")
|
||||
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
|
||||
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
|
||||
|
||||
@@ -11,12 +11,24 @@ import (
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Tools are the read tools over a modules state.
|
||||
func Tools(r view.Reader) []stdio.Tool {
|
||||
// Tools are the read tools over a modules state, and build_usage over the build modules' states.
|
||||
func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: view.BuildUsageTool,
|
||||
Description: "How the build nodes are used, per node: each build seat it holds (node-build-go, -node, -image) with its capacity " +
|
||||
"(mesh-config's resolved value, or the modules' default) and pause, and whether it is eligible to build and why not; the asks " +
|
||||
"running now by kind (delivery, check, recheck, try) and the keys left running (past the step bound, or their module not running); " +
|
||||
"the steps started and finished in the last hour and the last day, failed ones, total and median time, by kind, and how long and " +
|
||||
"what share of the window the node ran its capacity of asks. Mesh-wide: the backlog of asks waiting and held on each build seat's " +
|
||||
"stream, and the tries running. Read from the bus's states (have, mesh-config, the build modules' steps and pauses, the checker's tries); " +
|
||||
"the backlog from the view program's reading of the streams; CPU is not on the bus and the answer names the per-node tool. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return u.Usage(), nil }},
|
||||
{Name: view.ModuleReadTool,
|
||||
Description: "What the indexer stored for one module under modules.<module>, as stored: its fingerprint (fp, none), the " +
|
||||
"new-form fingerprint (fp2) or why it has none (none2), the inputs it was computed from (inputs2), the module's " +
|
||||
@@ -131,6 +143,24 @@ func nodeModuleOf(args map[string]any) (string, string) {
|
||||
return strings.TrimSpace(node), strings.TrimSpace(module)
|
||||
}
|
||||
|
||||
// socket is the view program's socket, as the program reads it (MESH_DELIVERY_SOCKET, else its default).
|
||||
func socket() string {
|
||||
if s := strings.TrimSpace(os.Getenv("MESH_DELIVERY_SOCKET")); s != "" {
|
||||
return s
|
||||
}
|
||||
return "/var/lib/mesh-delivery-view-cli/cli.sock"
|
||||
}
|
||||
|
||||
// backlogOf asks the view program on its socket for the backlog it reads from the seats' streams' workers: the tool
|
||||
// runtime grants a bundle states, and the stream and worker information is granted to the view program's own account
|
||||
// (mesh-sdk buscompose.BuildBacklogReads).
|
||||
func backlogOf(path string) func() ([]view.BacklogRow, error) {
|
||||
return func() ([]view.BacklogRow, error) {
|
||||
s, err := view.Ask(path)
|
||||
return s.Backlog, err
|
||||
}
|
||||
}
|
||||
|
||||
func read(state stdio.KeptState) view.StateRead {
|
||||
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
e, err := state.Get(key)
|
||||
@@ -150,7 +180,14 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
|
||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")),
|
||||
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
u.Steps[lang] = read(stdio.State(module + "." + delivery.StepsState))
|
||||
u.Pauses[lang] = read(stdio.State(module + "." + delivery.PausesState))
|
||||
}
|
||||
if err := stdio.Serve("", Tools(r, u)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ func (f fake) reader() view.Reader {
|
||||
|
||||
func run(t *testing.T, r view.Reader, name string, args map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tl := range Tools(r) {
|
||||
for _, tl := range Tools(r, view.UsageReader{}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run(args)
|
||||
}
|
||||
@@ -64,11 +64,7 @@ func TestModuleToolAnswersTheStoredValue(t *testing.T) {
|
||||
if err := json.Unmarshal(out.(json.RawMessage), &m); err != nil || m["fp"] != "a1" || m["fp2"] != "b2" || m["path"] != "." || m["inputs2"] == nil || m["toolchains"] == nil {
|
||||
t.Errorf("stored fields are not all there: %v %v", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A module with no key, and a name that is no key, are said so.
|
||||
func TestModuleToolRefusesWhatIsNotThere(t *testing.T) {
|
||||
r := fake{"a": withFP2}.reader()
|
||||
// A module with no key, and a name that is no key, are said so.
|
||||
if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no module gone") {
|
||||
t.Errorf("a missing module: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,592 @@
|
||||
package view
|
||||
|
||||
// The build nodes' use, in one answer (the operator, 2026-10-26: "how much are our build agents being used, and is ace
|
||||
// eligible for building?" took six calls over four tools). Read from the bus's state alone: which build seats each node
|
||||
// holds (the node-engine's have of mesh-build-<language>), its capacity for each (mesh-config's resolved value of the
|
||||
// module's setting `capacity` on that node, novox/hq ADR 0525 §5) and pause (the module's pauses state), and from each
|
||||
// build module's steps state (ADR 0525 §6: an ask ends on <node>.<ask id>) the asks running now by kind, the ones left
|
||||
// running, and the steps started and finished in the last hour and day with their total and median time and how long the
|
||||
// node was at its capacity. A try's asks are told from a candidate's by the checker's tries state (each try names the
|
||||
// asks it put). The backlog of the seats' streams is the view program's reading of the streams' workers (BacklogOver),
|
||||
// asked over the view's own socket: the tool runtime grants a bundle states, not stream information. CPU is not on the
|
||||
// bus: the disk-load module keeps no state, so the answer names the per-node tool that reads it.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// BuildUsageTool is the view's tool of the build nodes' use.
|
||||
const BuildUsageTool = "build_usage"
|
||||
|
||||
// UsageStepBound is how long an ask step may say running before it is taken as left running: the checker's step bound
|
||||
// (mesh-delivery-checker check.DefaultStepBound), after which no asker waits for it.
|
||||
const UsageStepBound = 30 * time.Minute
|
||||
|
||||
// The kinds a step is counted under: an ask's kind (delivery, check, recheck), a try (a check ask a try of the checker
|
||||
// put), or hand (a build tool called directly, outside the seat's asks, which takes no slot of the capacity).
|
||||
const (
|
||||
KindTry = "try"
|
||||
KindHand = "hand"
|
||||
)
|
||||
|
||||
// UsageReader reads what build_usage answers from.
|
||||
type UsageReader struct {
|
||||
// Steps and Pauses are each build module's states by language (go, node, image).
|
||||
Steps map[string]StateRead
|
||||
Pauses map[string]StateRead
|
||||
// Resolved is mesh-config's resolved values (<node>.<module>.<key>); Haves the node-engine's have.
|
||||
Resolved StateRead
|
||||
Haves StateRead
|
||||
// Tries is the checker's tries (try-<unix ns>), each naming the asks it put.
|
||||
Tries StateRead
|
||||
// Backlog is the seats' streams' waiting and held asks, as the view program reads them.
|
||||
Backlog func() ([]BacklogRow, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// UsageAnswer is build_usage's answer.
|
||||
type UsageAnswer struct {
|
||||
At time.Time `json:"at"`
|
||||
Nodes []NodeUsage `json:"nodes"`
|
||||
Backlog []BacklogRow `json:"backlog"`
|
||||
// TriesRunning are the tries the checker runs now; their asks are counted under check until the try ends and names them.
|
||||
TriesRunning []TryRunning `json:"tries_running"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// TryRunning is one try the checker runs now.
|
||||
type TryRunning struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Started time.Time `json:"started,omitzero"`
|
||||
}
|
||||
|
||||
// NodeUsage is one node: each build seat it holds or is set to build for, whether it is eligible to build, and its CPU.
|
||||
type NodeUsage struct {
|
||||
Node string `json:"node"`
|
||||
Eligible bool `json:"eligible"`
|
||||
// Said is one line per seat, the answer in words.
|
||||
Said []string `json:"said"`
|
||||
Seats []SeatUsage `json:"seats"`
|
||||
CPU string `json:"cpu"`
|
||||
}
|
||||
|
||||
// SeatUsage is one build seat on one node.
|
||||
type SeatUsage struct {
|
||||
Seat string `json:"seat"`
|
||||
Module string `json:"module"`
|
||||
// Holds says the node runs the seat's module (its have says running); Phase is what its have says, or absent.
|
||||
Holds bool `json:"holds"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
// Capacity is the asks it runs at once; CapacityFrom where that came from.
|
||||
Capacity int `json:"capacity"`
|
||||
CapacityFrom string `json:"capacity_from"`
|
||||
Paused bool `json:"paused"`
|
||||
// Eligible says it takes asks: it holds the seat, has a capacity above 0 and is not paused; WhyNot says why not.
|
||||
Eligible bool `json:"eligible"`
|
||||
WhyNot string `json:"why_not,omitempty"`
|
||||
Running RunningUsage `json:"running"`
|
||||
LeftRunning []StepUsage `json:"left_running"`
|
||||
LastHour WindowUsage `json:"last_hour"`
|
||||
LastDay WindowUsage `json:"last_day"`
|
||||
Steps []StepUsage `json:"running_steps"`
|
||||
}
|
||||
|
||||
// usageStep is a step read, with the kind it is counted under.
|
||||
type usageStep struct {
|
||||
step delivery.Step
|
||||
kind string
|
||||
}
|
||||
|
||||
// RunningUsage is the asks running now: how many in all and by kind, against the capacity.
|
||||
type RunningUsage struct {
|
||||
Total int `json:"total"`
|
||||
ByKind map[string]int `json:"by_kind"`
|
||||
}
|
||||
|
||||
// StepUsage is one step running now, or one left running and why.
|
||||
type StepUsage struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Tool string `json:"tool"`
|
||||
Started time.Time `json:"started"`
|
||||
Secs int64 `json:"secs"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// WindowUsage is the steps of one window: started and finished in it, the finished ones' total and median time, by kind,
|
||||
// and how long the node ran its capacity of asks (a hand step takes no slot).
|
||||
type WindowUsage struct {
|
||||
Started int `json:"started"`
|
||||
Finished int `json:"finished"`
|
||||
Failed int `json:"failed"`
|
||||
TotalSecs int64 `json:"total_secs"`
|
||||
MedianSecs int64 `json:"median_secs"`
|
||||
ByKind map[string]KindWindow `json:"by_kind"`
|
||||
AtCapacitySecs int64 `json:"at_capacity_secs"`
|
||||
AtCapacityShare float64 `json:"at_capacity_share"`
|
||||
}
|
||||
|
||||
// KindWindow is one kind's finished steps in a window.
|
||||
type KindWindow struct {
|
||||
Finished int `json:"finished"`
|
||||
TotalSecs int64 `json:"total_secs"`
|
||||
MedianSecs int64 `json:"median_secs"`
|
||||
}
|
||||
|
||||
// stepIDTime is when a step id was made: its base-36 milliseconds before the six hex digits (buildstep.NewID).
|
||||
func stepIDTime(id string) (time.Time, bool) {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), true
|
||||
}
|
||||
|
||||
// stepKind is the kind a step is counted under.
|
||||
func stepKind(s delivery.Step, tries map[string]bool) string {
|
||||
kind, ask := strings.CutSuffix(s.Tool, " ask")
|
||||
if !ask {
|
||||
return KindHand
|
||||
}
|
||||
if kind == delivery.AskCheck && tries[s.ID] {
|
||||
return KindTry
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
// Usage reads every state and answers. A state it cannot read is said in a note, never left out silently.
|
||||
func (u UsageReader) Usage() UsageAnswer {
|
||||
now := time.Now().UTC()
|
||||
if u.Now != nil {
|
||||
now = u.Now().UTC()
|
||||
}
|
||||
ans := UsageAnswer{At: now, Nodes: []NodeUsage{}, Backlog: []BacklogRow{}, TriesRunning: []TryRunning{}}
|
||||
note := func(f string, a ...any) { ans.Notes = append(ans.Notes, fmt.Sprintf(f, a...)) }
|
||||
|
||||
tries := u.tries(now, &ans, note)
|
||||
seats := map[string]map[string]*SeatUsage{} // node -> language -> seat
|
||||
seat := func(node, lang string) *SeatUsage {
|
||||
if seats[node] == nil {
|
||||
seats[node] = map[string]*SeatUsage{}
|
||||
}
|
||||
if s := seats[node][lang]; s != nil {
|
||||
return s
|
||||
}
|
||||
name, _ := delivery.BuildSeat(lang)
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
s := &SeatUsage{Seat: name, Module: module, Running: RunningUsage{ByKind: map[string]int{}}, LeftRunning: []StepUsage{}, Steps: []StepUsage{}}
|
||||
seats[node][lang] = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Which node runs which build module: the have, <node>.mesh-build-<language>.
|
||||
haves := map[string]delivery.Have{}
|
||||
if u.Haves.Keys == nil || u.Haves.Get == nil {
|
||||
note("the node-engine's have is not readable here, so no seat is said held")
|
||||
} else if keys, err := u.Haves.Keys(); err != nil {
|
||||
note("the node-engine's have cannot be listed, so no seat is said held: %v", err)
|
||||
} else {
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
if m, _ := delivery.BuildModuleOf(lang); strings.HasSuffix(k, "."+m) {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
raws, err := u.Haves.values(want)
|
||||
if err != nil {
|
||||
note("the node-engine's have cannot be read: %v", err)
|
||||
}
|
||||
for i, k := range want {
|
||||
if i < len(raws) && len(raws[i]) > 0 {
|
||||
h, err := delivery.DecodeHave(raws[i])
|
||||
if err != nil {
|
||||
note("the have %s cannot be read, so its seat is not said held: %v", k, err)
|
||||
continue
|
||||
}
|
||||
haves[k] = h
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, h := range haves {
|
||||
node, module, _ := strings.Cut(k, ".")
|
||||
lang := strings.TrimPrefix(module, "mesh-build-")
|
||||
s := seat(node, lang)
|
||||
s.Phase, s.Holds = string(h.Phase), h.Phase == delivery.PhaseRunning
|
||||
}
|
||||
|
||||
// Capacity: mesh-config's resolved value of the module's `capacity` on each node.
|
||||
resolved := map[string]string{} // <node>.<module> -> setting
|
||||
if u.Resolved.Keys == nil || u.Resolved.Get == nil {
|
||||
note("mesh-config's resolved values are not readable here, so each capacity is the modules' default %q", buildstep.DefaultCapacities)
|
||||
} else if keys, err := u.Resolved.Keys(); err != nil {
|
||||
note("mesh-config's resolved values cannot be listed, so each capacity is the modules' default %q: %v", buildstep.DefaultCapacities, err)
|
||||
} else {
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
if strings.HasSuffix(k, "."+buildstep.CapacitySetting) && strings.Contains(k, ".mesh-build-") {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
raws, err := u.Resolved.values(want)
|
||||
if err != nil {
|
||||
note("mesh-config's resolved capacities cannot be read: %v", err)
|
||||
}
|
||||
for i, k := range want {
|
||||
var r struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
setting := ""
|
||||
if i < len(raws) && json.Unmarshal(raws[i], &r) == nil && json.Unmarshal(r.Value, &setting) == nil {
|
||||
resolved[strings.TrimSuffix(k, "."+buildstep.CapacitySetting)] = setting
|
||||
}
|
||||
}
|
||||
}
|
||||
for key := range resolved {
|
||||
node, module, _ := strings.Cut(key, ".")
|
||||
seat(node, strings.TrimPrefix(module, "mesh-build-"))
|
||||
}
|
||||
// The steps of each build module.
|
||||
steps := map[string]map[string][]usageStep{} // node -> language -> steps
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
sr, ok := u.Steps[lang]
|
||||
if !ok || sr.Keys == nil || sr.Get == nil {
|
||||
note("the steps of %s are not readable here", module)
|
||||
continue
|
||||
}
|
||||
keys, err := sr.Keys()
|
||||
if err != nil {
|
||||
note("the steps of %s cannot be listed: %v", module, err)
|
||||
continue
|
||||
}
|
||||
// A step id is its start in base-36 milliseconds, so the newest keys are the ones of the last day; at most
|
||||
// MaxStateReads of them are read, newest first.
|
||||
type keyAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var recent []keyAt
|
||||
for _, k := range keys {
|
||||
_, id, err := delivery.ParseStepKey(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if at, ok := stepIDTime(id); ok && now.Sub(at) <= 24*time.Hour+UsageStepBound {
|
||||
recent = append(recent, keyAt{k, at})
|
||||
}
|
||||
}
|
||||
sort.Slice(recent, func(i, j int) bool { return recent[i].at.After(recent[j].at) })
|
||||
if len(recent) > MaxStateReads {
|
||||
note("the steps of %s: read the newest %d of %d keys of the last day; the older ones are not counted", module, MaxStateReads, len(recent))
|
||||
recent = recent[:MaxStateReads]
|
||||
}
|
||||
want := make([]string, len(recent))
|
||||
for i, r := range recent {
|
||||
want[i] = r.key
|
||||
}
|
||||
raws, err := sr.values(want)
|
||||
if err != nil {
|
||||
note("the steps of %s cannot all be read: %v", module, err)
|
||||
}
|
||||
for i := range want {
|
||||
if i >= len(raws) || len(raws[i]) == 0 {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(raws[i])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if steps[s.Node] == nil {
|
||||
steps[s.Node] = map[string][]usageStep{}
|
||||
}
|
||||
steps[s.Node][lang] = append(steps[s.Node][lang], usageStep{s, stepKind(s, tries)})
|
||||
seat(s.Node, lang)
|
||||
}
|
||||
}
|
||||
|
||||
// Pauses: paused.<node> of each module.
|
||||
paused := map[string]bool{} // <node>.<language>
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
module, _ := delivery.BuildModuleOf(lang)
|
||||
pr, ok := u.Pauses[lang]
|
||||
if !ok || pr.Get == nil {
|
||||
note("the pauses of %s are not readable here, so none is said paused", module)
|
||||
continue
|
||||
}
|
||||
for node := range seats {
|
||||
if seats[node][lang] == nil {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.PauseKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, err := pr.Get(key)
|
||||
if err != nil {
|
||||
note("the pause of %s on %s cannot be read: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
if p, err := delivery.DecodePause(raw); err == nil {
|
||||
paused[node+"."+lang] = p.Paused
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nodes := make([]string, 0, len(seats))
|
||||
for n := range seats {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, node := range nodes {
|
||||
nu := NodeUsage{Node: node, Seats: []SeatUsage{}, Said: []string{},
|
||||
CPU: "not on the bus: the disk-load module keeps no state; " + node + "/disk-load.cpu_top reads it (the build modules' share among its units)"}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
s := seats[node][lang]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
if setting, ok := resolved[node+"."+s.Module]; ok {
|
||||
s.CapacityFrom = "mesh-config, resolved for " + s.Module + " on " + node
|
||||
if caps, err := buildrun.ParseCapacity(setting); err != nil {
|
||||
s.CapacityFrom = fmt.Sprintf("mesh-config's value %q, which cannot be read, so the module pulls nothing: %v", setting, err)
|
||||
} else {
|
||||
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
||||
}
|
||||
} else {
|
||||
caps, _ := buildrun.ParseCapacity(buildstep.DefaultCapacities)
|
||||
s.Capacity = (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
||||
s.CapacityFrom = fmt.Sprintf("the modules' default %q: mesh-config holds no resolved value for %s on %s", buildstep.DefaultCapacities, s.Module, node)
|
||||
}
|
||||
s.Paused = paused[node+"."+lang]
|
||||
switch {
|
||||
case !s.Holds && s.Phase == "":
|
||||
s.WhyNot = s.Module + " is not on " + node
|
||||
case !s.Holds:
|
||||
s.WhyNot = s.Module + " on " + node + " is " + s.Phase
|
||||
case s.Capacity == 0:
|
||||
s.WhyNot = "its capacity on " + node + " is 0"
|
||||
case s.Paused:
|
||||
s.WhyNot = "paused (" + s.Seat + ".resume takes asks again)"
|
||||
default:
|
||||
s.Eligible = true
|
||||
}
|
||||
nu.Eligible = nu.Eligible || s.Eligible
|
||||
s.fill(steps[node][lang], now)
|
||||
nu.Said = append(nu.Said, s.said())
|
||||
nu.Seats = append(nu.Seats, *s)
|
||||
}
|
||||
ans.Nodes = append(ans.Nodes, nu)
|
||||
}
|
||||
|
||||
if u.Backlog == nil {
|
||||
note("the backlog of the seats' streams is not readable here")
|
||||
} else if rows, err := u.Backlog(); err != nil {
|
||||
note("the backlog of the seats' streams is read by the view program, which did not answer: %v", err)
|
||||
} else if rows != nil {
|
||||
ans.Backlog = rows
|
||||
}
|
||||
return ans
|
||||
}
|
||||
|
||||
// tries reads the checker's tries of the last day: the running ones, and the asks every one put.
|
||||
func (u UsageReader) tries(now time.Time, ans *UsageAnswer, note func(string, ...any)) map[string]bool {
|
||||
asks := map[string]bool{}
|
||||
if u.Tries.Keys == nil || u.Tries.Get == nil {
|
||||
note("the checker's tries are not readable here, so a try's asks are counted under check")
|
||||
return asks
|
||||
}
|
||||
keys, err := u.Tries.Keys()
|
||||
if err != nil {
|
||||
note("the checker's tries cannot be listed, so a try's asks are counted under check: %v", err)
|
||||
return asks
|
||||
}
|
||||
var want []string
|
||||
for _, k := range keys {
|
||||
ns, err := strconv.ParseInt(strings.TrimPrefix(k, "try-"), 10, 64)
|
||||
if err == nil && strings.HasPrefix(k, "try-") && now.Sub(time.Unix(0, ns)) <= 48*time.Hour {
|
||||
want = append(want, k)
|
||||
}
|
||||
}
|
||||
raws, err := u.Tries.values(want)
|
||||
if err != nil {
|
||||
note("the checker's tries cannot all be read: %v", err)
|
||||
}
|
||||
for i := range want {
|
||||
if i >= len(raws) || len(raws[i]) == 0 {
|
||||
continue
|
||||
}
|
||||
var t struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
State string `json:"state"`
|
||||
Started time.Time `json:"started"`
|
||||
Asks []string `json:"asks"`
|
||||
}
|
||||
if json.Unmarshal(raws[i], &t) != nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range t.Asks {
|
||||
asks[a] = true
|
||||
}
|
||||
if t.State == "running" {
|
||||
ans.TriesRunning = append(ans.TriesRunning, TryRunning{ID: t.ID, Repository: t.Repository, Started: t.Started})
|
||||
}
|
||||
}
|
||||
if len(ans.TriesRunning) > 0 {
|
||||
note("a running try's asks are counted under check until the try ends and names them")
|
||||
}
|
||||
return asks
|
||||
}
|
||||
|
||||
// fill counts a seat's steps: running now, left running (started past UsageStepBound, or its module no longer running on
|
||||
// the node: no step behind the key runs any more), and the two windows.
|
||||
func (s *SeatUsage) fill(steps []usageStep, now time.Time) {
|
||||
sort.Slice(steps, func(i, j int) bool { return steps[i].step.Started.Before(steps[j].step.Started) })
|
||||
var live []usageStep // ended, or running and not left: what the windows count
|
||||
for _, us := range steps {
|
||||
st := us.step
|
||||
if st.State != delivery.StepRunning {
|
||||
live = append(live, us)
|
||||
continue
|
||||
}
|
||||
line := StepUsage{ID: st.ID, Kind: us.kind, Tool: st.Tool, Started: st.Started, Secs: int64(now.Sub(st.Started).Seconds())}
|
||||
switch {
|
||||
case !s.Holds:
|
||||
line.Why = "its key says running, and " + s.Module + " is not running on its node, so no step behind it runs"
|
||||
case now.Sub(st.Started) > UsageStepBound:
|
||||
line.Why = fmt.Sprintf("its key says running since %s, past the step bound of %s: its module stopped or restarted before it wrote the end, and no asker waits for it", st.Started.Format(time.RFC3339), UsageStepBound)
|
||||
}
|
||||
if line.Why != "" {
|
||||
s.LeftRunning = append(s.LeftRunning, line)
|
||||
continue
|
||||
}
|
||||
s.Steps = append(s.Steps, line)
|
||||
live = append(live, us)
|
||||
if us.kind != KindHand {
|
||||
s.Running.Total++
|
||||
s.Running.ByKind[us.kind]++
|
||||
}
|
||||
}
|
||||
s.LastHour = window(live, now, time.Hour, s.Capacity)
|
||||
s.LastDay = window(live, now, 24*time.Hour, s.Capacity)
|
||||
}
|
||||
|
||||
// said is the seat's use in one line.
|
||||
func (s *SeatUsage) said() string {
|
||||
head := s.Seat + ": eligible"
|
||||
if !s.Eligible {
|
||||
head = s.Seat + ": not eligible, " + s.WhyNot
|
||||
}
|
||||
kinds := make([]string, 0, len(s.Running.ByKind))
|
||||
for k, n := range s.Running.ByKind {
|
||||
kinds = append(kinds, fmt.Sprintf("%s %d", k, n))
|
||||
}
|
||||
sort.Strings(kinds)
|
||||
line := fmt.Sprintf("%s; %d of capacity %d running", head, s.Running.Total, s.Capacity)
|
||||
if len(kinds) > 0 {
|
||||
line += " (" + strings.Join(kinds, ", ") + ")"
|
||||
}
|
||||
if len(s.LeftRunning) > 0 {
|
||||
line += fmt.Sprintf(", %d left running", len(s.LeftRunning))
|
||||
}
|
||||
return line + fmt.Sprintf("; last hour %d finished, median %d s, at capacity %.0f%% of it; last day %d finished, median %d s, at capacity %.1f%% of it",
|
||||
s.LastHour.Finished, s.LastHour.MedianSecs, s.LastHour.AtCapacityShare*100, s.LastDay.Finished, s.LastDay.MedianSecs, s.LastDay.AtCapacityShare*100)
|
||||
}
|
||||
|
||||
// window counts the steps of the last span before now: started in it, finished in it with their total and median time,
|
||||
// and the time in it during which capacity asks (not hand steps) ran at once.
|
||||
func window(steps []usageStep, now time.Time, span time.Duration, capacity int) WindowUsage {
|
||||
from := now.Add(-span)
|
||||
w := WindowUsage{ByKind: map[string]KindWindow{}}
|
||||
var all []int64
|
||||
byKind := map[string][]int64{}
|
||||
type edge struct {
|
||||
at time.Time
|
||||
d int
|
||||
}
|
||||
var edges []edge
|
||||
for _, us := range steps {
|
||||
st := us.step
|
||||
if !st.Started.Before(from) {
|
||||
w.Started++
|
||||
}
|
||||
if !st.Ended.IsZero() && !st.Ended.Before(from) {
|
||||
w.Finished++
|
||||
if st.State == delivery.StepFailed || st.State == delivery.StepRefused {
|
||||
w.Failed++
|
||||
}
|
||||
secs := int64(st.Ended.Sub(st.Started).Seconds())
|
||||
all = append(all, secs)
|
||||
byKind[us.kind] = append(byKind[us.kind], secs)
|
||||
}
|
||||
if us.kind == KindHand {
|
||||
continue
|
||||
}
|
||||
start, end := st.Started, st.Ended
|
||||
if end.IsZero() {
|
||||
end = now
|
||||
}
|
||||
if start.Before(from) {
|
||||
start = from
|
||||
}
|
||||
if end.After(start) {
|
||||
edges = append(edges, edge{start, 1}, edge{end, -1})
|
||||
}
|
||||
}
|
||||
w.TotalSecs, w.MedianSecs = sumMedian(all)
|
||||
for k, d := range byKind {
|
||||
kw := KindWindow{Finished: len(d)}
|
||||
kw.TotalSecs, kw.MedianSecs = sumMedian(d)
|
||||
w.ByKind[k] = kw
|
||||
}
|
||||
if capacity > 0 {
|
||||
// An end before a start at the same instant: a slot handed from one ask to the next is not a moment at capacity+1.
|
||||
sort.Slice(edges, func(i, j int) bool {
|
||||
if edges[i].at.Equal(edges[j].at) {
|
||||
return edges[i].d < edges[j].d
|
||||
}
|
||||
return edges[i].at.Before(edges[j].at)
|
||||
})
|
||||
var at time.Duration
|
||||
n := 0
|
||||
for i, e := range edges {
|
||||
n += e.d
|
||||
if n >= capacity && i+1 < len(edges) {
|
||||
at += edges[i+1].at.Sub(e.at)
|
||||
}
|
||||
}
|
||||
w.AtCapacitySecs = int64(at.Seconds())
|
||||
w.AtCapacityShare = float64(int64(at*1000/span)) / 1000
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// sumMedian is the sum and the median of d (the lower middle of an even count); zero for none.
|
||||
func sumMedian(d []int64) (sum, median int64) {
|
||||
if len(d) == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
s := append([]int64(nil), d...)
|
||||
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
||||
for _, v := range s {
|
||||
sum += v
|
||||
}
|
||||
return sum, s[(len(s)-1)/2]
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// usageNow is the fixture's clock.
|
||||
var usageNow = time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// at is the fixture's time hh:mm on its day.
|
||||
func at(hm string) time.Time {
|
||||
t, _ := time.Parse("2006-01-02 15:04", "2026-10-26 "+hm)
|
||||
return t.UTC()
|
||||
}
|
||||
|
||||
// stepID is a step id made at t, as buildstep.NewID makes it, with suffix as its six hex digits.
|
||||
func stepID(t time.Time, suffix string) string { return strconv.FormatInt(t.UnixMilli(), 36) + suffix }
|
||||
|
||||
func states(m map[string]string) StateRead {
|
||||
return StateRead{
|
||||
Keys: func() ([]string, error) {
|
||||
var k []string
|
||||
for key := range m {
|
||||
k = append(k, key)
|
||||
}
|
||||
return k, nil
|
||||
},
|
||||
Get: func(key string) (json.RawMessage, error) {
|
||||
if v, ok := m[key]; ok {
|
||||
return json.RawMessage(v), nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func step(node, id, tool, state string, started, ended time.Time) string {
|
||||
e := ""
|
||||
if !ended.IsZero() {
|
||||
e = `,"ended":"` + ended.Format(time.RFC3339) + `"`
|
||||
}
|
||||
return fmt.Sprintf(`{"v":1,"id":%q,"tool":%q,"node":%q,"state":%q,"started":%q%s}`, id, tool, node, state, started.Format(time.RFC3339), e)
|
||||
}
|
||||
|
||||
// usageFixture is four nodes: shanks builds Go at capacity 2 (two asks running, one left running, a try's ask, a check,
|
||||
// a recheck of the morning and a hand step ended); ace holds the Go seat at capacity 2 and is paused; g14 holds it at
|
||||
// capacity 0; the Node seat on shanks has no resolved capacity and takes the modules' default.
|
||||
func usageFixture() UsageReader {
|
||||
running := `{"v":1,"phase":"running","health":"ok"}`
|
||||
tryAsk := stepID(at("11:30"), "a00003")
|
||||
goSteps := map[string]string{}
|
||||
add := func(node, id, tool, state string, started, ended time.Time) {
|
||||
goSteps[node+"."+id] = step(node, id, tool, state, started, ended)
|
||||
}
|
||||
add("shanks", stepID(at("11:50"), "a00001"), "delivery ask", "running", at("11:50"), time.Time{})
|
||||
add("shanks", stepID(at("11:55"), "a00002"), "check ask", "running", at("11:55"), time.Time{})
|
||||
add("shanks", tryAsk, "check ask", "passed", at("11:30"), at("11:40"))
|
||||
add("shanks", stepID(at("11:20"), "a00004"), "check ask", "failed", at("11:20"), at("11:26"))
|
||||
add("shanks", stepID(at("06:00"), "a00005"), "recheck ask", "passed", at("06:00"), at("06:10"))
|
||||
add("shanks", stepID(at("10:00"), "a00006"), "check ask", "running", at("10:00"), time.Time{})
|
||||
add("shanks", stepID(at("11:45"), "a00007"), "build_go_test", "passed", at("11:45"), at("11:46"))
|
||||
goSteps["shanks.notastepkey"] = `{}` // a key that is no step id is passed over
|
||||
return UsageReader{
|
||||
Now: func() time.Time { return usageNow },
|
||||
Haves: states(map[string]string{"shanks.mesh-build-go": running, "shanks.mesh-build-node": running, "ace.mesh-build-go": running,
|
||||
"g14.mesh-build-go": running, "shanks.node": `{}`, "shanks.mesh-delivery-view": running}),
|
||||
Resolved: states(map[string]string{
|
||||
"shanks.mesh-build-go.capacity": `{"node":"shanks","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
|
||||
"ace.mesh-build-go.capacity": `{"node":"ace","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
|
||||
"g14.mesh-build-go.capacity": `{"node":"g14","module":"mesh-build-go","key":"capacity","value":"shanks=2,ace=2,g14=0","source":"mesh"}`,
|
||||
"shanks.mesh-config.other": `{"value":"x"}`,
|
||||
}),
|
||||
Steps: map[string]StateRead{"go": states(goSteps), "node": states(map[string]string{}), "image": states(map[string]string{})},
|
||||
Pauses: map[string]StateRead{"go": states(map[string]string{"paused.ace": `{"v":1,"paused":true,"by":"lead@shanks","at":"2026-10-26T09:00:00Z"}`}), "node": states(nil), "image": states(nil)},
|
||||
Tries: states(map[string]string{
|
||||
"try-" + strconv.FormatInt(at("11:29").UnixNano(), 10): `{"id":"try-1","repository":"novox/mesh-sdk","head":"h","trunk":"t","state":"pass","asks":["` + tryAsk + `"]}`,
|
||||
"try-" + strconv.FormatInt(at("11:58").UnixNano(), 10): `{"id":"try-2","repository":"novox/mesh-delivery","head":"h","trunk":"t","state":"running","started":"2026-10-26T11:58:00Z"}`,
|
||||
}),
|
||||
Backlog: func() ([]BacklogRow, error) {
|
||||
return []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func seatOf(t *testing.T, a UsageAnswer, node, seat string) SeatUsage {
|
||||
t.Helper()
|
||||
for _, n := range a.Nodes {
|
||||
for _, s := range n.Seats {
|
||||
if n.Node == node && s.Seat == seat {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("no seat %s on %s in %+v", seat, node, a.Nodes)
|
||||
return SeatUsage{}
|
||||
}
|
||||
|
||||
// build_usage answers, from the states alone, what the operator asked in six calls: per node its seats, capacity,
|
||||
// eligibility, the asks running by kind and the ones left running, the last hour and day, and mesh-wide the backlog.
|
||||
func TestBuildUsageAnswersEveryNodesUseFromTheStates(t *testing.T) {
|
||||
a := usageFixture().Usage()
|
||||
if testing.Verbose() {
|
||||
out, _ := json.MarshalIndent(a, "", " ")
|
||||
t.Logf("build_usage answers:\n%s", out)
|
||||
}
|
||||
sg := seatOf(t, a, "shanks", "node-build-go")
|
||||
eligible := map[string]bool{}
|
||||
for _, n := range a.Nodes {
|
||||
eligible[n.Node] = n.Eligible
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
got, want any
|
||||
}{
|
||||
{"nodes", len(a.Nodes), 3},
|
||||
{"eligible", eligible, map[string]bool{"shanks": true, "ace": false, "g14": false}},
|
||||
{"ace why not", seatOf(t, a, "ace", "node-build-go").WhyNot, "paused (node-build-go.resume takes asks again)"},
|
||||
{"g14 why not", seatOf(t, a, "g14", "node-build-go").WhyNot, "its capacity on g14 is 0"},
|
||||
{"shanks capacity", sg.Capacity, 2},
|
||||
{"shanks node seat takes the default", seatOf(t, a, "shanks", "node-build-node").Capacity, 3},
|
||||
{"running", sg.Running.Total, 2},
|
||||
{"running by kind", sg.Running.ByKind, map[string]int{"delivery": 1, "check": 1}},
|
||||
{"running steps, the hand step's ended", len(sg.Steps), 2},
|
||||
{"left running", len(sg.LeftRunning), 1},
|
||||
{"hour started", sg.LastHour.Started, 5}, // delivery, check, try, check, hand
|
||||
{"hour finished", sg.LastHour.Finished, 3}, // try 600 s, check 360 s, hand 60 s
|
||||
{"hour failed", sg.LastHour.Failed, 1},
|
||||
{"hour total", sg.LastHour.TotalSecs, int64(1020)},
|
||||
{"hour median", sg.LastHour.MedianSecs, int64(360)},
|
||||
{"hour try", sg.LastHour.ByKind["try"], KindWindow{Finished: 1, TotalSecs: 600, MedianSecs: 600}},
|
||||
{"hour hand", sg.LastHour.ByKind["hand"], KindWindow{Finished: 1, TotalSecs: 60, MedianSecs: 60}},
|
||||
{"hour at capacity", sg.LastHour.AtCapacitySecs, int64(300)}, // 11:55-12:00, two asks of two
|
||||
{"hour at capacity share", sg.LastHour.AtCapacityShare, 0.083},
|
||||
{"day finished", sg.LastDay.Finished, 4},
|
||||
{"day recheck", sg.LastDay.ByKind["recheck"].Finished, 1},
|
||||
{"day at capacity", sg.LastDay.AtCapacitySecs, int64(300)},
|
||||
{"backlog", a.Backlog, []BacklogRow{{Language: "go", Kind: "check", Waiting: 3, Held: 2, OldestSecs: 240}}},
|
||||
{"tries running", len(a.TriesRunning), 1},
|
||||
{"said", a.Nodes[0].Said, []string{"node-build-go: not eligible, paused (node-build-go.resume takes asks again); 0 of capacity 2 running; last hour 0 finished, median 0 s, at capacity 0% of it; last day 0 finished, median 0 s, at capacity 0.0% of it"}},
|
||||
{"said shanks", a.Nodes[2].Said[0], "node-build-go: eligible; 2 of capacity 2 running (check 1, delivery 1), 1 left running; last hour 3 finished, median 360 s, at capacity 8% of it; last day 4 finished, median 360 s, at capacity 0.3% of it"},
|
||||
{"cpu says where it is", strings.Contains(a.Nodes[0].CPU, "/disk-load.cpu_top"), true},
|
||||
} {
|
||||
if fmt.Sprint(c.got) != fmt.Sprint(c.want) {
|
||||
t.Errorf("%s = %v, want %v", c.what, c.got, c.want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(sg.LeftRunning[0].Why, "past the step bound") {
|
||||
t.Errorf("left running says %q", sg.LeftRunning[0].Why)
|
||||
}
|
||||
|
||||
// Nothing readable: each state is named in a note, the backlog's refusal too, and nothing is said held.
|
||||
none := UsageReader{Now: func() time.Time { return usageNow }, Backlog: func() ([]BacklogRow, error) { return nil, errors.New("no socket") }}.Usage()
|
||||
notes := strings.Join(none.Notes, " | ")
|
||||
for _, want := range []string{"have is not readable", "resolved values are not readable", "steps of mesh-build-go are not readable", "tries are not readable", "did not answer: no socket"} {
|
||||
if !strings.Contains(notes, want) {
|
||||
t.Errorf("the notes %q do not say %q", notes, want)
|
||||
}
|
||||
}
|
||||
if len(none.Nodes) != 0 || none.Backlog == nil {
|
||||
t.Errorf("nothing readable answered nodes %v, backlog %v", none.Nodes, none.Backlog)
|
||||
}
|
||||
}
|
||||
@@ -84,7 +84,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
if m.Module != "mesh-delivery-view" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
@@ -115,7 +115,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
}
|
||||
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
||||
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -13,7 +13,8 @@
|
||||
"assigner_wants",
|
||||
"assigner_want",
|
||||
"assigner_differences",
|
||||
"node_haves"
|
||||
"node_haves",
|
||||
"build_usage"
|
||||
],
|
||||
"emits": [
|
||||
"delivered",
|
||||
@@ -27,11 +28,19 @@
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-build-go.pauses",
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-image.pauses",
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.pauses",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.composed",
|
||||
"mesh-delivery-assigner.asks",
|
||||
"mesh-delivery-assigner.differences",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-checker.tries",
|
||||
"mesh-delivery-gate.asks",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
|
||||
Reference in New Issue
Block a user