diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 0f583d24f..e0262dc92 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -294,7 +294,8 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T times, err := h.Times() core, leaf := times[catalogue.ClassCore], times[catalogue.ClassLeaf] if err != nil || core.Budget != "10m0s" || core.Deliveries != 2 || len(core.Over) != 1 || core.Over[0].DeliveryTime != "11m0s" || - core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 { + core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 || + core.Last == nil || core.Last.Over || core.Last.DeliveryTime != "4m0s" || leaf.Last == nil || leaf.Last.Over { t.Fatalf("gate_times %+v %v", times, err) } }) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go index 330a09dd6..c8ab68e22 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -198,7 +198,8 @@ func recordTools(h *Holder, acts handActs) []stdio.Tool { {Name: "gate_times", Description: "Delivery times against their budgets (design 47 H2 §810; ADR 0282 §6): for the classes core and leaf, the " + "budget (the settings budget-core and budget-leaf), the last seven days' median and worst delivery time from " + - "merge-called to the last running of its movers, and every delivery over its budget, newest first, with its longest phase.", + "merge-called to the last running of its movers, every delivery over its budget, newest first, with its longest phase, " + + "and the newest delivery (last) with whether it was over.", Run: func(map[string]any) (any, error) { if h.Records == nil { return none() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go index 8d9a56cc5..da2bde271 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/times.go @@ -142,6 +142,13 @@ type TimesOver struct { LongestTime string `json:"longest-phase-time,omitempty"` } +// TimesLast is a class's newest ended delivery: whether it was over its budget, which the self-check's probe judges +// (design 47 H2 §810; H-A-396). +type TimesLast struct { + TimesOver + Over bool `json:"over"` +} + // ClassTimes are one class's times over the last seven days. type ClassTimes struct { Budget string `json:"budget"` @@ -149,6 +156,7 @@ type ClassTimes struct { Median string `json:"median,omitempty"` Worst string `json:"worst,omitempty"` Over []TimesOver `json:"over"` + Last *TimesLast `json:"last,omitempty"` } // Times is gate_times: each class's budget, the last seven days' median and worst, and every delivery over its budget, @@ -178,11 +186,14 @@ func (h *Holder) Times() (map[string]ClassTimes, error) { return nil, fmt.Errorf("the record %s keeps a delivery time %q that is no duration", r.Instance, r.DeliveryTime) } ds = append(ds, d) + o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()} + if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" { + o.Longest, o.LongestTime = name, l.String() + } + if ct.Last == nil { + ct.Last = &TimesLast{TimesOver: o, Over: d > budget} + } if d > budget { - o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()} - if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" { - o.Longest, o.LongestTime = name, l.String() - } ct.Over = append(ct.Over, o) } } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go index 6e944eea3..49fc64641 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -35,7 +35,7 @@ const ( // `reads`, which its test holds to these. var ( Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb, AssignmentsTool, SeatHoldersTool, - RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb} + RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb, TimesTool} Reads = []string{PresenceRead, HealthRead, ReadingsRead, GoodRead} // Consumes are the views families the probes follow (ADR 0615): the indexer's and the assigner's states are asked // once, by their owners' read tools, and then followed by their events, never read. @@ -104,6 +104,8 @@ func Registry() []Probe { Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly}, {ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs}, + {ID: "D-budget", Asserts: "every class's newest delivery ran on every machine within its budget from its merge call: " + + "the gate's gate_times", From: "ADR 0282 §7, design 47 H2 §810", Kind: kindOverBudget, Run: probeBudget}, {ID: "H-bus", Asserts: "every stream the mesh defines is on the bus, and a request crosses it " + "to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth}, } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_budget.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_budget.go new file mode 100644 index 000000000..b2e56db1b --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_budget.go @@ -0,0 +1,73 @@ +package main + +// The delivery budget's probe (design 47 H2 §810; ADR 0282 §7; H-A-396): it reads the gate's times (gate_times) and raises +// over-budget with the class as its id when that class's newest delivery took longer than its budget, naming the source +// revision and its longest phase; a newest delivery within its budget clears it. It reads no records itself, and the gate +// raises nothing. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" +) + +// TimesTool is the gate's read of its delivery times against their budgets. +const TimesTool = "mesh-delivery-gate.gate_times" + +// scopeDelivery is the scope of the delivery's findings, the gate's own (A-42). +const scopeDelivery = "delivery" + +// kindOverBudget is the finding of a class whose newest delivery took longer than its budget. +const kindOverBudget = "over-budget" + +// classTimes is one class as gate_times answers it: its budget and its newest delivery. +type classTimes struct { + Budget string `json:"budget"` + Last *struct { + Record string `json:"record"` + Milestone string `json:"milestone"` + Ended string `json:"ended"` + DeliveryTime string `json:"delivery-time"` + Longest string `json:"longest-phase"` + LongestTime string `json:"longest-phase-time"` + Over bool `json:"over"` + } `json:"last"` +} + +func probeBudget(_ context.Context, c *Checker) ([]Finding, error) { + raw, err := c.Ask(TimesTool, map[string]any{}) + if err != nil { + return nil, fmt.Errorf("the gate did not answer %s: %w", TimesTool, err) + } + var times map[string]classTimes + if err := json.Unmarshal(raw, ×); err != nil { + return nil, fmt.Errorf("the gate answered %s with something unreadable: %w", TimesTool, err) + } + classes := make([]string, 0, len(times)) + for class := range times { + classes = append(classes, class) + } + sort.Strings(classes) + var out []Finding + for _, class := range classes { + t := times[class] + if t.Last == nil || !t.Last.Over { + continue + } + longest := "no phase recorded" + if t.Last.Longest != "" { + longest = fmt.Sprintf("its longest phase %s took %s", t.Last.Longest, t.Last.LongestTime) + } + out = append(out, finding(scopeDelivery, class, "", "", Finding{Kind: kindOverBudget, Severity: meshconditionsclient.Warning, + Summary: fmt.Sprintf("the newest %s delivery, %s, took %s from its merge call to running everywhere, over its budget of %s; %s", + class, t.Last.Milestone, t.Last.DeliveryTime, t.Budget, longest), + Evidence: said(fmt.Sprintf("record %s ended %s", t.Last.Record, t.Last.Ended)), + Headline: capital(class) + " deliveries are over their budget", + Explanation: capital(fmt.Sprintf("a delivery moving a %s module should run on every machine within %s of its merge "+ + "call; the newest took %s. The next %s delivery within its budget clears this.", class, t.Budget, t.Last.DeliveryTime, class))})) + } + return sorted(out), nil +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index 7a369328d..9a7739169 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -34,6 +34,8 @@ type bus struct { banned map[string]string // fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap. fills []fillRow + // times is what the gate's gate_times answers. + times string } func (b bus) rows() map[string]*streamRow { @@ -87,6 +89,8 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) { return nil, errors.New("no stream called " + stream) } return json.Marshal(map[string]any{"detail": detail}) + case TimesTool: + return json.RawMessage(b.times), nil case ClosedTool: if b.closed == "" { return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil @@ -133,6 +137,12 @@ func TestTheBusProbes(t *testing.T) { {"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]}, {"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`}, []string{"bus.g14=2enode-tools.bus-reconnects"}, "", nil}, + // design 47 H2 §810: a core delivery 11 minutes after its merge call is over its 10 minutes; the next within clears it. + {"D-budget a core delivery over", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_7","milestone":"main.7", + "ended":"2026-11-22T00:11:00Z","delivery-time":"11m0s","longest-phase":"build","longest-phase-time":"7m0s","over":true}}, + "leaf":{"budget":"3m0s"}}`}, []string{"delivery.core.over-budget"}, "", nil}, + {"D-budget the next core delivery within", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_8", + "milestone":"main.8","delivery-time":"4m0s","over":false}},"leaf":{"budget":"3m0s"}}`}, nil, "", nil}, {"DW answers", probeWatchdogs, bus{}, nil, "", nil}, {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.watchdogs-silent"}, "", nil}, {"H-bus healthy", probeBusHealth, bus{}, nil, "", nil}, @@ -338,7 +348,7 @@ func agents(verdicts map[string]string) *memStore { // it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4). func TestTheRegistryIsADR0564sTable(t *testing.T) { moved := []string{"D2", "D3", "D4", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", - "DW", "H-bus"} + "DW", "H-bus", "D-budget"} var ids []string for _, p := range Registry() { ids = append(ids, p.ID) diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json index fdff2735c..9b07d05fe 100644 --- a/modules/mesh-self-check/module.json +++ b/modules/mesh-self-check/module.json @@ -26,7 +26,8 @@ "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "mesh-delivery-gate.gate_times" ], "reads": [ "mesh-delivery-watchman.presence",