SC1: the self-check raises over-budget from the gate's times

gate_times answers per class its newest delivery (last) with whether it was
over its budget; the self-check's probe D-budget reads gate_times and raises
over-budget with the class as its id, naming the source revision and its
longest phase, and a newest delivery within its budget clears it.

Builds: design 47 H2 §810; H-A-396
This commit is contained in:
2026-11-22 00:37:03 +01:00
parent 872a02b378
commit 5d2d0bade8
7 changed files with 108 additions and 9 deletions
@@ -294,7 +294,8 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T
times, err := h.Times()
core, leaf := times[catalogue.ClassCore], times[catalogue.ClassLeaf]
if err != nil || core.Budget != "10m0s" || core.Deliveries != 2 || len(core.Over) != 1 || core.Over[0].DeliveryTime != "11m0s" ||
core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 {
core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 ||
core.Last == nil || core.Last.Over || core.Last.DeliveryTime != "4m0s" || leaf.Last == nil || leaf.Last.Over {
t.Fatalf("gate_times %+v %v", times, err)
}
})
@@ -198,7 +198,8 @@ func recordTools(h *Holder, acts handActs) []stdio.Tool {
{Name: "gate_times",
Description: "Delivery times against their budgets (design 47 H2 §810; ADR 0282 §6): for the classes core and leaf, the " +
"budget (the settings budget-core and budget-leaf), the last seven days' median and worst delivery time from " +
"merge-called to the last running of its movers, and every delivery over its budget, newest first, with its longest phase.",
"merge-called to the last running of its movers, every delivery over its budget, newest first, with its longest phase, " +
"and the newest delivery (last) with whether it was over.",
Run: func(map[string]any) (any, error) {
if h.Records == nil {
return none()
@@ -142,6 +142,13 @@ type TimesOver struct {
LongestTime string `json:"longest-phase-time,omitempty"`
}
// TimesLast is a class's newest ended delivery: whether it was over its budget, which the self-check's probe judges
// (design 47 H2 §810; H-A-396).
type TimesLast struct {
TimesOver
Over bool `json:"over"`
}
// ClassTimes are one class's times over the last seven days.
type ClassTimes struct {
Budget string `json:"budget"`
@@ -149,6 +156,7 @@ type ClassTimes struct {
Median string `json:"median,omitempty"`
Worst string `json:"worst,omitempty"`
Over []TimesOver `json:"over"`
Last *TimesLast `json:"last,omitempty"`
}
// Times is gate_times: each class's budget, the last seven days' median and worst, and every delivery over its budget,
@@ -178,11 +186,14 @@ func (h *Holder) Times() (map[string]ClassTimes, error) {
return nil, fmt.Errorf("the record %s keeps a delivery time %q that is no duration", r.Instance, r.DeliveryTime)
}
ds = append(ds, d)
o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()}
if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" {
o.Longest, o.LongestTime = name, l.String()
}
if ct.Last == nil {
ct.Last = &TimesLast{TimesOver: o, Over: d > budget}
}
if d > budget {
o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()}
if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" {
o.Longest, o.LongestTime = name, l.String()
}
ct.Over = append(ct.Over, o)
}
}
@@ -35,7 +35,7 @@ const (
// `reads`, which its test holds to these.
var (
Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb, AssignmentsTool, SeatHoldersTool,
RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb}
RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb, TimesTool}
Reads = []string{PresenceRead, HealthRead, ReadingsRead, GoodRead}
// Consumes are the views families the probes follow (ADR 0615): the indexer's and the assigner's states are asked
// once, by their owners' read tools, and then followed by their events, never read.
@@ -104,6 +104,8 @@ func Registry() []Probe {
Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly},
{ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals",
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs},
{ID: "D-budget", Asserts: "every class's newest delivery ran on every machine within its budget from its merge call: " +
"the gate's gate_times", From: "ADR 0282 §7, design 47 H2 §810", Kind: kindOverBudget, Run: probeBudget},
{ID: "H-bus", Asserts: "every stream the mesh defines is on the bus, and a request crosses it " +
"to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth},
}
@@ -0,0 +1,73 @@
package main
// The delivery budget's probe (design 47 H2 §810; ADR 0282 §7; H-A-396): it reads the gate's times (gate_times) and raises
// over-budget with the class as its id when that class's newest delivery took longer than its budget, naming the source
// revision and its longest phase; a newest delivery within its budget clears it. It reads no records itself, and the gate
// raises nothing.
import (
"context"
"encoding/json"
"fmt"
"sort"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
)
// TimesTool is the gate's read of its delivery times against their budgets.
const TimesTool = "mesh-delivery-gate.gate_times"
// scopeDelivery is the scope of the delivery's findings, the gate's own (A-42).
const scopeDelivery = "delivery"
// kindOverBudget is the finding of a class whose newest delivery took longer than its budget.
const kindOverBudget = "over-budget"
// classTimes is one class as gate_times answers it: its budget and its newest delivery.
type classTimes struct {
Budget string `json:"budget"`
Last *struct {
Record string `json:"record"`
Milestone string `json:"milestone"`
Ended string `json:"ended"`
DeliveryTime string `json:"delivery-time"`
Longest string `json:"longest-phase"`
LongestTime string `json:"longest-phase-time"`
Over bool `json:"over"`
} `json:"last"`
}
func probeBudget(_ context.Context, c *Checker) ([]Finding, error) {
raw, err := c.Ask(TimesTool, map[string]any{})
if err != nil {
return nil, fmt.Errorf("the gate did not answer %s: %w", TimesTool, err)
}
var times map[string]classTimes
if err := json.Unmarshal(raw, &times); err != nil {
return nil, fmt.Errorf("the gate answered %s with something unreadable: %w", TimesTool, err)
}
classes := make([]string, 0, len(times))
for class := range times {
classes = append(classes, class)
}
sort.Strings(classes)
var out []Finding
for _, class := range classes {
t := times[class]
if t.Last == nil || !t.Last.Over {
continue
}
longest := "no phase recorded"
if t.Last.Longest != "" {
longest = fmt.Sprintf("its longest phase %s took %s", t.Last.Longest, t.Last.LongestTime)
}
out = append(out, finding(scopeDelivery, class, "", "", Finding{Kind: kindOverBudget, Severity: meshconditionsclient.Warning,
Summary: fmt.Sprintf("the newest %s delivery, %s, took %s from its merge call to running everywhere, over its budget of %s; %s",
class, t.Last.Milestone, t.Last.DeliveryTime, t.Budget, longest),
Evidence: said(fmt.Sprintf("record %s ended %s", t.Last.Record, t.Last.Ended)),
Headline: capital(class) + " deliveries are over their budget",
Explanation: capital(fmt.Sprintf("a delivery moving a %s module should run on every machine within %s of its merge "+
"call; the newest took %s. The next %s delivery within its budget clears this.", class, t.Budget, t.Last.DeliveryTime, class))}))
}
return sorted(out), nil
}
@@ -34,6 +34,8 @@ type bus struct {
banned map[string]string
// fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap.
fills []fillRow
// times is what the gate's gate_times answers.
times string
}
func (b bus) rows() map[string]*streamRow {
@@ -87,6 +89,8 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) {
return nil, errors.New("no stream called " + stream)
}
return json.Marshal(map[string]any{"detail": detail})
case TimesTool:
return json.RawMessage(b.times), nil
case ClosedTool:
if b.closed == "" {
return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil
@@ -133,6 +137,12 @@ func TestTheBusProbes(t *testing.T) {
{"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]},
{"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`},
[]string{"bus.g14=2enode-tools.bus-reconnects"}, "", nil},
// design 47 H2 §810: a core delivery 11 minutes after its merge call is over its 10 minutes; the next within clears it.
{"D-budget a core delivery over", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_7","milestone":"main.7",
"ended":"2026-11-22T00:11:00Z","delivery-time":"11m0s","longest-phase":"build","longest-phase-time":"7m0s","over":true}},
"leaf":{"budget":"3m0s"}}`}, []string{"delivery.core.over-budget"}, "", nil},
{"D-budget the next core delivery within", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_8",
"milestone":"main.8","delivery-time":"4m0s","over":false}},"leaf":{"budget":"3m0s"}}`}, nil, "", nil},
{"DW answers", probeWatchdogs, bus{}, nil, "", nil},
{"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.watchdogs-silent"}, "", nil},
{"H-bus healthy", probeBusHealth, bus{}, nil, "", nil},
@@ -338,7 +348,7 @@ func agents(verdicts map[string]string) *memStore {
// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4).
func TestTheRegistryIsADR0564sTable(t *testing.T) {
moved := []string{"D2", "D3", "D4", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard",
"DW", "H-bus"}
"DW", "H-bus", "D-budget"}
var ids []string
for _, p := range Registry() {
ids = append(ids, p.ID)
+2 -1
View File
@@ -26,7 +26,8 @@
"seat:mesh-catalog.revision",
"seat:mesh-catalog.registrations",
"seat:mesh-delivery.source",
"seat:mesh-hand-acts.hand-act"
"seat:mesh-hand-acts.hand-act",
"mesh-delivery-gate.gate_times"
],
"reads": [
"mesh-delivery-watchman.presence",