SC1: the self-check raises over-budget from the gate's times
gate_times answers per class its newest delivery (last) with whether it was over its budget; the self-check's probe D-budget reads gate_times and raises over-budget with the class as its id, naming the source revision and its longest phase, and a newest delivery within its budget clears it. Builds: design 47 H2 §810; H-A-396
This commit is contained in:
@@ -294,7 +294,8 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByItsSourceRevision(t *testing.T
|
||||
times, err := h.Times()
|
||||
core, leaf := times[catalogue.ClassCore], times[catalogue.ClassLeaf]
|
||||
if err != nil || core.Budget != "10m0s" || core.Deliveries != 2 || len(core.Over) != 1 || core.Over[0].DeliveryTime != "11m0s" ||
|
||||
core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 {
|
||||
core.Over[0].Longest != "build" || core.Worst != "11m0s" || leaf.Deliveries != 1 || len(leaf.Over) != 0 ||
|
||||
core.Last == nil || core.Last.Over || core.Last.DeliveryTime != "4m0s" || leaf.Last == nil || leaf.Last.Over {
|
||||
t.Fatalf("gate_times %+v %v", times, err)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -198,7 +198,8 @@ func recordTools(h *Holder, acts handActs) []stdio.Tool {
|
||||
{Name: "gate_times",
|
||||
Description: "Delivery times against their budgets (design 47 H2 §810; ADR 0282 §6): for the classes core and leaf, the " +
|
||||
"budget (the settings budget-core and budget-leaf), the last seven days' median and worst delivery time from " +
|
||||
"merge-called to the last running of its movers, and every delivery over its budget, newest first, with its longest phase.",
|
||||
"merge-called to the last running of its movers, every delivery over its budget, newest first, with its longest phase, " +
|
||||
"and the newest delivery (last) with whether it was over.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
if h.Records == nil {
|
||||
return none()
|
||||
|
||||
@@ -142,6 +142,13 @@ type TimesOver struct {
|
||||
LongestTime string `json:"longest-phase-time,omitempty"`
|
||||
}
|
||||
|
||||
// TimesLast is a class's newest ended delivery: whether it was over its budget, which the self-check's probe judges
|
||||
// (design 47 H2 §810; H-A-396).
|
||||
type TimesLast struct {
|
||||
TimesOver
|
||||
Over bool `json:"over"`
|
||||
}
|
||||
|
||||
// ClassTimes are one class's times over the last seven days.
|
||||
type ClassTimes struct {
|
||||
Budget string `json:"budget"`
|
||||
@@ -149,6 +156,7 @@ type ClassTimes struct {
|
||||
Median string `json:"median,omitempty"`
|
||||
Worst string `json:"worst,omitempty"`
|
||||
Over []TimesOver `json:"over"`
|
||||
Last *TimesLast `json:"last,omitempty"`
|
||||
}
|
||||
|
||||
// Times is gate_times: each class's budget, the last seven days' median and worst, and every delivery over its budget,
|
||||
@@ -178,11 +186,14 @@ func (h *Holder) Times() (map[string]ClassTimes, error) {
|
||||
return nil, fmt.Errorf("the record %s keeps a delivery time %q that is no duration", r.Instance, r.DeliveryTime)
|
||||
}
|
||||
ds = append(ds, d)
|
||||
o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()}
|
||||
if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" {
|
||||
o.Longest, o.LongestTime = name, l.String()
|
||||
}
|
||||
if ct.Last == nil {
|
||||
ct.Last = &TimesLast{TimesOver: o, Over: d > budget}
|
||||
}
|
||||
if d > budget {
|
||||
o := TimesOver{Record: r.Instance, Milestone: r.Milestone, Ended: r.Ended.At.Format(time.RFC3339), DeliveryTime: d.String()}
|
||||
if name, l := longestPhase(r.Phases, r.MergeCalled); name != "" {
|
||||
o.Longest, o.LongestTime = name, l.String()
|
||||
}
|
||||
ct.Over = append(ct.Over, o)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ const (
|
||||
// `reads`, which its test holds to these.
|
||||
var (
|
||||
Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb, AssignmentsTool, SeatHoldersTool,
|
||||
RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb}
|
||||
RevisionVerb, RegistrationsVerb, SourceVerb, HandActVerb, TimesTool}
|
||||
Reads = []string{PresenceRead, HealthRead, ReadingsRead, GoodRead}
|
||||
// Consumes are the views families the probes follow (ADR 0615): the indexer's and the assigner's states are asked
|
||||
// once, by their owners' read tools, and then followed by their events, never read.
|
||||
@@ -104,6 +104,8 @@ func Registry() []Probe {
|
||||
Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly},
|
||||
{ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals",
|
||||
From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs},
|
||||
{ID: "D-budget", Asserts: "every class's newest delivery ran on every machine within its budget from its merge call: " +
|
||||
"the gate's gate_times", From: "ADR 0282 §7, design 47 H2 §810", Kind: kindOverBudget, Run: probeBudget},
|
||||
{ID: "H-bus", Asserts: "every stream the mesh defines is on the bus, and a request crosses it " +
|
||||
"to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
// The delivery budget's probe (design 47 H2 §810; ADR 0282 §7; H-A-396): it reads the gate's times (gate_times) and raises
|
||||
// over-budget with the class as its id when that class's newest delivery took longer than its budget, naming the source
|
||||
// revision and its longest phase; a newest delivery within its budget clears it. It reads no records itself, and the gate
|
||||
// raises nothing.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
)
|
||||
|
||||
// TimesTool is the gate's read of its delivery times against their budgets.
|
||||
const TimesTool = "mesh-delivery-gate.gate_times"
|
||||
|
||||
// scopeDelivery is the scope of the delivery's findings, the gate's own (A-42).
|
||||
const scopeDelivery = "delivery"
|
||||
|
||||
// kindOverBudget is the finding of a class whose newest delivery took longer than its budget.
|
||||
const kindOverBudget = "over-budget"
|
||||
|
||||
// classTimes is one class as gate_times answers it: its budget and its newest delivery.
|
||||
type classTimes struct {
|
||||
Budget string `json:"budget"`
|
||||
Last *struct {
|
||||
Record string `json:"record"`
|
||||
Milestone string `json:"milestone"`
|
||||
Ended string `json:"ended"`
|
||||
DeliveryTime string `json:"delivery-time"`
|
||||
Longest string `json:"longest-phase"`
|
||||
LongestTime string `json:"longest-phase-time"`
|
||||
Over bool `json:"over"`
|
||||
} `json:"last"`
|
||||
}
|
||||
|
||||
func probeBudget(_ context.Context, c *Checker) ([]Finding, error) {
|
||||
raw, err := c.Ask(TimesTool, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the gate did not answer %s: %w", TimesTool, err)
|
||||
}
|
||||
var times map[string]classTimes
|
||||
if err := json.Unmarshal(raw, ×); err != nil {
|
||||
return nil, fmt.Errorf("the gate answered %s with something unreadable: %w", TimesTool, err)
|
||||
}
|
||||
classes := make([]string, 0, len(times))
|
||||
for class := range times {
|
||||
classes = append(classes, class)
|
||||
}
|
||||
sort.Strings(classes)
|
||||
var out []Finding
|
||||
for _, class := range classes {
|
||||
t := times[class]
|
||||
if t.Last == nil || !t.Last.Over {
|
||||
continue
|
||||
}
|
||||
longest := "no phase recorded"
|
||||
if t.Last.Longest != "" {
|
||||
longest = fmt.Sprintf("its longest phase %s took %s", t.Last.Longest, t.Last.LongestTime)
|
||||
}
|
||||
out = append(out, finding(scopeDelivery, class, "", "", Finding{Kind: kindOverBudget, Severity: meshconditionsclient.Warning,
|
||||
Summary: fmt.Sprintf("the newest %s delivery, %s, took %s from its merge call to running everywhere, over its budget of %s; %s",
|
||||
class, t.Last.Milestone, t.Last.DeliveryTime, t.Budget, longest),
|
||||
Evidence: said(fmt.Sprintf("record %s ended %s", t.Last.Record, t.Last.Ended)),
|
||||
Headline: capital(class) + " deliveries are over their budget",
|
||||
Explanation: capital(fmt.Sprintf("a delivery moving a %s module should run on every machine within %s of its merge "+
|
||||
"call; the newest took %s. The next %s delivery within its budget clears this.", class, t.Budget, t.Last.DeliveryTime, class))}))
|
||||
}
|
||||
return sorted(out), nil
|
||||
}
|
||||
@@ -34,6 +34,8 @@ type bus struct {
|
||||
banned map[string]string
|
||||
// fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap.
|
||||
fills []fillRow
|
||||
// times is what the gate's gate_times answers.
|
||||
times string
|
||||
}
|
||||
|
||||
func (b bus) rows() map[string]*streamRow {
|
||||
@@ -87,6 +89,8 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) {
|
||||
return nil, errors.New("no stream called " + stream)
|
||||
}
|
||||
return json.Marshal(map[string]any{"detail": detail})
|
||||
case TimesTool:
|
||||
return json.RawMessage(b.times), nil
|
||||
case ClosedTool:
|
||||
if b.closed == "" {
|
||||
return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil
|
||||
@@ -133,6 +137,12 @@ func TestTheBusProbes(t *testing.T) {
|
||||
{"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]},
|
||||
{"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`},
|
||||
[]string{"bus.g14=2enode-tools.bus-reconnects"}, "", nil},
|
||||
// design 47 H2 §810: a core delivery 11 minutes after its merge call is over its 10 minutes; the next within clears it.
|
||||
{"D-budget a core delivery over", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_7","milestone":"main.7",
|
||||
"ended":"2026-11-22T00:11:00Z","delivery-time":"11m0s","longest-phase":"build","longest-phase-time":"7m0s","over":true}},
|
||||
"leaf":{"budget":"3m0s"}}`}, []string{"delivery.core.over-budget"}, "", nil},
|
||||
{"D-budget the next core delivery within", probeBudget, bus{times: `{"core":{"budget":"10m0s","last":{"record":"record/main_8",
|
||||
"milestone":"main.8","delivery-time":"4m0s","over":false}},"leaf":{"budget":"3m0s"}}`}, nil, "", nil},
|
||||
{"DW answers", probeWatchdogs, bus{}, nil, "", nil},
|
||||
{"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.watchdogs-silent"}, "", nil},
|
||||
{"H-bus healthy", probeBusHealth, bus{}, nil, "", nil},
|
||||
@@ -338,7 +348,7 @@ func agents(verdicts map[string]string) *memStore {
|
||||
// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4).
|
||||
func TestTheRegistryIsADR0564sTable(t *testing.T) {
|
||||
moved := []string{"D2", "D3", "D4", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard",
|
||||
"DW", "H-bus"}
|
||||
"DW", "H-bus", "D-budget"}
|
||||
var ids []string
|
||||
for _, p := range Registry() {
|
||||
ids = append(ids, p.ID)
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-catalog.registrations",
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-hand-acts.hand-act"
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"mesh-delivery-gate.gate_times"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-watchman.presence",
|
||||
|
||||
Reference in New Issue
Block a user