Keep each walk's phases and say a delivery over its budget (hq ADR 0282 slice 1, issue 382)

The operator's budget (a leaf module running everywhere within five minutes
of its merge, a core module within ten) cannot be held to without knowing
where a walk's time goes. A walk now keeps when its batch's window closed,
when it was cut and its class (migration 0093), each gate reading, and what
each machine of the rest was sent; 'delivery walks' and plan-moved say its
phases from the merge to every machine of the rest reporting the build
applied, and ended walks keep them as walk-phase durations per class. Probe
D16 reads mesh-delivery's 'times' and raises delivery.<class>.over-budget.
A phase that cannot be measured is said unknown, never zero. Measurement
only: no walk is held, sent or judged differently.

cmd/mesh-controller/over_budget.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads
+// mesh-delivery's `times` and raises the warning `delivery.<class>.over-budget` when the newest delivery of a
+// class whose delivery time is known took longer than its class's budget — five minutes for a leaf module, ten
+// for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class
+// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery.
+
+// probeBudgetsID is the probe that reads the delivery times.
+const probeBudgetsID = "D16"
+
+// kindOverBudget is what a delivery over its class's budget raises.
+const kindOverBudget = "over-budget"
+
+// deliveryBudgets are the budgets by class (ADR 0282 decision 1); the probe raises nothing for another class.
+var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 5 * time.Minute, inventory.ClassCore: 10 * time.Minute}
+
+// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it.
+type timesAnswer struct {
+	Classes []timesClass `json:"classes"`
+}
+
+// timesClass is one class's line of `times`.
+type timesClass struct {
+	Class  string       `json:"class"`
+	Latest *timesLatest `json:"latest,omitempty"`
+}
+
+// timesLatest is the newest delivery of a class whose delivery time is known.
+type timesLatest struct {
+	ID      string `json:"id"`
+	TookMS  int64  `json:"took_ms"`
+	Longest string `json:"longest,omitempty"`
+	Landed  string `json:"landed,omitempty"`
+}
+
+// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or
+// none answers (D3 says that one).
+func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) {
+	if !held {
+		return nil, nil
+	}
+	raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{})
+	if errors.Is(err, link.ErrNothingServes) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var a timesAnswer
+	if err := json.Unmarshal(raw, &a); err != nil {
+		return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err)
+	}
+	return &a, nil
+}
+
+// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget:
+// strictly longer, so a delivery of exactly its budget is within it.
+func overBudgetObservations(a *timesAnswer) []conditions.Observation {
+	if a == nil {
+		return nil
+	}
+	var out []conditions.Observation
+	for _, c := range a.Classes {
+		budget, ok := deliveryBudgets[c.Class]
+		if !ok || c.Latest == nil {
+			continue
+		}
+		took := time.Duration(c.Latest.TookMS) * time.Millisecond
+		if took <= budget {
+			continue
+		}
+		longest := c.Latest.Longest
+		if longest == "" {
+			longest = "not known"
+		}
+		out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget,
+			Severity: conditions.Warning,
+			Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+
+				"its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took),
+				humanDuration(budget), longest),
+			Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest),
+			Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took),
+				humanDuration(budget)),
+			Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+
+				"held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+
+				"a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest),
+			Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)),
+		})
+	}
+	return out
+}
+
+// probeBudgets is D16: the newest delivery of each class lands within its class's budget.
+func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
+	entries, err := d.open.inventory.Catalogued(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var conn *nats.Conn
+	if d.js != nil {
+		conn = d.js.Conn()
+	}
+	a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries))
+	if err != nil {
+		return nil, err
+	}
+	return overBudgetObservations(a), nil
+}