Keep each walk's phases and say a delivery over its budget (hq ADR 0282 slice 1, issue 382)

The operator's budget (a leaf module running everywhere within five minutes
of its merge, a core module within ten) cannot be held to without knowing
where a walk's time goes. A walk now keeps when its batch's window closed,
when it was cut and its class (migration 0093), each gate reading, and what
each machine of the rest was sent; 'delivery walks' and plan-moved say its
phases from the merge to every machine of the rest reporting the build
applied, and ended walks keep them as walk-phase durations per class. Probe
D16 reads mesh-delivery's 'times' and raises delivery.<class>.over-budget.
A phase that cannot be measured is said unknown, never zero. Measurement
only: no walk is held, sent or judged differently.

cmd/mesh-controller/delivery.go

@@ -596,12 +596,81 @@ func deliveryCommand(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
+		// Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6).
+		now := time.Now()
+		for i := range walks {
+			walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
+		}
 		return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
 			"own-path": sortedKeysOf(ownPathWords())})
 	}
 	return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
 }
 
+// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
+// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
+func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
+	return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
+		r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
+			return inventory.AppliedReport{}, false
+		}
+		return r, ok
+	}
+}
+
+// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind
+// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown
+// past ApplySilentAfter keeps its measured phases without its total.
+func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error {
+	recent, err := inv.RecentPlans(ctx, 30)
+	if err != nil {
+		return err
+	}
+	for _, p := range recent {
+		if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour {
+			continue
+		}
+		anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID)
+		if err != nil {
+			return err
+		}
+		if totalKept {
+			continue
+		}
+		ph := p.WalkPhases(now, appliedFrom(ctx, inv))
+		if ph != nil && ph.End == nil && anyKept {
+			continue // kept without its end; kept again only once its end is known
+		}
+		if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) {
+			continue
+		}
+		class := ph.Class
+		if class == "" {
+			class = "unclassed"
+		}
+		for _, x := range ph.Phases {
+			if x.State != inventory.PhaseMeasured || x.Start == nil {
+				continue
+			}
+			if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
+				Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start,
+				Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil {
+				return err
+			}
+		}
+		if ph.End != nil && ph.From != nil {
+			if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase,
+				Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From,
+				Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // ownPathWords is the controller's own path as words, for an answer.
 func ownPathWords() map[string]string { return onTheControllersPath }
 
@@ -728,6 +797,9 @@ func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
 }
 
 func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
+	// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
+	// read by whoever asks for the walk (`delivery walks`).
+	p.Phases = p.WalkPhases(time.Now(), nil)
 	body, err := json.Marshal(p)
 	if err != nil {
 		return