Keep each walk's phases and say a delivery over its budget (hq ADR 0282 slice 1, issue 382)

The operator's budget (a leaf module running everywhere within five minutes
of its merge, a core module within ten) cannot be held to without knowing
where a walk's time goes. A walk now keeps when its batch's window closed,
when it was cut and its class (migration 0093), each gate reading, and what
each machine of the rest was sent; 'delivery walks' and plan-moved say its
phases from the merge to every machine of the rest reporting the build
applied, and ended walks keep them as walk-phase durations per class. Probe
D16 reads mesh-delivery's 'times' and raises delivery.<class>.over-budget.
A phase that cannot be measured is said unknown, never zero. Measurement
only: no walk is held, sent or judged differently.

internal/inventory/plans.go

@@ -54,6 +54,25 @@ type Plan struct {
 	// walk can carry several. Repository and Commit above keep one of them, for a reader that knows one. Empty
 	// for a walk kept before it was: Repository and Commit are then the whole of it.
 	Commits []PlanCommit `json:"commits,omitempty"`
+	// Times are the moments of a walk no other field keeps (novox/hq ADR 0282 decision 6): when its batch's
+	// window closed, when it was cut, and its class. Nil for a walk kept before they were, whose phases before
+	// its build are said unknown.
+	Times *PlanTimes `json:"times,omitempty"`
+	// Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out
+	// from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`).
+	Phases *WalkPhases `json:"phases,omitempty"`
+}
+
+// PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6).
+type PlanTimes struct {
+	// WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum.
+	// Nil for a walk no window assembled (one merge walked alone).
+	WindowClosed *time.Time `json:"window_closed,omitempty"`
+	// Cut is when the batch became the walk.
+	Cut *time.Time `json:"cut,omitempty"`
+	// Class is the walk's module class, read at its cut: core when it moves a module on the controller's own
+	// path or the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
+	Class string `json:"class,omitempty"`
 }
 
 // PlanCommit is one repository's commit a walk carries: the latest merge of its branch in the batch.
@@ -76,6 +95,10 @@ type PlanMerge struct {
 	Number int      `json:"number,omitempty"`
 	Title  string   `json:"title,omitempty"`
 	Moves  []string `json:"moves,omitempty"`
+	// Merged is when the forge made the merge and Heard when the controller heard it (novox/hq ADR 0282): where
+	// its delivery time starts. Zero for a merge named before they were kept.
+	Merged time.Time `json:"merged,omitzero"`
+	Heard  time.Time `json:"heard,omitzero"`
 }
 
 // PlanBatch is a batch's window while it is one (novox/hq ADR 0276): when it closes unless another merge
@@ -220,6 +243,9 @@ type PlanModule struct {
 	// went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for
 	// the module the gate is kept on, and for a plan from before tiers were sent whole.
 	GatedBy string `json:"gated_by,omitempty"`
+	// Rest is, per machine of the rest, the declaration the send after the gate carried there (novox/hq ADR
+	// 0282 decision 6): the machine's first report of it, applied, is when the build runs there.
+	Rest map[string]SentDeclaration `json:"rest,omitempty"`
 }
 
 // PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
@@ -276,6 +302,35 @@ type PlanGate struct {
 	BrokenWhy string   `json:"broken_why,omitempty"`
 	// Returned names the broken modules already put back, at once, while the rest of the send is judged.
 	Returned []string `json:"returned,omitempty"`
+	// Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that
+	// counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept.
+	Readings []GateReading `json:"readings,omitempty"`
+}
+
+// GateReading is one reading of a first-node gate.
+type GateReading struct {
+	At      time.Time `json:"at"`
+	Healthy bool      `json:"healthy"`
+	// Said is what a reading that did not pass found wanting.
+	Said string `json:"said,omitempty"`
+}
+
+// maxReadings bounds a gate's readings: a judging that never passes reads every few seconds for ten minutes.
+const maxReadings = 24
+
+// Read keeps one reading: a pass always, and a reading that did not pass only when the one before passed or
+// there is none, so a judging waiting for a module to start keeps one line of it, not hundreds.
+func (g *PlanGate) Read(at time.Time, healthy bool, said string) {
+	if !healthy && len(g.Readings) > 0 && !g.Readings[len(g.Readings)-1].Healthy {
+		return
+	}
+	if len(said) > 200 {
+		said = said[:200]
+	}
+	g.Readings = append(g.Readings, GateReading{At: at, Healthy: healthy, Said: said})
+	if len(g.Readings) > maxReadings {
+		g.Readings = g.Readings[len(g.Readings)-maxReadings:]
+	}
 }
 
 // CarriedMove is one module's build moving on a machine with a gated send.
@@ -342,7 +397,12 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
 	if err != nil {
 		return err
 	}
-	var release, delivery, commits []byte
+	var release, delivery, commits, times []byte
+	if p.Times != nil {
+		if times, err = json.Marshal(p.Times); err != nil {
+			return err
+		}
+	}
 	if len(p.Commits) > 0 {
 		if commits, err = json.Marshal(p.Commits); err != nil {
 			return err
@@ -373,18 +433,18 @@ func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
 	var revision int64
 	err = tx.QueryRow(ctx,
 		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
-		                           branch, tier_entered, revision, epoch, release, delivery, merged_at, commits)
-		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17)
+		                           branch, tier_entered, revision, epoch, release, delivery, merged_at, commits, times)
+		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17, $18)
 		 on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
 		        tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
 		        tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
 		        release = excluded.release, delivery = excluded.delivery, repository = excluded.repository,
 		        commit_hash = excluded.commit_hash, merged_at = excluded.merged_at, commits = excluded.commits,
-		        created = excluded.created
+		        created = excluded.created, times = excluded.times
 		  where release_plan.revision = $12
 		 returning revision`,
 		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
-		p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits).Scan(&revision)
+		p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits, times).Scan(&revision)
 	if errors.Is(err, pgx.ErrNoRows) {
 		// The row is there and at another revision — moved since this was read, or there already
 		// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
@@ -467,7 +527,7 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
 func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
-		        coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits
+		        coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits, times
 		   from release_plan `+tail, args...)
 	if err != nil {
 		return nil, err
@@ -476,14 +536,19 @@ func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan
 	var out []Plan
 	for rows.Next() {
 		var p Plan
-		var tiers, modules, release, delivery, commits []byte
+		var tiers, modules, release, delivery, commits, times []byte
 		var epoch int64
 		var merged *time.Time
 		if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
 			&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release,
-			&delivery, &merged, &commits); err != nil {
+			&delivery, &merged, &commits, &times); err != nil {
 			return nil, err
 		}
+		if len(times) > 0 {
+			if err := json.Unmarshal(times, &p.Times); err != nil {
+				return nil, err
+			}
+		}
 		if len(commits) > 0 {
 			if err := json.Unmarshal(commits, &p.Commits); err != nil {
 				return nil, err