Count a plan's time from its tier, and save it only when a step changed it (hq issue 296)

plans said a build queued for minutes had been building for a few seconds: the line counted from
the last save, and every advance saved the plan whether or not it moved, bumping its revision and
saying plan-moved on the bus. The line, status and LATE now count from when the plan entered its
tier with the stalled condition's bound, and an advance that changes nothing writes nothing.

cmd/mesh-controller/watchdogs.go

@@ -439,13 +439,9 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
 	if len(plans) == 0 {
 		return nil, nil, nil
 	}
-	tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
+	bounds, err := measuredTierBounds(ctx, inv, now)
 	if err != nil {
-		return nil, nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
-	}
-	measured := map[string][]time.Duration{}
-	for _, d := range tiers {
-		measured[d.Subject] = append(measured[d.Subject], d.Took)
+		return nil, nil, err
 	}
 	pause := buildSeatPause(ctx, inv, plans)
 	var out []planFacts
@@ -459,13 +455,59 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) (
 			continue
 		}
 		_, paused := pausedWaiting(p, pause, now)
+		bound := bounds.of(p.Repository)
 		out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,
-			tiers: len(p.Tiers), entered: p.TierEntered, bound: max(tierAtLeast, 3*p90(measured[p.Repository])),
-			waiting: planLineWith(p, now, pause), paused: paused})
+			tiers: len(p.Tiers), entered: inTierSince(p), bound: bound,
+			waiting: planLineWith(p, now, pause, bound), paused: paused})
 	}
 	return out, waits, nil
 }
 
+// tierBounds is how long a plan's tier may take before it is stalled (S3) and `plans` and `status` call
+// it LATE, by repository: three times the ninetieth percentile of the tiers measured for it, and never
+// less than tierAtLeast. One bound for the watchdog and the lines a person reads, so the two cannot
+// disagree about one plan (novox/hq issue 296). A repository nothing was measured for, or a nil
+// tierBounds, is given tierAtLeast.
+type tierBounds map[string]time.Duration
+
+func (b tierBounds) of(repository string) time.Duration {
+	if d, ok := b[repository]; ok {
+		return d
+	}
+	return tierAtLeast
+}
+
+// measuredTierBounds is the tier bounds from the last fourteen days of measured plan tiers.
+func measuredTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) (tierBounds, error) {
+	tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour))
+	if err != nil {
+		return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err)
+	}
+	return boundsOf(tiers), nil
+}
+
+// readTierBounds is measuredTierBounds for a line a person reads: what cannot be read is said, and
+// every plan is then given the least bound, as the watchdog gives a repository nothing was measured for.
+func readTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) tierBounds {
+	bounds, err := measuredTierBounds(ctx, inv, now)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v; a plan is called late past %s\n", err, tierAtLeast)
+	}
+	return bounds
+}
+
+func boundsOf(tiers []inventory.Duration) tierBounds {
+	measured := map[string][]time.Duration{}
+	for _, d := range tiers {
+		measured[d.Subject] = append(measured[d.Subject], d.Took)
+	}
+	out := tierBounds{}
+	for repository, took := range measured {
+		out[repository] = max(tierAtLeast, 3*p90(took))
+	}
+	return out
+}
+
 // p90 is the ninetieth percentile of measurements; zero for none.
 func p90(took []time.Duration) time.Duration {
 	if len(took) == 0 {