novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 2 Releases Wiki Activity

Count a plan's time from its tier, and save it only when a step changed it (hq issue 296)

Browse Source 
plans said a build queued for minutes had been building for a few seconds: the line counted from
the last save, and every advance saved the plan whether or not it moved, bumping its revision and
saying plan-moved on the bus. The line, status and LATE now count from when the plan entered its
tier with the stalled condition's bound, and an advance that changes nothing writes nothing.
This commit is contained in:
jochen
2026-10-07 20:42:13 +02:00
parent 85d664438f
commit 875a4e5758
7 changed files with 255 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/release_plan.go
+70 -18
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
@@ -506,23 +507,32 @@ func advanceHeld(ctx context.Context, open *stores) {
for i := range plans {
p := &plans[i]
for p.Open() {
// **Saved only when the step changed it** (novox/hq issue 296). Every step was saved, and
// the steps run on every tick and after every build outcome on the mesh: a plan whose tier
// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
// said on the bus, and its last save read as when it began building.
before := planSnapshot(*p)
moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
if err != nil {
fmt.Printf("%s: %v\n", p.ID, err)
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
// the state is left as it was and the step is tried again on the next tick.
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
if planSnapshot(*p) != before {
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
}
}
break
}
if p.State == inventory.PlanFailed {
sayUnsent(p, rollsOut)
}
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
break
if planSnapshot(*p) != before {
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
break
}
}
if !moved {
break
@@ -531,6 +541,18 @@ func advanceHeld(ctx context.Context, open *stores) {
}
}
// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
// stamps itself (when it was saved, its revision and epoch, when its tier was entered) is left out.
func planSnapshot(p inventory.Plan) string {
p.Updated, p.Revision, p.Epoch, p.TierEntered = time.Time{}, 0, 0, time.Time{}
b, err := json.Marshal(p)
if err != nil {
// Unreadable is never the same: the plan is saved, as every step was before.
return fmt.Sprintf("unmarshallable %p %v", &p, err)
}
return string(b)
}
// advanceOnce takes one step of one plan and says whether anything changed.
func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
edges []inventory.Edge, rollsOut func(string) bool) (bool, error) {
@@ -1070,12 +1092,37 @@ func planTicker(ctx context.Context, open *stores) {
}
}
// planLine is one plan as `status` says it.
func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
// planLine is one plan as `status` says it, given the least bound of a tier.
func planLine(p inventory.Plan, now time.Time) string {
return planLineWith(p, now, pauseView{}, tierAtLeast)
}
// inTierSince is when a plan began waiting on what it waits on now, which `plans`, `status` and the
// watchdog of a plan's progress (S3) all count from (novox/hq issue 296): a walk waiting for its
// delivery's word from when it was opened, as S16 counts; any other plan from when it entered its tier,
// or, when the delivery let it go after that, from the go. Never from the plan's last save: a plan is
// saved for many reasons while its tier stands still, and a clock reset by each save said a build queued
// for minutes had been building for a few seconds. A plan read without the stamp (one from before it
// was kept) counts from its last save, the only time it has.
func inTierSince(p inventory.Plan) time.Time {
if p.Waiting() {
return p.Created
}
since := p.TierEntered
if since.IsZero() {
since = p.Updated
}
if d := p.Delivery; d != nil && d.Go != nil && d.Go.After(since) {
since = *d.Go
}
return since
}
// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
// waiting on builds nobody will take until a person resumes the seat says so, and is not late. bound is
// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition
// said on the line (novox/hq issue 296).
func planLineWith(p inventory.Plan, now time.Time, pause pauseView, bound time.Duration) string {
where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
switch p.State {
case inventory.PlanDone:
@@ -1085,7 +1132,7 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
case inventory.PlanSuperseded:
return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
}
since := now.Sub(p.Updated).Round(time.Second)
since := now.Sub(inTierSince(p)).Round(time.Second)
if p.Waiting() {
// Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239).
return fmt.Sprintf("%s %s %s, %s, for %s", p.Repository, short(p.Commit), where, waitingNote(p), since)
@@ -1094,7 +1141,7 @@ func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
}
late := ""
if since > planWaitBound {
if since > bound {
late = " — LATE"
}
what := "building"
@@ -1167,17 +1214,19 @@ type planStatus struct {
Late bool `json:"late"`
}
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) []planStatus {
out := make([]planStatus, 0, len(plans))
for _, p := range plans {
ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated}
if p.Open() {
ps.Since = inTierSince(p)
ps.Waiting = p.Note
if ps.Waiting == "" {
ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
}
ps.Late = now.Sub(p.Updated) > planWaitBound
// A walk waiting for its delivery's word is no tier late (ADR 0239): S16 is its watchdog.
ps.Late = !p.Waiting() && now.Sub(ps.Since) > bounds.of(p.Repository)
// Paused is a person's decision, not lateness (novox/hq ADR 0219).
if waiting, paused := pausedWaiting(p, pause, now); paused {
ps.Waiting, ps.Late = waiting, false
@@ -1189,16 +1238,16 @@ func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []plan
}
// openPlans is the open plans among the recent ones, and how many have waited past the bound.
func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
func openPlans(plans []inventory.Plan, now time.Time, pause pauseView, bounds tierBounds) ([]inventory.Plan, int) {
var open []inventory.Plan
late := 0
for _, p := range plans {
if p.Open() {
open = append(open, p)
if _, paused := pausedWaiting(p, pause, time.Now()); paused {
if _, paused := pausedWaiting(p, pause, now); paused || p.Waiting() {
continue
}
if time.Since(p.Updated) > planWaitBound {
if now.Sub(inTierSince(p)) > bounds.of(p.Repository) {
late++
}
}
@@ -1240,7 +1289,9 @@ func plansCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
bounds := readTierBounds(ctx, inv, now)
fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p}),
bounds.of(p.Repository)))
if r := p.Release; r != nil {
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
@@ -1369,8 +1420,9 @@ func plansCommand(ctx context.Context, args []string) error {
return nil
}
pause := buildSeatPause(ctx, inv, plans)
bounds := readTierBounds(ctx, inv, now)
for _, p := range plans {
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause, bounds.of(p.Repository)))
}
return nil
}