Show and change the build queue through the controller, and have plans follow it (hq ADR 0219)

Nothing showed what waited for a build machine, and an ask could not be dropped without leaving the plan that made it waiting for ever. New verbs: queue, cancel, clear, rebuild, replay, kill, pause, resume, and plans retry. Every ask a person drops is recorded failed through the same take-in as a failed build; a plan keeps the id it asked each module under and matches its outcome by it. replay is a dry run unless registered, and registering an older commit than one registered since needs --older (hq issue 207). A plan waiting on a seat paused on every holder says so and is not late; a failed plan can be retried, and a rebuild joins the plan holding the module instead of running beside it.
2026-10-05 19:17:56 +02:00
parent e610f2d92c
commit 106507b1d3
13 changed files with 1772 additions and 58 deletions
@@ -324,37 +324,52 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
 	for _, e := range entries {
 		byName[e.Manifest.Module] = e
 	}
-	now := time.Now().UTC()
 	for _, name := range p.Tiers[p.Tier] {
-		state := p.Modules[name]
-		if state == nil {
-			state = &inventory.PlanModule{}
-			p.Modules[name] = state
-		}
-		e, known := byName[name]
-		if !known {
-			state.State = "failed"
-			state.Why = "no longer in the catalogue"
-			p.State = inventory.PlanFailed
-			p.Note = name + " is no longer in the catalogue"
-			continue
-		}
-		source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
-		fmt.Printf("  tier %d: ", p.Tier)
-		// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
-		if err := buildOne(ctx, source, e.Source.Path, followedBranch(e.Source.Ref), 0); err != nil {
-			state.State = "failed"
-			state.Why = err.Error()
-			p.State = inventory.PlanFailed
-			p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
-			continue
-		}
-		state.State = "asked"
-		state.AskedAt = &now
+		askModule(ctx, p, name, byName)
 	}
 	return nil
 }

+// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
+// variable so a test of what a plan does around an ask needs no build machine.
+var askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
+	return buildOneAsked(ctx, source, path, ref, 0, false)
+}
+
+// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
+// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
+func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
+	now := time.Now().UTC()
+	state := p.Modules[name]
+	if state == nil {
+		state = &inventory.PlanModule{}
+		p.Modules[name] = state
+	}
+	e, known := byName[name]
+	if !known {
+		state.State = "failed"
+		state.Why = "no longer in the catalogue"
+		p.State = inventory.PlanFailed
+		p.Note = name + " is no longer in the catalogue"
+		return
+	}
+	source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
+	fmt.Printf("  tier %d: ", p.Tier)
+	// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215).
+	id, err := askABuild(ctx, source, e.Source.Path, followedBranch(e.Source.Ref))
+	if err != nil {
+		state.State = "failed"
+		state.Why = err.Error()
+		p.State = inventory.PlanFailed
+		p.Note = fmt.Sprintf("%s could not be asked for: %v", name, err)
+		return
+	}
+	state.State = "asked"
+	state.AskedAt = &now
+	state.Build = id
+	state.Why, state.Commit, state.BuiltAt = "", "", nil
+}
+
 // planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
 // advances what that completes. Called from the daemon's take-in of every outcome.
 //
@@ -363,7 +378,7 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
 // the later plan's answer — it stood on the bases from before the later plan's merge, and taking it
 // would send machines, and the next tier, what the later merge replaced. asked is zero when the
 // build's request time is not known, and such an outcome is taken as before.
-func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time) {
+func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
 	inv := open.inventory
 	// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
 	// than write over what the holder is about to save. Not taken, it is still in the build records,
@@ -399,7 +414,9 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 			state = &inventory.PlanModule{}
 			p.Modules[module] = state
 		}
-		if askedBefore(asked, state.AskedAt) {
+		// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
+		// is, as before, when it was asked at or after the plan's ask (issue 219).
+		if !(id != "" && state.Build == id) && askedBefore(asked, state.AskedAt) {
 			continue
 		}
 		if failed != "" {
@@ -523,6 +540,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
 	// outcome, whoever was listening.
 	recorded := map[string][]inventory.Build{}
+	byID := map[string]inventory.Build{}
 	for _, m := range tier {
 		if s := p.Modules[m]; s != nil && s.State == "asked" {
 			builds, err := inv.Builds(ctx, m, 5)
@@ -530,9 +548,19 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 				return false, err
 			}
 			recorded[m] = builds
+			// Its own ask's record, by id — found even when the outcome named no module (ADR 0219).
+			if s.Build != "" {
+				b, found, err := inv.BuildByID(ctx, s.Build)
+				if err != nil {
+					return false, err
+				}
+				if found {
+					byID[s.Build] = b
+				}
+			}
 		}
 	}
-	if settleFromRecords(p, tier, recorded) {
+	if settleFromRecords(p, tier, recorded, byID) {
 		return true, nil
 	}
 	// Asked: wait for every build.
@@ -811,7 +839,11 @@ func planTicker(ctx context.Context, open *stores) {
 }

 // planLine is one plan as `status` says it.
-func planLine(p inventory.Plan, now time.Time) string {
+func planLine(p inventory.Plan, now time.Time) string { return planLineWith(p, now, pauseView{}) }
+
+// planLineWith is planLine knowing whether the build seat is paused (novox/hq ADR 0219): a plan
+// waiting on builds nobody will take until a person resumes the seat says so, and is not late.
+func planLineWith(p inventory.Plan, now time.Time, pause pauseView) string {
 	where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers))
 	switch p.State {
 	case inventory.PlanDone:
@@ -822,6 +854,9 @@ func planLine(p inventory.Plan, now time.Time) string {
 		return fmt.Sprintf("%s %s %s", p.Repository, short(p.Commit), p.Note)
 	}
 	since := now.Sub(p.Updated).Round(time.Second)
+	if waiting, paused := pausedWaiting(p, pause, now); paused {
+		return fmt.Sprintf("%s %s %s, %s", p.Repository, short(p.Commit), where, waiting)
+	}
 	late := ""
 	if since > planWaitBound {
 		late = " — LATE"
@@ -835,20 +870,49 @@ func planLine(p inventory.Plan, now time.Time) string {

 // planFailedBuild marks the module a failed build was for when the result names no module: by the
 // repository and path the plan's modules were asked at.
+//
+// **By the id first** (novox/hq ADR 0219): a plan keeps the id it asked each module under, so an
+// outcome that never learnt its module's name — cancelled, killed, failed at the clone — is matched
+// to the module it was asked for exactly. Repository and path remain for a plan from before ids
+// were kept.
 func planFailedBuild(ctx context.Context, open *stores, result link.BuildResult) {
+	asked, _ := link.BuildAskedAt(result.ID)
+	if plans, err := open.inventory.OpenPlans(ctx); err == nil {
+		if module := moduleAskedAs(plans, result.ID); module != "" {
+			planBuilt(ctx, open, module, result.Commit, result.Failed, asked, result.ID)
+			return
+		}
+	}
 	entries, err := open.inventory.Catalogued(ctx)
 	if err != nil {
 		return
 	}
 	for _, e := range entries {
 		if repositoryMatches(e.Source.Repository, result.Repository) && e.Source.Path == result.Path {
-			asked, _ := link.BuildAskedAt(result.ID)
-			planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked)
+			planBuilt(ctx, open, e.Manifest.Module, result.Commit, result.Failed, asked, result.ID)
 			return
 		}
 	}
 }

+// moduleAskedAs is the module an open plan's current tier asked for under this id, or nothing.
+func moduleAskedAs(plans []inventory.Plan, id string) string {
+	if id == "" {
+		return ""
+	}
+	for _, p := range plans {
+		if p.Tier >= len(p.Tiers) {
+			continue
+		}
+		for _, m := range p.Tiers[p.Tier] {
+			if s := p.Modules[m]; s != nil && s.Build == id {
+				return m
+			}
+		}
+	}
+	return ""
+}
+
 func repositoryMatches(a, b string) bool {
 	trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) }
 	return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a))
@@ -867,7 +931,7 @@ type planStatus struct {
 	Late       bool      `json:"late"`
 }

-func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
+func planStatuses(plans []inventory.Plan, now time.Time, pause pauseView) []planStatus {
 	out := make([]planStatus, 0, len(plans))
 	for _, p := range plans {
 		ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State,
@@ -878,6 +942,10 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
 				ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier)
 			}
 			ps.Late = now.Sub(p.Updated) > planWaitBound
+			// Paused is a person's decision, not lateness (novox/hq ADR 0219).
+			if waiting, paused := pausedWaiting(p, pause, now); paused {
+				ps.Waiting, ps.Late = waiting, false
+			}
 		}
 		out = append(out, ps)
 	}
@@ -885,12 +953,15 @@ func planStatuses(plans []inventory.Plan, now time.Time) []planStatus {
 }

 // openPlans is the open plans among the recent ones, and how many have waited past the bound.
-func openPlans(plans []inventory.Plan) ([]inventory.Plan, int) {
+func openPlans(plans []inventory.Plan, pause pauseView) ([]inventory.Plan, int) {
 	var open []inventory.Plan
 	late := 0
 	for _, p := range plans {
 		if p.Open() {
 			open = append(open, p)
+			if _, paused := pausedWaiting(p, pause, time.Now()); paused {
+				continue
+			}
 			if time.Since(p.Updated) > planWaitBound {
 				late++
 			}
@@ -923,7 +994,7 @@ func plansCommand(ctx context.Context, args []string) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s — %s\n", p.ID, planLine(p, now))
+		fmt.Printf("%s — %s\n", p.ID, planLineWith(p, now, buildSeatPause(ctx, inv, []inventory.Plan{p})))
 		for i, tier := range p.Tiers {
 			marker := " "
 			if i == p.Tier && p.Open() {
@@ -938,6 +1009,9 @@ func plansCommand(ctx context.Context, args []string) error {
 					if s.Commit != "" {
 						state += " from " + short(s.Commit)
 					}
+					if s.Build != "" && s.State != "built" {
+						state += " (" + s.Build + ")"
+					}
 					if s.Why != "" {
 						state += ": " + s.Why
 					}
@@ -950,6 +1024,15 @@ func plansCommand(ctx context.Context, args []string) error {
 	if *whatIf != "" {
 		return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules))
 	}
+	// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
+	if len(positionals) == 2 && positionals[0] == "retry" {
+		said, err := retryPlan(ctx, open, positionals[1])
+		if err != nil {
+			return err
+		}
+		fmt.Println(said)
+		return nil
+	}
 	// `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one
 	// waiting on a report that cannot come — so it stops reading as work in progress. Marked failed
 	// with who ended it; what it asked still builds and registers.
@@ -991,8 +1074,9 @@ func plansCommand(ctx context.Context, args []string) error {
 		fmt.Println("no merge has produced a plan yet")
 		return nil
 	}
+	pause := buildSeatPause(ctx, inv, plans)
 	for _, p := range plans {
-		fmt.Printf("%-28s %s\n", p.ID, planLine(p, now))
+		fmt.Printf("%-28s %s\n", p.ID, planLineWith(p, now, pause))
 	}
 	return nil
 }
@@ -1094,7 +1178,12 @@ func splitList(s string) []string {
 // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
 // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
 // Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
-func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build) bool {
+//
+// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR
+// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is
+// found by nothing else.
+func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build,
+	byID map[string]inventory.Build) bool {
 	changed := false
 	for _, m := range tier {
 		s := p.Modules[m]
@@ -1114,6 +1203,9 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 			}
 			outcome = &b
 		}
+		if own, found := byID[s.Build]; s.Build != "" && found {
+			outcome = &own
+		}
 		if outcome == nil {
 			continue
 		}