Resume a walk stopped by hand with plans retry, so a stop is not the end of what its tier waited on (issue 1886)

2026-10-23 02:05:16 +02:00
parent bbadbb2af0
commit eed4976483
2 changed files with 179 additions and 0 deletions
@@ -203,6 +203,21 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st
 		return oneWalkAtATime(p, plans, onPath)
 	}
 	stopped := stoppedRollouts(p)
+	// **A walk stopped by hand while its tier waited is resumed** (issue 1886): nothing in it failed, so nothing is
+	// asked again; it goes on from where it stopped, unless a newer walk holds what it has yet to send.
+	if len(stopped) == 0 && stoppedByHandWaiting(p) {
+		if q, found := newerOpenPlan(p, plans); found {
+			return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would build "+
+				"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
+		}
+		for _, m := range yetToSend(p) {
+			if q, found := newerPlanFor(m, p, plans); found {
+				return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): resuming %s would send the older build of it "+
+					"over the newer one", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
+			}
+		}
+		return oneWalkAtATime(p, plans, onPath)
+	}
 	if len(stopped) == 0 {
 		return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
 			p.Tier, p.ID, p.Note)
@@ -333,6 +348,8 @@ func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inv

 // retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
 // that tier: what follows is the plan going on as if they had built the first time.
+// A walk stopped by hand while its tier waited, with nothing failed, is resumed instead: nothing asked again
+// (resumeStopped, issue 1886).
 func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	inv := open.inventory
 	release, err := inv.HoldPlans(ctx, true)
@@ -388,6 +405,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
 		return retryTierWhole(ctx, open, &p, again)
 	}
+	if len(failed) == 0 && len(stoppedRollouts(p)) == 0 && stoppedByHandWaiting(p) {
+		said := resumeStopped(&p)
+		if err := inv.SavePlan(ctx, &p); err != nil {
+			return "", err
+		}
+		return said, nil
+	}
 	if len(failed) == 0 {
 		return retryRollouts(ctx, open, &p)
 	}
@@ -583,3 +607,69 @@ func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[
 	}
 	return failedIn(*p)
 }
+
+// stoppedByHandWaiting says a walk was stopped by hand while its current tier still waited (issue 1886): stopped by a
+// person, nothing of the tier failed, and a module of it asked, built and not sent, or not yet asked. A stop that left
+// merges out is not one: retryPlan refuses it before this is read (novox/hq ADR 0299).
+func stoppedByHandWaiting(p inventory.Plan) bool {
+	if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped == "" ||
+		p.Tier >= len(p.Tiers) || len(failedIn(p)) > 0 {
+		return false
+	}
+	asked, unsent, unasked := waitingIn(p)
+	return asked+unsent+unasked > 0
+}
+
+// waitingIn counts what a plan's current tier waits on: modules asked, built and not sent, not yet asked.
+func waitingIn(p inventory.Plan) (asked, unsent, unasked int) {
+	if p.Tier >= len(p.Tiers) {
+		return 0, 0, 0
+	}
+	for _, m := range p.Tiers[p.Tier] {
+		s := p.Modules[m]
+		switch {
+		case s == nil || s.State == "":
+			unasked++
+		case s.State == "asked":
+			asked++
+		case s.State == "built" && s.SentAt == nil:
+			unsent++
+		}
+	}
+	return asked, unsent, unasked
+}
+
+// yetToSend is every module a stopped walk has still to send: its current tier's modules not sent and every module of
+// the tiers after, sorted.
+func yetToSend(p inventory.Plan) []string {
+	var out []string
+	for i := p.Tier; i < len(p.Tiers); i++ {
+		for _, m := range p.Tiers[i] {
+			if s := p.Modules[m]; i > p.Tier || s == nil || s.SentAt == nil {
+				out = append(out, m)
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// resumeStopped sets a walk stopped by hand building again at the tier it stopped at, its stop cleared and its asks kept
+// (issue 1886): the next look carries it on as any walk — a build done waits for its reference, and one done on the
+// builder's key is handed on (novox/hq ADR 0522 §3). Its note keeps who stopped it and that a person resumed it.
+func resumeStopped(p *inventory.Plan) string {
+	asked, unsent, unasked := waitingIn(*p)
+	later := 0
+	for i := p.Tier + 1; i < len(p.Tiers); i++ {
+		later += len(p.Tiers[i])
+	}
+	by, why := p.Delivery.Stopped, p.Delivery.StoppedWhy
+	waits := fmt.Sprintf("%d asked, %d built and not sent, %d not yet asked in it; %d not yet asked in the tiers after",
+		asked, unsent, unasked, later)
+	p.State = inventory.PlanBuilding
+	goesOnAgain(p)
+	p.Delivery.Kept = nil
+	p.Note = fmt.Sprintf("tier %d resumed by hand after the stop by %s (%s): %s", p.Tier, by, why, waits)
+	return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; nothing asked again — the next look carries it "+
+		"on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits)
+}
@@ -0,0 +1,89 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A walk stopped by hand while its tier waited — modules asked on the builder's keys, two built and never sent, the next
+// tier not yet asked — is resumed by `plans retry` (issue 1886): set building again, its stop cleared, the asks it holds
+// kept, so the next look carries it on as any walk. Nothing in it failed, so the retry of failed builds had nothing to do
+// and refused it, leaving a stopped R4e with no verb to carry its 48 modules on.
+func TestRetryResumesAWalkStoppedByHand(t *testing.T) {
+	at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC)
+	sent := at.Add(-time.Hour)
+	p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16",
+		Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1,
+		Note: "stopped by a person at tier 1: issue 1886",
+		Delivery: &inventory.PlanDelivery{Stopped: "a person (fix-1886@shanks)", StoppedWhy: "issue 1886",
+			Kept: []inventory.PlanKept{{Module: "mesh-controller"}}},
+		Modules: map[string]*inventory.PlanModule{}}
+	tier0 := []string{"mesh-controller", "mesh-sdk", "pacman", "route-proxy"}
+	for _, m := range tier0 {
+		p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
+	}
+	var tier1 []string
+	for _, m := range []string{"build-agent", "mesh-tools"} {
+		tier1 = append(tier1, m)
+		p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent}
+	}
+	onKeys := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node", "mesh-delivery-checker",
+		"mesh-delivery-forge-bridge", "mesh-delivery-gate", "mesh-delivery-indexer", "mesh-delivery-view"}
+	for _, m := range onKeys {
+		tier1 = append(tier1, m)
+		p.Modules[m] = &inventory.PlanModule{State: "asked", AskedAt: &sent, Key: true, Build: "build-" + m}
+	}
+	tier2 := []string{"gitea", "mesh-catalog", "mongodb"}
+	for _, m := range tier2 {
+		p.Modules[m] = &inventory.PlanModule{}
+	}
+	p.Tiers = [][]string{tier0, tier1, tier2}
+
+	if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil {
+		t.Fatalf("a walk stopped by hand with its tier waiting was refused a retry: %v", err)
+	}
+	said := resumeStopped(&p)
+	if p.State != inventory.PlanBuilding {
+		t.Errorf("resumed walk is %s, wanted building", p.State)
+	}
+	if p.Delivery.Stopped != "" || p.Delivery.StoppedWhy != "" || len(p.Delivery.Kept) > 0 {
+		t.Errorf("resumed walk still says it stopped: %+v", p.Delivery)
+	}
+	if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(p.Note, "a person (fix-1886@shanks)") {
+		t.Errorf("the note does not record the hand resume and the stop it undid: %q", p.Note)
+	}
+	for _, m := range onKeys {
+		if s := p.Modules[m]; s.State != "asked" || !s.Key || s.Build != "build-"+m {
+			t.Errorf("%s's ask was not kept for the look to hand on: %+v", m, s)
+		}
+	}
+	for _, w := range []string{"plan-r4e", "tier 2 of 3", "8 asked", "2 built and not sent", "3 not yet asked"} {
+		if !strings.Contains(said, w) {
+			t.Errorf("the answer %q does not say %q", said, w)
+		}
+	}
+
+	// What it still refuses: a walk stopped with nothing left in its tier, and one a related open walk runs beside.
+	done := p
+	done.State, done.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
+	done.Modules = map[string]*inventory.PlanModule{}
+	for _, tier := range done.Tiers {
+		for _, m := range tier {
+			done.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent}
+		}
+	}
+	if err := retryRefusal(done, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 1") {
+		t.Errorf("a stopped walk with nothing left in its tier was retried: %v", err)
+	}
+	stopped := p
+	stopped.State, stopped.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
+	beside := inventory.Plan{ID: "plan-442", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a",
+		Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}},
+		Modules: map[string]*inventory.PlanModule{"mesh-sdk": {State: "asked"}}}
+	if err := retryRefusal(stopped, []inventory.Plan{beside}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
+		t.Errorf("a stopped walk was resumed beside a related open walk: %v", err)
+	}
+}