From eed497648304f69fbeb05d79fb0e6625b08c5b3b Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 23 Oct 2026 02:05:16 +0200 Subject: [PATCH] Resume a walk stopped by hand with plans retry, so a stop is not the end of what its tier waited on (issue 1886) --- cmd/mesh-controller/plan_retry.go | 90 +++++++++++++++++++ .../plan_retry_stopped_test.go | 89 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 cmd/mesh-controller/plan_retry_stopped_test.go diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 5db2d366..e9a55850 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -203,6 +203,21 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st return oneWalkAtATime(p, plans, onPath) } stopped := stoppedRollouts(p) + // **A walk stopped by hand while its tier waited is resumed** (issue 1886): nothing in it failed, so nothing is + // asked again; it goes on from where it stopped, unless a newer walk holds what it has yet to send. + if len(stopped) == 0 && stoppedByHandWaiting(p) { + if q, found := newerOpenPlan(p, plans); found { + return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would build "+ + "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) + } + for _, m := range yetToSend(p) { + if q, found := newerPlanFor(m, p, plans); found { + return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): resuming %s would send the older build of it "+ + "over the newer one", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID) + } + } + return oneWalkAtATime(p, plans, onPath) + } if len(stopped) == 0 { return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s", p.Tier, p.ID, p.Note) @@ -333,6 +348,8 @@ func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inv // retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at // that tier: what follows is the plan going on as if they had built the first time. +// A walk stopped by hand while its tier waited, with nothing failed, is resumed instead: nothing asked again +// (resumeStopped, issue 1886). func retryPlan(ctx context.Context, open *stores, id string) (string, error) { inv := open.inventory release, err := inv.HoldPlans(ctx, true) @@ -388,6 +405,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 { return retryTierWhole(ctx, open, &p, again) } + if len(failed) == 0 && len(stoppedRollouts(p)) == 0 && stoppedByHandWaiting(p) { + said := resumeStopped(&p) + if err := inv.SavePlan(ctx, &p); err != nil { + return "", err + } + return said, nil + } if len(failed) == 0 { return retryRollouts(ctx, open, &p) } @@ -583,3 +607,69 @@ func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[ } return failedIn(*p) } + +// stoppedByHandWaiting says a walk was stopped by hand while its current tier still waited (issue 1886): stopped by a +// person, nothing of the tier failed, and a module of it asked, built and not sent, or not yet asked. A stop that left +// merges out is not one: retryPlan refuses it before this is read (novox/hq ADR 0299). +func stoppedByHandWaiting(p inventory.Plan) bool { + if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped == "" || + p.Tier >= len(p.Tiers) || len(failedIn(p)) > 0 { + return false + } + asked, unsent, unasked := waitingIn(p) + return asked+unsent+unasked > 0 +} + +// waitingIn counts what a plan's current tier waits on: modules asked, built and not sent, not yet asked. +func waitingIn(p inventory.Plan) (asked, unsent, unasked int) { + if p.Tier >= len(p.Tiers) { + return 0, 0, 0 + } + for _, m := range p.Tiers[p.Tier] { + s := p.Modules[m] + switch { + case s == nil || s.State == "": + unasked++ + case s.State == "asked": + asked++ + case s.State == "built" && s.SentAt == nil: + unsent++ + } + } + return asked, unsent, unasked +} + +// yetToSend is every module a stopped walk has still to send: its current tier's modules not sent and every module of +// the tiers after, sorted. +func yetToSend(p inventory.Plan) []string { + var out []string + for i := p.Tier; i < len(p.Tiers); i++ { + for _, m := range p.Tiers[i] { + if s := p.Modules[m]; i > p.Tier || s == nil || s.SentAt == nil { + out = append(out, m) + } + } + } + sort.Strings(out) + return out +} + +// resumeStopped sets a walk stopped by hand building again at the tier it stopped at, its stop cleared and its asks kept +// (issue 1886): the next look carries it on as any walk — a build done waits for its reference, and one done on the +// builder's key is handed on (novox/hq ADR 0522 §3). Its note keeps who stopped it and that a person resumed it. +func resumeStopped(p *inventory.Plan) string { + asked, unsent, unasked := waitingIn(*p) + later := 0 + for i := p.Tier + 1; i < len(p.Tiers); i++ { + later += len(p.Tiers[i]) + } + by, why := p.Delivery.Stopped, p.Delivery.StoppedWhy + waits := fmt.Sprintf("%d asked, %d built and not sent, %d not yet asked in it; %d not yet asked in the tiers after", + asked, unsent, unasked, later) + p.State = inventory.PlanBuilding + goesOnAgain(p) + p.Delivery.Kept = nil + p.Note = fmt.Sprintf("tier %d resumed by hand after the stop by %s (%s): %s", p.Tier, by, why, waits) + return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; nothing asked again — the next look carries it "+ + "on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits) +} diff --git a/cmd/mesh-controller/plan_retry_stopped_test.go b/cmd/mesh-controller/plan_retry_stopped_test.go new file mode 100644 index 00000000..150fdfae --- /dev/null +++ b/cmd/mesh-controller/plan_retry_stopped_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A walk stopped by hand while its tier waited — modules asked on the builder's keys, two built and never sent, the next +// tier not yet asked — is resumed by `plans retry` (issue 1886): set building again, its stop cleared, the asks it holds +// kept, so the next look carries it on as any walk. Nothing in it failed, so the retry of failed builds had nothing to do +// and refused it, leaving a stopped R4e with no verb to carry its 48 modules on. +func TestRetryResumesAWalkStoppedByHand(t *testing.T) { + at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC) + sent := at.Add(-time.Hour) + p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16", + Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1, + Note: "stopped by a person at tier 1: issue 1886", + Delivery: &inventory.PlanDelivery{Stopped: "a person (fix-1886@shanks)", StoppedWhy: "issue 1886", + Kept: []inventory.PlanKept{{Module: "mesh-controller"}}}, + Modules: map[string]*inventory.PlanModule{}} + tier0 := []string{"mesh-controller", "mesh-sdk", "pacman", "route-proxy"} + for _, m := range tier0 { + p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent} + } + var tier1 []string + for _, m := range []string{"build-agent", "mesh-tools"} { + tier1 = append(tier1, m) + p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent} + } + onKeys := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node", "mesh-delivery-checker", + "mesh-delivery-forge-bridge", "mesh-delivery-gate", "mesh-delivery-indexer", "mesh-delivery-view"} + for _, m := range onKeys { + tier1 = append(tier1, m) + p.Modules[m] = &inventory.PlanModule{State: "asked", AskedAt: &sent, Key: true, Build: "build-" + m} + } + tier2 := []string{"gitea", "mesh-catalog", "mongodb"} + for _, m := range tier2 { + p.Modules[m] = &inventory.PlanModule{} + } + p.Tiers = [][]string{tier0, tier1, tier2} + + if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil { + t.Fatalf("a walk stopped by hand with its tier waiting was refused a retry: %v", err) + } + said := resumeStopped(&p) + if p.State != inventory.PlanBuilding { + t.Errorf("resumed walk is %s, wanted building", p.State) + } + if p.Delivery.Stopped != "" || p.Delivery.StoppedWhy != "" || len(p.Delivery.Kept) > 0 { + t.Errorf("resumed walk still says it stopped: %+v", p.Delivery) + } + if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(p.Note, "a person (fix-1886@shanks)") { + t.Errorf("the note does not record the hand resume and the stop it undid: %q", p.Note) + } + for _, m := range onKeys { + if s := p.Modules[m]; s.State != "asked" || !s.Key || s.Build != "build-"+m { + t.Errorf("%s's ask was not kept for the look to hand on: %+v", m, s) + } + } + for _, w := range []string{"plan-r4e", "tier 2 of 3", "8 asked", "2 built and not sent", "3 not yet asked"} { + if !strings.Contains(said, w) { + t.Errorf("the answer %q does not say %q", said, w) + } + } + + // What it still refuses: a walk stopped with nothing left in its tier, and one a related open walk runs beside. + done := p + done.State, done.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} + done.Modules = map[string]*inventory.PlanModule{} + for _, tier := range done.Tiers { + for _, m := range tier { + done.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent} + } + } + if err := retryRefusal(done, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 1") { + t.Errorf("a stopped walk with nothing left in its tier was retried: %v", err) + } + stopped := p + stopped.State, stopped.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} + beside := inventory.Plan{ID: "plan-442", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a", + Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}}, + Modules: map[string]*inventory.PlanModule{"mesh-sdk": {State: "asked"}}} + if err := retryRefusal(stopped, []inventory.Plan{beside}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") { + t.Errorf("a stopped walk was resumed beside a related open walk: %v", err) + } +}