From 248fe521a39e2c5e41edfc552c9a05a59c022e73 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Mon, 19 Oct 2026 21:26:43 +0000 Subject: [PATCH] Raise plan-failed for a failed delivery plan, with Retry when a retry would run (issue 1314, ADR 0449) --- cmd/mesh-controller/plain_words.go | 5 + cmd/mesh-controller/plan_failed.go | 213 ++++++++++++++++++++++++ cmd/mesh-controller/plan_failed_test.go | 184 ++++++++++++++++++++ cmd/mesh-controller/signals.go | 38 +++++ cmd/mesh-controller/signals_test.go | 11 ++ cmd/mesh-controller/watchdogs.go | 4 + 6 files changed, 455 insertions(+) create mode 100644 cmd/mesh-controller/plan_failed.go create mode 100644 cmd/mesh-controller/plan_failed_test.go diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index f07fa940..965b8892 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -429,6 +429,11 @@ var plainWordings = map[string]func(conditions.Observation) words{ "out) has not let it start yet.", Resolved: "Resolved: the delivery is no longer waiting"} }), + kindPlanFailed: worded(func(o conditions.Observation) words { + return words{Headline: "A delivery failed", + Explanation: "A delivery did not finish. Nothing is tried again until you say so.", + Resolved: "Resolved: the delivery is no longer failed"} + }), "ask-lost": worded(func(o conditions.Observation) words { return words{Headline: "A build has no result", Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", diff --git a/cmd/mesh-controller/plan_failed.go b/cmd/mesh-controller/plan_failed.go new file mode 100644 index 00000000..16853324 --- /dev/null +++ b/cmd/mesh-controller/plan_failed.go @@ -0,0 +1,213 @@ +package main + +import ( + "context" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// failedModule is one module of a failed plan that failed to build, where it runs and what it said. +type failedModule struct { + module string + machines []string + why string +} + +// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1). +type failedPlanFacts struct { + id, repository, commit string + modules []string + failed []failedModule + // note is where the plan stopped when no module failed to build (a rollout stopped at its first machine). + note string + // since is the plan's last change: when it failed. + since time.Time + // refusal is why `plans retry` would not run it, empty when it would. + refusal string +} + +// machines are the machines the failed modules run on, sorted, each once. +func (p failedPlanFacts) machines() []string { + seen := map[string]bool{} + var out []string + for _, m := range p.failed { + for _, n := range m.machines { + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + } + sort.Strings(out) + return out +} + +// failureSaid is what failed, in the mesh's own words, for the condition's evidence. +func (p failedPlanFacts) failureSaid() string { + var parts []string + for _, m := range p.failed { + line := m.module + if len(m.machines) > 0 { + line += " on " + strings.Join(m.machines, ", ") + } + if m.why != "" { + line += ": " + oneLine(m.why) + } + parts = append(parts, line) + } + if len(parts) == 0 { + return "stopped: " + oneLine(p.note) + } + return strings.Join(parts, "; ") +} + +// refusalSaid is why a retry would not run, for the evidence. +func (p failedPlanFacts) refusalSaid() string { + if p.refusal == "" { + return "" + } + return "; a retry would not run: " + oneLine(p.refusal) +} + +// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan, +// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs). +func retryAction(plan string) conditions.Action { + return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, + Arguments: map[string]string{"retry": plan}} +} + +var ( + planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`) + walkWd = regexp.MustCompile(`\bwalks\b`) + walkOne = regexp.MustCompile(`\bwalk\b`) +) + +// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a +// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the +// words say only that a retry would not run. +func plainRefusal(refusal string) string { + r := planIDWord.ReplaceAllString(refusal, "the plan") + r = walkWd.ReplaceAllString(r, "deliveries") + r = walkOne.ReplaceAllString(r, "delivery") + r = strings.TrimSpace(r) + if _, ok := conditions.Plain(r); !ok { + return "a retry would not run now; the details say why" + } + return r +} + +// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that +// is plain, and Retry when a retry would run, else the refusal's words. +func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) { + name := deliveryName(p.modules, p.repository) + headline = conditions.Capital(name) + " failed" + what := deliveryWhat(p.modules, p.repository) + var parts []string + for i, m := range p.failed { + if i == 2 { + parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2)) + break + } + line := m.module + if len(m.machines) > 0 { + line += " for " + readableList(m.machines) + } + if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" { + if _, ok := conditions.Plain(w); ok { + line += " (" + w + ")" + } + } + parts = append(parts, line) + } + if len(parts) > 0 { + explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts)) + } else { + explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what) + } + if p.refusal == "" { + explanation += " Nothing is tried again until you say so." + needs = "try the failed builds again " + FromMeshMCPServer + actions = []conditions.Action{retryAction(p.id)} + } else { + explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "." + needs = "look at why it failed; a retry would not run now." + } + resolved = conditions.Capital(name) + " is no longer failed" + return +} + +// cutWords is s cut to at most n bytes at a word. +func cutWords(s string, n int) string { + if len(s) <= n { + return s + } + t := s[:n] + if i := strings.LastIndexByte(t, ' '); i > n/2 { + t = t[:i] + } + return strings.TrimRight(t, " ,;:") + "…" +} + +// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator +// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not +// stopped by a person. Each carries why `plans retry` would not run it, when it would not. +func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) { + plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin)) + if err != nil { + return nil, fmt.Errorf("the plans cannot be read: %w", err) + } + var failed []inventory.Plan + for _, p := range plans { + if failedAndAsked(p) { + failed = append(failed, p) + } + } + if len(failed) == 0 { + return nil, nil + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, fmt.Errorf("the catalogue cannot be read: %w", err) + } + onPath := ownPathOf(entries) + runsOn := map[string][]string{} + for _, e := range entries { + runsOn[e.Manifest.Module] = e.On + } + var out []failedPlanFacts + for _, p := range failed { + out = append(out, failedFactsOf(p, plans, onPath, runsOn)) + } + return out, nil +} + +// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not +// stopped by a person. +func failedAndAsked(p inventory.Plan) bool { + return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == "" +} + +// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each +// module runs. +func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string, + runsOn map[string][]string) failedPlanFacts { + f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p), + note: p.Note, since: p.Updated} + for _, m := range f.modules { + if s := p.Modules[m]; s != nil && s.State == "failed" { + machines := append([]string(nil), runsOn[m]...) + sort.Strings(machines) + f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why}) + } + } + if err := retryRefusal(p, plans, onPath); err != nil { + f.refusal = err.Error() + } + return f +} diff --git a/cmd/mesh-controller/plan_failed_test.go b/cmd/mesh-controller/plan_failed_test.go new file mode 100644 index 00000000..6ddf0b93 --- /dev/null +++ b/cmd/mesh-controller/plan_failed_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314). + +func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan { + p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed, + Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC), + Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}}, + Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} + if mutate != nil { + mutate(&p) + } + return p +} + +func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts { + return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{}, + map[string][]string{"openrazer": {"g14", "shanks"}}) +} + +// 1314/AC1 +func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + p := aFailedPlan("plan-1792433168676870614", nil) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(p)} + got := watchPlansFailed(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator || + o.Scope != conditions.ScopePlan || o.ID != p.ID { + t.Fatalf("kind, severity, resolver or subject: %+v", o) + } + if len(o.Actions) != 1 { + t.Fatalf("actions: %+v", o.Actions) + } + a := o.Actions[0] + if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove || + len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID { + t.Fatalf("the action is not Retry with only the plan: %+v", a) + } + if o.Needs == "" { + t.Fatal("no Needs: the asker wants an action with a need") + } + for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} { + if !strings.Contains(o.Headline+o.Explanation+o.Said, want) { + t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said) + } + } + if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") { + t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation) + } + words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions} + if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { + t.Errorf("the words are not plain: %s\n%+v", why, words) + } +} + +// 1314/AC2 +func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) { + for name, p := range map[string]inventory.Plan{ + "held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }), + "stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }), + "no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }), + "building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }), + } { + if failedAndAsked(p) { + t.Errorf("%s: a plan not asked of the operator is asked", name) + } + } + if !failedAndAsked(aFailedPlan("plan-5", nil)) { + t.Error("a failed delivery plan is not asked") + } +} + +// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it. +func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) }) + newer := aFailedPlan("plan-2", func(p *inventory.Plan) { + p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute) + }) + superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" }) + done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone }) + for name, c := range map[string]struct { + p inventory.Plan + others []inventory.Plan + says string + }{ + "a newer plan": {older, []inventory.Plan{newer}, "supersedes it"}, + "superseded": {superseded, nil, "superseded"}, + "done": {done, nil, "nothing to retry"}, + } { + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)} + got := watchPlansFailed(f) + if len(got) != 1 { + t.Fatalf("%s: %+v", name, got) + } + o := got[0] + if len(o.Actions) != 0 { + t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions) + } + if !strings.Contains(o.Said, c.says) { + t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said) + } + if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) { + t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation) + } + words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} + if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { + t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words) + } + } +} + +// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23. +func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + defer k.Close(t.Context()) + p := aFailedPlan("plan-1792433168676870614", nil) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(p)} + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + open, _ := k.Open(t.Context()) + if len(open) != 1 || open[0].Kind != "plan-failed" { + t.Fatalf("not raised: %+v", open) + } + // Retried: the plan is building, so it is no longer among the failed. + f = calm(now.Add(time.Minute)) + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + if open, _ = k.Open(t.Context()); len(open) != 0 { + t.Fatalf("the condition outlived the failure: %+v", open) + } + // Failed again: asked again, with Retry, and nothing retried it. + f = calm(now.Add(2 * time.Minute)) + again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) }) + f.failed = []failedPlanFacts{failedFactsFor(again)} + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + open, _ = k.Open(t.Context()) + if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" { + t.Fatalf("a second failure was not asked again with Retry: %+v", open) + } +} + +// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing +// S23 produces is an observation that offers an answer. +func TestNoClockRetriesAFailedPlan(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))} + for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} { + f.now = now.Add(later) + got := watchPlansFailed(f) + if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" { + t.Fatalf("after %s: %+v", later, got) + } + } + // A plan failed longer ago than the week it is kept is left to `plans`. + f.now = now.Add(planFailedWithin + time.Second) + if got := watchPlansFailed(f); len(got) != 0 { + t.Fatalf("a plan failed beyond the window is still raised: %+v", got) + } +} diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index c311efb4..33a49603 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -234,6 +234,16 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.batches, func(b batchFacts) time.Time { return b.created }) }}, + {Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan", + Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)", + Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " + + "words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " + + "from the plan's last change, and ended when the plan is no longer failed", + Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3, + needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed, + newest: func(f *signalFacts) time.Time { + return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since }) + }}, {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + @@ -404,6 +414,34 @@ func watchPlans(f *signalFacts) []conditions.Observation { return out } +// kindPlanFailed is S23's kind: plan..plan-failed. +const kindPlanFailed = "plan-failed" + +// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no +// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to +// the operator). A plan failed longer ago than this is left to `plans`. +const planFailedWithin = 7 * 24 * time.Hour + +// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when +// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2). +// The condition ends when the plan is no longer failed, because it is no longer among the facts. +func watchPlansFailed(f *signalFacts) []conditions.Observation { + var out []conditions.Observation + for _, p := range f.failed { + if f.now.Sub(p.since) > planFailedWithin { + continue + } + headline, explanation, needs, actions, resolved := planFailedWords(p) + out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed, + Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit), + p.failureSaid()), + Said: p.failureSaid() + p.refusalSaid(), + Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved}) + } + return out +} + // kindWalkWaiting is S16's kind: plan..waiting. const kindWalkWaiting = "waiting" diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 3fc0561c..488117da 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -160,6 +160,17 @@ var suppressions = map[string]suppression{ created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}} }, }, + // A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449). + "S23": { + inside: func(f *signalFacts) { + f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, + since: f.now.Add(-planFailedWithin - time.Second)}} + }, + past: func(f *signalFacts) { + f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, + since: f.now.Add(-time.Second)}} + }, + }, // A batch queued behind an open walk past that walk's bound, named. "S19": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 3ec0ff1c..88605563 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -55,6 +55,9 @@ type signalFacts struct { waits []waitFacts // batches are the batches not yet cut (S18, S19, novox/hq ADR 0276). batches []batchFacts + // failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449). + failed []failedPlanFacts + failedErr error // refusedSends are the sends a machine refused within the hour for the generation they were composed // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send @@ -353,6 +356,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { if f.plansErr == nil { f.batches, f.plansErr = gatherBatches(ctx, inv, now) } + f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now) f.loop, f.loopErr = w.gatherLoop() f.asks, f.asksErr = w.gatherAsks(ctx, inv) if open, err := w.keeper.Open(ctx); err != nil {