cmd/mesh-controller/plain_words.go

@@ -429,6 +429,11 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"out) has not let it start yet.",
 			Resolved: "Resolved: the delivery is no longer waiting"}
 	}),
+	kindPlanFailed: worded(func(o conditions.Observation) words {
+		return words{Headline: "A delivery failed",
+			Explanation: "A delivery did not finish. Nothing is tried again until you say so.",
+			Resolved:    "Resolved: the delivery is no longer failed"}
+	}),
 	"ask-lost": worded(func(o conditions.Observation) words {
 		return words{Headline: "A build has no result",
 			Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",

cmd/mesh-controller/plan_failed.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// failedModule is one module of a failed plan that failed to build, where it runs and what it said.
+type failedModule struct {
+	module   string
+	machines []string
+	why      string
+}
+
+// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1).
+type failedPlanFacts struct {
+	id, repository, commit string
+	modules                []string
+	failed                 []failedModule
+	// note is where the plan stopped when no module failed to build (a rollout stopped at its first machine).
+	note string
+	// since is the plan's last change: when it failed.
+	since time.Time
+	// refusal is why `plans retry` would not run it, empty when it would.
+	refusal string
+}
+
+// machines are the machines the failed modules run on, sorted, each once.
+func (p failedPlanFacts) machines() []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, m := range p.failed {
+		for _, n := range m.machines {
+			if !seen[n] {
+				seen[n] = true
+				out = append(out, n)
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// failureSaid is what failed, in the mesh's own words, for the condition's evidence.
+func (p failedPlanFacts) failureSaid() string {
+	var parts []string
+	for _, m := range p.failed {
+		line := m.module
+		if len(m.machines) > 0 {
+			line += " on " + strings.Join(m.machines, ", ")
+		}
+		if m.why != "" {
+			line += ": " + oneLine(m.why)
+		}
+		parts = append(parts, line)
+	}
+	if len(parts) == 0 {
+		return "stopped: " + oneLine(p.note)
+	}
+	return strings.Join(parts, "; ")
+}
+
+// refusalSaid is why a retry would not run, for the evidence.
+func (p failedPlanFacts) refusalSaid() string {
+	if p.refusal == "" {
+		return ""
+	}
+	return "; a retry would not run: " + oneLine(p.refusal)
+}
+
+// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan,
+// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs).
+func retryAction(plan string) conditions.Action {
+	return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
+		Arguments: map[string]string{"retry": plan}}
+}
+
+var (
+	planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`)
+	walkWd     = regexp.MustCompile(`\bwalks\b`)
+	walkOne    = regexp.MustCompile(`\bwalk\b`)
+)
+
+// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a
+// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the
+// words say only that a retry would not run.
+func plainRefusal(refusal string) string {
+	r := planIDWord.ReplaceAllString(refusal, "the plan")
+	r = walkWd.ReplaceAllString(r, "deliveries")
+	r = walkOne.ReplaceAllString(r, "delivery")
+	r = strings.TrimSpace(r)
+	if _, ok := conditions.Plain(r); !ok {
+		return "a retry would not run now; the details say why"
+	}
+	return r
+}
+
+// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that
+// is plain, and Retry when a retry would run, else the refusal's words.
+func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) {
+	name := deliveryName(p.modules, p.repository)
+	headline = conditions.Capital(name) + " failed"
+	what := deliveryWhat(p.modules, p.repository)
+	var parts []string
+	for i, m := range p.failed {
+		if i == 2 {
+			parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2))
+			break
+		}
+		line := m.module
+		if len(m.machines) > 0 {
+			line += " for " + readableList(m.machines)
+		}
+		if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" {
+			if _, ok := conditions.Plain(w); ok {
+				line += " (" + w + ")"
+			}
+		}
+		parts = append(parts, line)
+	}
+	if len(parts) > 0 {
+		explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts))
+	} else {
+		explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what)
+	}
+	if p.refusal == "" {
+		explanation += " Nothing is tried again until you say so."
+		needs = "try the failed builds again " + FromMeshMCPServer
+		actions = []conditions.Action{retryAction(p.id)}
+	} else {
+		explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "."
+		needs = "look at why it failed; a retry would not run now."
+	}
+	resolved = conditions.Capital(name) + " is no longer failed"
+	return
+}
+
+// cutWords is s cut to at most n bytes at a word.
+func cutWords(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	t := s[:n]
+	if i := strings.LastIndexByte(t, ' '); i > n/2 {
+		t = t[:i]
+	}
+	return strings.TrimRight(t, " ,;:") + "…"
+}
+
+// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator
+// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not
+// stopped by a person. Each carries why `plans retry` would not run it, when it would not.
+func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) {
+	plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin))
+	if err != nil {
+		return nil, fmt.Errorf("the plans cannot be read: %w", err)
+	}
+	var failed []inventory.Plan
+	for _, p := range plans {
+		if failedAndAsked(p) {
+			failed = append(failed, p)
+		}
+	}
+	if len(failed) == 0 {
+		return nil, nil
+	}
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
+	}
+	onPath := ownPathOf(entries)
+	runsOn := map[string][]string{}
+	for _, e := range entries {
+		runsOn[e.Manifest.Module] = e.On
+	}
+	var out []failedPlanFacts
+	for _, p := range failed {
+		out = append(out, failedFactsOf(p, plans, onPath, runsOn))
+	}
+	return out, nil
+}
+
+// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not
+// stopped by a person.
+func failedAndAsked(p inventory.Plan) bool {
+	return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == ""
+}
+
+// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each
+// module runs.
+func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string,
+	runsOn map[string][]string) failedPlanFacts {
+	f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p),
+		note: p.Note, since: p.Updated}
+	for _, m := range f.modules {
+		if s := p.Modules[m]; s != nil && s.State == "failed" {
+			machines := append([]string(nil), runsOn[m]...)
+			sort.Strings(machines)
+			f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why})
+		}
+	}
+	if err := retryRefusal(p, plans, onPath); err != nil {
+		f.refusal = err.Error()
+	}
+	return f
+}

cmd/mesh-controller/plan_failed_test.go

@@ -0,0 +1,184 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314).
+
+func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan {
+	p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed,
+		Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC),
+		Modules:  map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}},
+		Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
+	if mutate != nil {
+		mutate(&p)
+	}
+	return p
+}
+
+func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts {
+	return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{},
+		map[string][]string{"openrazer": {"g14", "shanks"}})
+}
+
+// 1314/AC1
+func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) {
+	now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
+	p := aFailedPlan("plan-1792433168676870614", nil)
+	f := calm(now)
+	f.failed = []failedPlanFacts{failedFactsFor(p)}
+	got := watchPlansFailed(f)
+	if len(got) != 1 {
+		t.Fatalf("%+v", got)
+	}
+	o := got[0]
+	if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator ||
+		o.Scope != conditions.ScopePlan || o.ID != p.ID {
+		t.Fatalf("kind, severity, resolver or subject: %+v", o)
+	}
+	if len(o.Actions) != 1 {
+		t.Fatalf("actions: %+v", o.Actions)
+	}
+	a := o.Actions[0]
+	if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove ||
+		len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID {
+		t.Fatalf("the action is not Retry with only the plan: %+v", a)
+	}
+	if o.Needs == "" {
+		t.Fatal("no Needs: the asker wants an action with a need")
+	}
+	for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} {
+		if !strings.Contains(o.Headline+o.Explanation+o.Said, want) {
+			t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said)
+		}
+	}
+	if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") {
+		t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation)
+	}
+	words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}
+	if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
+		t.Errorf("the words are not plain: %s\n%+v", why, words)
+	}
+}
+
+// 1314/AC2
+func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) {
+	for name, p := range map[string]inventory.Plan{
+		"held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }),
+		"stopped":      aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }),
+		"no delivery":  aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }),
+		"building":     aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }),
+	} {
+		if failedAndAsked(p) {
+			t.Errorf("%s: a plan not asked of the operator is asked", name)
+		}
+	}
+	if !failedAndAsked(aFailedPlan("plan-5", nil)) {
+		t.Error("a failed delivery plan is not asked")
+	}
+}
+
+// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it.
+func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) {
+	now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
+	older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) })
+	newer := aFailedPlan("plan-2", func(p *inventory.Plan) {
+		p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute)
+	})
+	superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" })
+	done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone })
+	for name, c := range map[string]struct {
+		p      inventory.Plan
+		others []inventory.Plan
+		says   string
+	}{
+		"a newer plan": {older, []inventory.Plan{newer}, "supersedes it"},
+		"superseded":   {superseded, nil, "superseded"},
+		"done":         {done, nil, "nothing to retry"},
+	} {
+		f := calm(now)
+		f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)}
+		got := watchPlansFailed(f)
+		if len(got) != 1 {
+			t.Fatalf("%s: %+v", name, got)
+		}
+		o := got[0]
+		if len(o.Actions) != 0 {
+			t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions)
+		}
+		if !strings.Contains(o.Said, c.says) {
+			t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said)
+		}
+		if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) {
+			t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation)
+		}
+		words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
+		if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
+			t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words)
+		}
+	}
+}
+
+// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23.
+func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) {
+	now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
+		Teller: &conditions.Told{}, Now: func() time.Time { return now }})
+	defer k.Close(t.Context())
+	p := aFailedPlan("plan-1792433168676870614", nil)
+	f := calm(now)
+	f.failed = []failedPlanFacts{failedFactsFor(p)}
+	if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
+		t.Fatal(err)
+	}
+	open, _ := k.Open(t.Context())
+	if len(open) != 1 || open[0].Kind != "plan-failed" {
+		t.Fatalf("not raised: %+v", open)
+	}
+	// Retried: the plan is building, so it is no longer among the failed.
+	f = calm(now.Add(time.Minute))
+	if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
+		t.Fatal(err)
+	}
+	if open, _ = k.Open(t.Context()); len(open) != 0 {
+		t.Fatalf("the condition outlived the failure: %+v", open)
+	}
+	// Failed again: asked again, with Retry, and nothing retried it.
+	f = calm(now.Add(2 * time.Minute))
+	again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) })
+	f.failed = []failedPlanFacts{failedFactsFor(again)}
+	if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
+		t.Fatal(err)
+	}
+	open, _ = k.Open(t.Context())
+	if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" {
+		t.Fatalf("a second failure was not asked again with Retry: %+v", open)
+	}
+}
+
+// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing
+// S23 produces is an observation that offers an answer.
+func TestNoClockRetriesAFailedPlan(t *testing.T) {
+	now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
+	f := calm(now)
+	f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))}
+	for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} {
+		f.now = now.Add(later)
+		got := watchPlansFailed(f)
+		if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" {
+			t.Fatalf("after %s: %+v", later, got)
+		}
+	}
+	// A plan failed longer ago than the week it is kept is left to `plans`.
+	f.now = now.Add(planFailedWithin + time.Second)
+	if got := watchPlansFailed(f); len(got) != 0 {
+		t.Fatalf("a plan failed beyond the window is still raised: %+v", got)
+	}
+}

cmd/mesh-controller/signals.go

@@ -234,6 +234,16 @@ var signalsTable = []signalRow{
 		newest: func(f *signalFacts) time.Time {
 			return newestOf(f.batches, func(b batchFacts) time.Time { return b.created })
 		}},
+	{Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan",
+		Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)",
+		Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " +
+			"words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " +
+			"from the plan's last change, and ended when the plan is no longer failed",
+		Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3,
+		needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed,
+		newest: func(f *signalFacts) time.Time {
+			return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
+		}},
 	{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
 		Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
 		Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
@@ -404,6 +414,34 @@ func watchPlans(f *signalFacts) []conditions.Observation {
 	return out
 }
 
+// kindPlanFailed is S23's kind: plan.<id>.plan-failed.
+const kindPlanFailed = "plan-failed"
+
+// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no
+// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to
+// the operator). A plan failed longer ago than this is left to `plans`.
+const planFailedWithin = 7 * 24 * time.Hour
+
+// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when
+// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2).
+// The condition ends when the plan is no longer failed, because it is no longer among the facts.
+func watchPlansFailed(f *signalFacts) []conditions.Observation {
+	var out []conditions.Observation
+	for _, p := range f.failed {
+		if f.now.Sub(p.since) > planFailedWithin {
+			continue
+		}
+		headline, explanation, needs, actions, resolved := planFailedWords(p)
+		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed,
+			Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
+			Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit),
+				p.failureSaid()),
+			Said:     p.failureSaid() + p.refusalSaid(),
+			Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved})
+	}
+	return out
+}
+
 // kindWalkWaiting is S16's kind: plan.<id>.waiting.
 const kindWalkWaiting = "waiting"
 

cmd/mesh-controller/signals_test.go

@@ -160,6 +160,17 @@ var suppressions = map[string]suppression{
 				created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}}
 		},
 	},
+	// A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449).
+	"S23": {
+		inside: func(f *signalFacts) {
+			f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
+				since: f.now.Add(-planFailedWithin - time.Second)}}
+		},
+		past: func(f *signalFacts) {
+			f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
+				since: f.now.Add(-time.Second)}}
+		},
+	},
 	// A batch queued behind an open walk past that walk's bound, named.
 	"S19": {
 		inside: func(f *signalFacts) {

cmd/mesh-controller/watchdogs.go

@@ -55,6 +55,9 @@ type signalFacts struct {
 	waits []waitFacts
 	// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
 	batches []batchFacts
+	// failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449).
+	failed    []failedPlanFacts
+	failedErr error
 	// refusedSends are the sends a machine refused within the hour for the generation they were composed
 	// from, each naming its sender (S20, novox/hq issue 234).
 	refusedSends    []inventory.Send
@@ -353,6 +356,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 	if f.plansErr == nil {
 		f.batches, f.plansErr = gatherBatches(ctx, inv, now)
 	}
+	f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now)
 	f.loop, f.loopErr = w.gatherLoop()
 	f.asks, f.asksErr = w.gatherAsks(ctx, inv)
 	if open, err := w.keeper.Open(ctx); err != nil {