Raise plan-failed for a failed delivery plan, with Retry when a retry would run (issue 1314, ADR 0449)

cmd/mesh-controller/signals.go

@@ -234,6 +234,16 @@ var signalsTable = []signalRow{
 		newest: func(f *signalFacts) time.Time {
 			return newestOf(f.batches, func(b batchFacts) time.Time { return b.created })
 		}},
+	{Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan",
+		Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)",
+		Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " +
+			"words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " +
+			"from the plan's last change, and ended when the plan is no longer failed",
+		Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3,
+		needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed,
+		newest: func(f *signalFacts) time.Time {
+			return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
+		}},
 	{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
 		Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
 		Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
@@ -404,6 +414,34 @@ func watchPlans(f *signalFacts) []conditions.Observation {
 	return out
 }
 
+// kindPlanFailed is S23's kind: plan.<id>.plan-failed.
+const kindPlanFailed = "plan-failed"
+
+// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no
+// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to
+// the operator). A plan failed longer ago than this is left to `plans`.
+const planFailedWithin = 7 * 24 * time.Hour
+
+// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when
+// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2).
+// The condition ends when the plan is no longer failed, because it is no longer among the facts.
+func watchPlansFailed(f *signalFacts) []conditions.Observation {
+	var out []conditions.Observation
+	for _, p := range f.failed {
+		if f.now.Sub(p.since) > planFailedWithin {
+			continue
+		}
+		headline, explanation, needs, actions, resolved := planFailedWords(p)
+		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed,
+			Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
+			Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit),
+				p.failureSaid()),
+			Said:     p.failureSaid() + p.refusalSaid(),
+			Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved})
+	}
+	return out
+}
+
 // kindWalkWaiting is S16's kind: plan.<id>.waiting.
 const kindWalkWaiting = "waiting"