Raise plan-failed for a failed delivery plan, with Retry when a retry would run (issue 1314, ADR 0449)
This commit is contained in:
@@ -0,0 +1,184 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314).
|
||||
|
||||
func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan {
|
||||
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed,
|
||||
Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC),
|
||||
Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
||||
if mutate != nil {
|
||||
mutate(&p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts {
|
||||
return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{},
|
||||
map[string][]string{"openrazer": {"g14", "shanks"}})
|
||||
}
|
||||
|
||||
// 1314/AC1
|
||||
func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator ||
|
||||
o.Scope != conditions.ScopePlan || o.ID != p.ID {
|
||||
t.Fatalf("kind, severity, resolver or subject: %+v", o)
|
||||
}
|
||||
if len(o.Actions) != 1 {
|
||||
t.Fatalf("actions: %+v", o.Actions)
|
||||
}
|
||||
a := o.Actions[0]
|
||||
if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove ||
|
||||
len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID {
|
||||
t.Fatalf("the action is not Retry with only the plan: %+v", a)
|
||||
}
|
||||
if o.Needs == "" {
|
||||
t.Fatal("no Needs: the asker wants an action with a need")
|
||||
}
|
||||
for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} {
|
||||
if !strings.Contains(o.Headline+o.Explanation+o.Said, want) {
|
||||
t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") {
|
||||
t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("the words are not plain: %s\n%+v", why, words)
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC2
|
||||
func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) {
|
||||
for name, p := range map[string]inventory.Plan{
|
||||
"held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }),
|
||||
"stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }),
|
||||
"no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }),
|
||||
"building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }),
|
||||
} {
|
||||
if failedAndAsked(p) {
|
||||
t.Errorf("%s: a plan not asked of the operator is asked", name)
|
||||
}
|
||||
}
|
||||
if !failedAndAsked(aFailedPlan("plan-5", nil)) {
|
||||
t.Error("a failed delivery plan is not asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it.
|
||||
func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) })
|
||||
newer := aFailedPlan("plan-2", func(p *inventory.Plan) {
|
||||
p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute)
|
||||
})
|
||||
superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" })
|
||||
done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone })
|
||||
for name, c := range map[string]struct {
|
||||
p inventory.Plan
|
||||
others []inventory.Plan
|
||||
says string
|
||||
}{
|
||||
"a newer plan": {older, []inventory.Plan{newer}, "supersedes it"},
|
||||
"superseded": {superseded, nil, "superseded"},
|
||||
"done": {done, nil, "nothing to retry"},
|
||||
} {
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%s: %+v", name, got)
|
||||
}
|
||||
o := got[0]
|
||||
if len(o.Actions) != 0 {
|
||||
t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions)
|
||||
}
|
||||
if !strings.Contains(o.Said, c.says) {
|
||||
t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said)
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) {
|
||||
t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23.
|
||||
func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(t.Context())
|
||||
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ := k.Open(t.Context())
|
||||
if len(open) != 1 || open[0].Kind != "plan-failed" {
|
||||
t.Fatalf("not raised: %+v", open)
|
||||
}
|
||||
// Retried: the plan is building, so it is no longer among the failed.
|
||||
f = calm(now.Add(time.Minute))
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ = k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition outlived the failure: %+v", open)
|
||||
}
|
||||
// Failed again: asked again, with Retry, and nothing retried it.
|
||||
f = calm(now.Add(2 * time.Minute))
|
||||
again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) })
|
||||
f.failed = []failedPlanFacts{failedFactsFor(again)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(t.Context())
|
||||
if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" {
|
||||
t.Fatalf("a second failure was not asked again with Retry: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing
|
||||
// S23 produces is an observation that offers an answer.
|
||||
func TestNoClockRetriesAFailedPlan(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))}
|
||||
for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} {
|
||||
f.now = now.Add(later)
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" {
|
||||
t.Fatalf("after %s: %+v", later, got)
|
||||
}
|
||||
}
|
||||
// A plan failed longer ago than the week it is kept is left to `plans`.
|
||||
f.now = now.Add(planFailedWithin + time.Second)
|
||||
if got := watchPlansFailed(f); len(got) != 0 {
|
||||
t.Fatalf("a plan failed beyond the window is still raised: %+v", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user