package main import ( "context" "fmt" "sort" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) // **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314). func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan { p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed, Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC), Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}}, Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} if mutate != nil { mutate(&p) } return p } func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts { return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{}, map[string][]string{"openrazer": {"g14", "shanks"}}) } // 1314/AC1 func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) { now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) p := aFailedPlan("plan-1792433168676870614", nil) f := calm(now) f.failed = []failedPlanFacts{failedFactsFor(p)} got := watchPlansFailed(f) if len(got) != 1 { t.Fatalf("%+v", got) } o := got[0] if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator || o.Scope != conditions.ScopePlan || o.ID != p.ID { t.Fatalf("kind, severity, resolver or subject: %+v", o) } if len(o.Actions) != 1 { t.Fatalf("actions: %+v", o.Actions) } a := o.Actions[0] if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove || len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID { t.Fatalf("the action is not Retry with only the plan: %+v", a) } if o.Needs == "" { t.Fatal("no Needs: the asker wants an action with a need") } for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} { if !strings.Contains(o.Headline+o.Explanation+o.Said, want) { t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said) } } if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") { t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation) } words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions} if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { t.Errorf("the words are not plain: %s\n%+v", why, words) } } // 1314/AC2 func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) { for name, p := range map[string]inventory.Plan{ "held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }), "stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }), "no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }), "building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }), } { if failedAndAsked(p) { t.Errorf("%s: a plan not asked of the operator is asked", name) } } if !failedAndAsked(aFailedPlan("plan-5", nil)) { t.Error("a failed delivery plan is not asked") } } // 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it. func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) { now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) }) newer := aFailedPlan("plan-2", func(p *inventory.Plan) { p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute) }) superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" }) done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone }) first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC) gate := aFailedPlan("plan-5", func(p *inventory.Plan) { p.Note = "openrazer stopped at its first machine" p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"}, Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}} }) for name, c := range map[string]struct { p inventory.Plan others []inventory.Plan says string }{ "a newer plan": {older, []inventory.Plan{newer}, "supersedes it"}, "superseded": {superseded, nil, "superseded"}, "done": {done, nil, "nothing to retry"}, "gate failed": {gate, nil, "failed its gate"}, } { f := calm(now) f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)} got := watchPlansFailed(f) if len(got) != 1 { t.Fatalf("%s: %+v", name, got) } o := got[0] if len(o.Actions) != 0 { t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions) } if !strings.Contains(o.Said, c.says) { t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said) } if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) { t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation) } words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words) } } } // 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23. func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) { now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) store := conditions.NewInMemory() k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}, Now: func() time.Time { return now }}) defer k.Close(t.Context()) p := aFailedPlan("plan-1792433168676870614", nil) f := calm(now) f.failed = []failedPlanFacts{failedFactsFor(p)} if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { t.Fatal(err) } open, _ := k.Open(t.Context()) if len(open) != 1 || open[0].Kind != "plan-failed" { t.Fatalf("not raised: %+v", open) } // Retried: the plan is building, so it is no longer among the failed. f = calm(now.Add(time.Minute)) if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { t.Fatal(err) } if open, _ = k.Open(t.Context()); len(open) != 0 { t.Fatalf("the condition outlived the failure: %+v", open) } // Failed again: asked again, with Retry, and nothing retried it. f = calm(now.Add(2 * time.Minute)) again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) }) f.failed = []failedPlanFacts{failedFactsFor(again)} if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { t.Fatal(err) } open, _ = k.Open(t.Context()) if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" { t.Fatalf("a second failure was not asked again with Retry: %+v", open) } } // 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing // S23 produces is an observation that offers an answer. func TestNoClockRetriesAFailedPlan(t *testing.T) { now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) f := calm(now) f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))} for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} { f.now = now.Add(later) got := watchPlansFailed(f) if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" { t.Fatalf("after %s: %+v", later, got) } } // A plan failed longer ago than the week it is kept is left to `plans`. f.now = now.Add(planFailedWithin + time.Second) if got := watchPlansFailed(f); len(got) != 0 { t.Fatalf("a plan failed beyond the window is still raised: %+v", got) } } // A later plan of the same repository and branch, in a state, made after the failed one. func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan { q := aFailedPlan(id, func(p *inventory.Plan) { p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour) }) return q } // 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan. func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) { failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) later := laterPlan("plan-2", "main", inventory.PlanDone, failed) err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{}) if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") { t.Fatalf("the retry was not refused, naming the later plan: %v", err) } } // 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing. func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) { failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed) err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{}) if err == nil || !strings.Contains(err.Error(), "supersedes it") { t.Fatalf("with the later plan open the refusal is not today's: %v", err) } other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed) if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil { t.Fatalf("a later done plan of another branch refused the retry: %v", err) } earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed) earlier.Created = failed.Created.Add(-time.Hour) if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil { t.Fatalf("an earlier done plan refused the retry: %v", err) } } // 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery. func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) { now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) }) later := laterPlan("plan-2", "main", inventory.PlanDone, failed) f := calm(now) f.failed = []failedPlanFacts{failedFactsFor(failed, later)} got := watchPlansFailed(f) if len(got) != 1 || len(got[0].Actions) != 0 { t.Fatalf("%+v", got) } o := got[0] if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") { t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation) } if strings.Contains(o.Explanation, "details say why") { t.Errorf("the generic fallback was said: %q", o.Explanation) } words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { t.Errorf("not plain: %s\n%+v", why, words) } } // plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since. type plansBehind []inventory.Plan func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) { var out []inventory.Plan for _, p := range s { if p.Open() { out = append(out, p) } } return out, nil } func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) { all := append([]inventory.Plan(nil), s...) sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) }) if len(all) > limit { all = all[:limit] } return all, nil } func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) { var out []inventory.Plan for _, p := range s { if p.Created.After(since) || p.Open() { out = append(out, p) } } return out, nil } // 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it. func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) { failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) }) all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)} for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50 other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed) other.Repository = "novox/other" other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour) all = append(all, other) } plans, err := plansForRetry(t.Context(), all, failed) if err != nil { t.Fatal(err) } if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") { t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err) } } // ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its // plan's last change passes 7 days. func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) { now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) }) f := calm(now) f.failed = []failedPlanFacts{failedFactsFor(old)} if got := watchPlansFailed(f); len(got) != 0 { t.Fatalf("a plan last changed 8 days ago raised %+v", got) } store := conditions.NewInMemory() k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}, Now: func() time.Time { return now }}) defer k.Close(t.Context()) fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) }) f.failed = []failedPlanFacts{failedFactsFor(fresh)} if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { t.Fatal(err) } if open, _ := k.Open(t.Context()); len(open) != 1 { t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open) } f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { t.Fatal(err) } if open, _ := k.Open(t.Context()); len(open) != 0 { t.Fatalf("the condition outlived the 7 days: %+v", open) } }