mesh-tools
7f23c537cc
mesh/merge-gate head 7f23c537cced merged into trunk 1f708e2d8d72 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 7f23c537cced merged into trunk 1f708e2d8d72 — pass: its merge-check.sh passed
mesh/delivery delivered
339 lines
15 KiB
Go
339 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314).
|
|
|
|
func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan {
|
|
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed,
|
|
Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC),
|
|
Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}},
|
|
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
|
if mutate != nil {
|
|
mutate(&p)
|
|
}
|
|
return p
|
|
}
|
|
|
|
func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts {
|
|
return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{},
|
|
map[string][]string{"openrazer": {"g14", "shanks"}})
|
|
}
|
|
|
|
// 1314/AC1
|
|
func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) {
|
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
|
p := aFailedPlan("plan-1792433168676870614", nil)
|
|
f := calm(now)
|
|
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
|
got := watchPlansFailed(f)
|
|
if len(got) != 1 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
o := got[0]
|
|
if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator ||
|
|
o.Scope != conditions.ScopePlan || o.ID != p.ID {
|
|
t.Fatalf("kind, severity, resolver or subject: %+v", o)
|
|
}
|
|
if len(o.Actions) != 1 {
|
|
t.Fatalf("actions: %+v", o.Actions)
|
|
}
|
|
a := o.Actions[0]
|
|
if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove ||
|
|
len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID {
|
|
t.Fatalf("the action is not Retry with only the plan: %+v", a)
|
|
}
|
|
if o.Needs == "" {
|
|
t.Fatal("no Needs: the asker wants an action with a need")
|
|
}
|
|
for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} {
|
|
if !strings.Contains(o.Headline+o.Explanation+o.Said, want) {
|
|
t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said)
|
|
}
|
|
}
|
|
if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") {
|
|
t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation)
|
|
}
|
|
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}
|
|
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
|
t.Errorf("the words are not plain: %s\n%+v", why, words)
|
|
}
|
|
}
|
|
|
|
// 1314/AC2
|
|
func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) {
|
|
for name, p := range map[string]inventory.Plan{
|
|
"held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }),
|
|
"stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }),
|
|
"no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }),
|
|
"building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }),
|
|
} {
|
|
if failedAndAsked(p) {
|
|
t.Errorf("%s: a plan not asked of the operator is asked", name)
|
|
}
|
|
}
|
|
if !failedAndAsked(aFailedPlan("plan-5", nil)) {
|
|
t.Error("a failed delivery plan is not asked")
|
|
}
|
|
}
|
|
|
|
// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it.
|
|
func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) {
|
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
|
older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) })
|
|
newer := aFailedPlan("plan-2", func(p *inventory.Plan) {
|
|
p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute)
|
|
})
|
|
superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" })
|
|
done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone })
|
|
first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC)
|
|
gate := aFailedPlan("plan-5", func(p *inventory.Plan) {
|
|
p.Note = "openrazer stopped at its first machine"
|
|
p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"},
|
|
Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"},
|
|
Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}}
|
|
})
|
|
for name, c := range map[string]struct {
|
|
p inventory.Plan
|
|
others []inventory.Plan
|
|
says string
|
|
}{
|
|
"a newer plan": {older, []inventory.Plan{newer}, "supersedes it"},
|
|
"superseded": {superseded, nil, "superseded"},
|
|
"done": {done, nil, "nothing to retry"},
|
|
"gate failed": {gate, nil, "failed its gate"},
|
|
} {
|
|
f := calm(now)
|
|
f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)}
|
|
got := watchPlansFailed(f)
|
|
if len(got) != 1 {
|
|
t.Fatalf("%s: %+v", name, got)
|
|
}
|
|
o := got[0]
|
|
if len(o.Actions) != 0 {
|
|
t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions)
|
|
}
|
|
if !strings.Contains(o.Said, c.says) {
|
|
t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said)
|
|
}
|
|
if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) {
|
|
t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation)
|
|
}
|
|
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
|
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
|
t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23.
|
|
func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) {
|
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
defer k.Close(t.Context())
|
|
p := aFailedPlan("plan-1792433168676870614", nil)
|
|
f := calm(now)
|
|
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
open, _ := k.Open(t.Context())
|
|
if len(open) != 1 || open[0].Kind != "plan-failed" {
|
|
t.Fatalf("not raised: %+v", open)
|
|
}
|
|
// Retried: the plan is building, so it is no longer among the failed.
|
|
f = calm(now.Add(time.Minute))
|
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if open, _ = k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("the condition outlived the failure: %+v", open)
|
|
}
|
|
// Failed again: asked again, with Retry, and nothing retried it.
|
|
f = calm(now.Add(2 * time.Minute))
|
|
again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) })
|
|
f.failed = []failedPlanFacts{failedFactsFor(again)}
|
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
open, _ = k.Open(t.Context())
|
|
if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" {
|
|
t.Fatalf("a second failure was not asked again with Retry: %+v", open)
|
|
}
|
|
}
|
|
|
|
// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing
|
|
// S23 produces is an observation that offers an answer.
|
|
func TestNoClockRetriesAFailedPlan(t *testing.T) {
|
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))}
|
|
for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} {
|
|
f.now = now.Add(later)
|
|
got := watchPlansFailed(f)
|
|
if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" {
|
|
t.Fatalf("after %s: %+v", later, got)
|
|
}
|
|
}
|
|
// A plan failed longer ago than the week it is kept is left to `plans`.
|
|
f.now = now.Add(planFailedWithin + time.Second)
|
|
if got := watchPlansFailed(f); len(got) != 0 {
|
|
t.Fatalf("a plan failed beyond the window is still raised: %+v", got)
|
|
}
|
|
}
|
|
|
|
// A later plan of the same repository and branch, in a state, made after the failed one.
|
|
func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan {
|
|
q := aFailedPlan(id, func(p *inventory.Plan) {
|
|
p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour)
|
|
})
|
|
return q
|
|
}
|
|
|
|
// 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan.
|
|
func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) {
|
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
|
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
|
err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{})
|
|
if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") {
|
|
t.Fatalf("the retry was not refused, naming the later plan: %v", err)
|
|
}
|
|
}
|
|
|
|
// 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing.
|
|
func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) {
|
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
|
open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed)
|
|
err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{})
|
|
if err == nil || !strings.Contains(err.Error(), "supersedes it") {
|
|
t.Fatalf("with the later plan open the refusal is not today's: %v", err)
|
|
}
|
|
other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed)
|
|
if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil {
|
|
t.Fatalf("a later done plan of another branch refused the retry: %v", err)
|
|
}
|
|
earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed)
|
|
earlier.Created = failed.Created.Add(-time.Hour)
|
|
if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil {
|
|
t.Fatalf("an earlier done plan refused the retry: %v", err)
|
|
}
|
|
}
|
|
|
|
// 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery.
|
|
func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) {
|
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) })
|
|
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
|
f := calm(now)
|
|
f.failed = []failedPlanFacts{failedFactsFor(failed, later)}
|
|
got := watchPlansFailed(f)
|
|
if len(got) != 1 || len(got[0].Actions) != 0 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
o := got[0]
|
|
if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") {
|
|
t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation)
|
|
}
|
|
if strings.Contains(o.Explanation, "details say why") {
|
|
t.Errorf("the generic fallback was said: %q", o.Explanation)
|
|
}
|
|
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
|
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
|
t.Errorf("not plain: %s\n%+v", why, words)
|
|
}
|
|
}
|
|
|
|
// plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since.
|
|
type plansBehind []inventory.Plan
|
|
|
|
func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) {
|
|
var out []inventory.Plan
|
|
for _, p := range s {
|
|
if p.Open() {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) {
|
|
all := append([]inventory.Plan(nil), s...)
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) })
|
|
if len(all) > limit {
|
|
all = all[:limit]
|
|
}
|
|
return all, nil
|
|
}
|
|
|
|
func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) {
|
|
var out []inventory.Plan
|
|
for _, p := range s {
|
|
if p.Created.After(since) || p.Open() {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it.
|
|
func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) {
|
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) })
|
|
all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)}
|
|
for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50
|
|
other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed)
|
|
other.Repository = "novox/other"
|
|
other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour)
|
|
all = append(all, other)
|
|
}
|
|
plans, err := plansForRetry(t.Context(), all, failed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") {
|
|
t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err)
|
|
}
|
|
}
|
|
|
|
// ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its
|
|
// plan's last change passes 7 days.
|
|
func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) {
|
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
|
old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) })
|
|
f := calm(now)
|
|
f.failed = []failedPlanFacts{failedFactsFor(old)}
|
|
if got := watchPlansFailed(f); len(got) != 0 {
|
|
t.Fatalf("a plan last changed 8 days ago raised %+v", got)
|
|
}
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
defer k.Close(t.Context())
|
|
fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) })
|
|
f.failed = []failedPlanFacts{failedFactsFor(fresh)}
|
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
|
t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open)
|
|
}
|
|
f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old
|
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("the condition outlived the 7 days: %+v", open)
|
|
}
|
|
}
|