diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index 4bf3b117..24ca3296 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -636,6 +636,8 @@ func doesWords(act conditions.Action) string { return "the delivery starts" case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "": return "the delivery is stopped" + case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "": + return "the failed builds are asked again" case strings.HasSuffix(act.Verb, ".restart"): return "its service is restarted on " + act.Machine } diff --git a/cmd/mesh-controller/asker_retry_integration_test.go b/cmd/mesh-controller/asker_retry_integration_test.go new file mode 100644 index 00000000..b3c6365b --- /dev/null +++ b/cmd/mesh-controller/asker_retry_integration_test.go @@ -0,0 +1,141 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own +// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 ยง1). +func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + ended := before.Add(time.Minute) + failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Delivery: &inventory.PlanDelivery{}, + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.KilledByHand}}, + Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // The controller's seat answers `plans` as it does: by running the verb. + sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) { + var args map[string]string + _ = json.Unmarshal(m.Data, &args) + answer := link.Answer{Result: json.RawMessage(`"ok"`)} + if len(args) != 1 || args["retry"] == "" { + answer = link.Answer{Error: "the retry was asked with more than the plan"} + } else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil { + answer = link.Answer{Error: err.Error()} + } + body, _ := json.Marshal(answer) + _ = m.Respond(body) + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + + // What S23 raises for it, read from the store as the watchdog reads it. + raised := func(context.Context) ([]conditions.Condition, error) { + facts, err := gatherFailedPlans(ctx, open.inventory, time.Now()) + if err != nil { + return nil, err + } + var out []conditions.Condition + for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) { + out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}) + } + return out, nil + } + now := time.Now() + store := memAskedStore{} + var sent []published + var acts []link.HandAct + a := &asker{open: raised, store: store, + silence: func(context.Context, string, time.Duration, string, string) error { return nil }, + publish: func(_ context.Context, subject string, body []byte, id string) error { + sent = append(sent, published{subject, id, body}) + return nil + }, + call: callAction(conn), + record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil }, + now: func() time.Time { return now }, + logf: t.Logf} + cs, _ := raised(ctx) + if len(cs) != 1 || len(cs[0].Actions) != 1 { + t.Fatalf("the failed plan raised %+v", cs) + } + if err := a.reconcile(ctx); err != nil { + t.Fatal(err) + } + var w asks.Warrant + for _, r := range store { + for _, o := range r.Ask.Options { + if o.Label == "Retry" { + w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen, + Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now, + AskDigest: r.Ask.Digest(), + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + } + } + } + if w.Ask == "" { + t.Fatalf("no ask offers Retry: %+v", store) + } + body, _ := json.Marshal(w) + if err := a.Decided(ctx, body); err != nil { + t.Fatal(err) + } + if err := a.Decided(ctx, body); err != nil { // heard again: nothing more + t.Fatal(err) + } + + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + m := p.Modules["a"] + if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" { + t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m) + } + if len(*asked) != 1 { + t.Fatalf("the build was asked %d times: %v", len(*asked), *asked) + } + if got := store[w.Ask]; got.Acted != "done" { + t.Fatalf("the ask ended %q", got.Acted) + } + if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) { + t.Fatalf("the warrant was recorded as %+v", acts) + } + if cs, _ = raised(ctx); len(cs) != 0 { + t.Fatalf("the plan-failed condition did not end: %+v", cs) + } +} diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 554214d9..71066b70 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -1013,3 +1013,64 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) { t.Errorf("held %v (%q, %v) with nothing to read", held, why, err) } } + +// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it. +func failedPlanCondition(plan string) conditions.Condition { + p := aFailedPlan(plan, nil) + f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)) + f.failed = []failedPlanFacts{failedFactsFor(p)} + o := watchPlansFailed(f)[0] + return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} +} + +// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again". +func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) { + r := newAskerRig(t) + c := failedPlanCondition("plan-1792433168676870614") + r.open = []conditions.Condition{c} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 1 || len(sent[0].Options) != 1 { + t.Fatalf("asked %+v", sent) + } + o := sent[0].Options[0] + if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" { + t.Errorf("the option is %+v", o) + } + if err := sent[0].Check(r.now); err != nil { + t.Errorf("the ask is refused as made: %v", err) + } + if strings.Contains(sent[0].Explanation, "mesh MCP server") { + t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation) + } +} + +// 1315/AC2: the warrant calls plans with retry= and nothing else, once, and is recorded as the hand act; +// the same warrant heard again does nothing. +func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) { + plan := "plan-1792433168676870614" + r := newAskerRig(t) + c := failedPlanCondition(plan) + r.open = []conditions.Condition{c} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, c.Key, "Retry") + answerWith(t, r, w) + answerWith(t, r, w) + if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want { + t.Fatalf("called %v, want exactly %s", r.called, want) + } + if len(r.acts) != 1 { + t.Fatalf("hand-acts %+v", r.acts) + } + act := r.acts[0] + if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan || + act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer { + t.Errorf("the hand-act is %+v", act) + } + if got := r.store[w.Ask]; got.Acted != "done" { + t.Errorf("kept %+v", got) + } +}