2026-10-19 22:44:08 +00:00
3 changed files with 204 additions and 0 deletions
@@ -636,6 +636,8 @@ func doesWords(act conditions.Action) string {
 		return "the delivery starts"
 	case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
 		return "the delivery is stopped"
+	case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "":
+		return "the failed builds are asked again"
 	case strings.HasSuffix(act.Verb, ".restart"):
 		return "its service is restarted on " + act.Machine
 	}
@@ -0,0 +1,141 @@
+//go:build integration
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/asks"
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
+// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
+func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
+	twoTiers(t, open)
+	before := time.Now().UTC().Add(-time.Hour)
+	ended := before.Add(time.Minute)
+	failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
+		Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
+		Note:     "a failed to build in tier 0",
+		Delivery: &inventory.PlanDelivery{},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
+			Why: link.KilledByHand}},
+		Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
+	if err := open.inventory.SavePlan(ctx, &failed); err != nil {
+		t.Fatal(err)
+	}
+
+	conn, err := nats.Connect(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	// The controller's seat answers `plans` as it does: by running the verb.
+	sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
+		var args map[string]string
+		_ = json.Unmarshal(m.Data, &args)
+		answer := link.Answer{Result: json.RawMessage(`"ok"`)}
+		if len(args) != 1 || args["retry"] == "" {
+			answer = link.Answer{Error: "the retry was asked with more than the plan"}
+		} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
+			answer = link.Answer{Error: err.Error()}
+		}
+		body, _ := json.Marshal(answer)
+		_ = m.Respond(body)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+
+	// What S23 raises for it, read from the store as the watchdog reads it.
+	raised := func(context.Context) ([]conditions.Condition, error) {
+		facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
+		if err != nil {
+			return nil, err
+		}
+		var out []conditions.Condition
+		for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
+			out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
+				Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
+		}
+		return out, nil
+	}
+	now := time.Now()
+	store := memAskedStore{}
+	var sent []published
+	var acts []link.HandAct
+	a := &asker{open: raised, store: store,
+		silence: func(context.Context, string, time.Duration, string, string) error { return nil },
+		publish: func(_ context.Context, subject string, body []byte, id string) error {
+			sent = append(sent, published{subject, id, body})
+			return nil
+		},
+		call:   callAction(conn),
+		record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
+		now:    func() time.Time { return now },
+		logf:   t.Logf}
+	cs, _ := raised(ctx)
+	if len(cs) != 1 || len(cs[0].Actions) != 1 {
+		t.Fatalf("the failed plan raised %+v", cs)
+	}
+	if err := a.reconcile(ctx); err != nil {
+		t.Fatal(err)
+	}
+	var w asks.Warrant
+	for _, r := range store {
+		for _, o := range r.Ask.Options {
+			if o.Label == "Retry" {
+				w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
+					Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
+					AskDigest: r.Ask.Digest(),
+					By:        &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
+			}
+		}
+	}
+	if w.Ask == "" {
+		t.Fatalf("no ask offers Retry: %+v", store)
+	}
+	body, _ := json.Marshal(w)
+	if err := a.Decided(ctx, body); err != nil {
+		t.Fatal(err)
+	}
+	if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
+		t.Fatal(err)
+	}
+
+	p, err := open.inventory.PlanByID(ctx, failed.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := p.Modules["a"]
+	if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" {
+		t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
+	}
+	if len(*asked) != 1 {
+		t.Fatalf("the build was asked %d times: %v", len(*asked), *asked)
+	}
+	if got := store[w.Ask]; got.Acted != "done" {
+		t.Fatalf("the ask ended %q", got.Acted)
+	}
+	if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
+		t.Fatalf("the warrant was recorded as %+v", acts)
+	}
+	if cs, _ = raised(ctx); len(cs) != 0 {
+		t.Fatalf("the plan-failed condition did not end: %+v", cs)
+	}
+}
@@ -1013,3 +1013,64 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
 		t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
 	}
 }
+
+// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
+func failedPlanCondition(plan string) conditions.Condition {
+	p := aFailedPlan(plan, nil)
+	f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
+	f.failed = []failedPlanFacts{failedFactsFor(p)}
+	o := watchPlansFailed(f)[0]
+	return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
+		Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
+}
+
+// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
+func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
+	r := newAskerRig(t)
+	c := failedPlanCondition("plan-1792433168676870614")
+	r.open = []conditions.Condition{c}
+	if err := r.a.reconcile(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	sent := r.asksSent(t)
+	if len(sent) != 1 || len(sent[0].Options) != 1 {
+		t.Fatalf("asked %+v", sent)
+	}
+	o := sent[0].Options[0]
+	if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
+		t.Errorf("the option is %+v", o)
+	}
+	if err := sent[0].Check(r.now); err != nil {
+		t.Errorf("the ask is refused as made: %v", err)
+	}
+	if strings.Contains(sent[0].Explanation, "mesh MCP server") {
+		t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
+	}
+}
+
+// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
+// the same warrant heard again does nothing.
+func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
+	plan := "plan-1792433168676870614"
+	r := newAskerRig(t)
+	c := failedPlanCondition(plan)
+	r.open = []conditions.Condition{c}
+	_ = r.a.reconcile(context.Background())
+	w := r.warrantFor(t, c.Key, "Retry")
+	answerWith(t, r, w)
+	answerWith(t, r, w)
+	if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
+		t.Fatalf("called %v, want exactly %s", r.called, want)
+	}
+	if len(r.acts) != 1 {
+		t.Fatalf("hand-acts %+v", r.acts)
+	}
+	act := r.acts[0]
+	if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
+		act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
+		t.Errorf("the hand-act is %+v", act)
+	}
+	if got := r.store[w.Ask]; got.Acted != "done" {
+		t.Errorf("kept %+v", got)
+	}
+}