diff --git a/cmd/mesh-controller/asker.go b/cmd/mesh-controller/asker.go index 4bf3b117..b2081d9d 100644 --- a/cmd/mesh-controller/asker.go +++ b/cmd/mesh-controller/asker.go @@ -208,8 +208,11 @@ type asker struct { // raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be // asked, and why. Nil raises nothing (a test that does not look). raise func(ctx context.Context, obs []conditions.Observation) error - now func() time.Time - logf func(string, ...any) + // tell tells the operator one plain notice that needs no answer (operator-channel.tell, novox/hq ADR 0380): + // the refusal of an answer to a plan's ask (ADR 0449 §3). Nil tells nothing (a test that does not look). + tell func(ctx context.Context, headline string, lines []string) error + now func() time.Time + logf func(string, ...any) saidNoRouter bool saidNoGrant bool @@ -406,7 +409,9 @@ func (a *asker) reconcile(ctx context.Context) error { for _, r := range all { k := partKey(r.Condition, r.Part) if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer { - answered[k] = r + if prior, has := answered[k]; !has || r.Ended.After(prior.Ended) { + answered[k] = r + } } if r.State == string(asks.OutcomeRefused) { if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) { @@ -497,7 +502,9 @@ func (a *asker) reconcile(ctx context.Context) error { continue // refused lately, and nothing it was refused for has changed: asked again after the wait } } - if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) { + // **A condition raised after the answered ask ended is a new one** (novox/hq ADR 0449 §2): a retried + // plan that fails again is asked again, not held back for the hour its earlier answer holds. + if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) && !c.Raised.After(r.Ended) { if _, held := byCondition[key]; !held { continue } @@ -636,6 +643,8 @@ func doesWords(act conditions.Action) string { return "the delivery starts" case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "": return "the delivery is stopped" + case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "": + return "the failed builds are asked again" case strings.HasSuffix(act.Verb, ".restart"): return "its service is restarted on " + act.Machine } @@ -884,6 +893,7 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { ended := a.now() if acted != nil { outcome = "failed: " + acted.Error() + a.tellRefused(ctx, *r, act, acted) } r.Ended, r.Acted = ended, outcome if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool { @@ -918,6 +928,55 @@ func (a *asker) Decided(ctx context.Context, body []byte) error { return nil } +// tellRefused tells the operator what became of the answer they gave to a plan's ask, when it was not done (novox/hq +// ADR 0449 §3): the controller's `plans` verb refuses a Retry, Start or Stop when the plan moved on between the ask and +// the answer, and the refusal was only recorded, so the operator who pressed the answer heard nothing. **A refusal is +// told as one, and nothing else is**: a verb that said no changed nothing, but a call that got no answer (a timeout, a +// refused grant) may have run, and is told as not known. One plain notice with the ask's headline, beside the hand act +// the caller still records. An answer to any other ask is not told here. A notice that cannot be told is said in the +// log and changes nothing. +func (a *asker) tellRefused(ctx context.Context, r asked, act conditions.Action, failure error) { + if a.tell == nil || act.Verb != broker.ControllerSeat+".plans" { + return + } + var lines []string + var refused verbRefused + if errors.As(failure, &refused) { + lines = []string{ + fmt.Sprintf("Your answer, %s, could not be done: %s.", act.Label, strings.TrimRight(refusalWords(refused.words, act), ".")), + "Nothing was changed.", + } + } else { + lines = []string{ + fmt.Sprintf("Your answer, %s, was given, and it is not known whether it was done.", act.Label), + "Look at the delivery before you answer again.", + } + } + if err := a.tell(ctx, r.Ask.Headline, lines); err != nil { + a.logf("the operator could NOT be told that %s was refused (ask %s): %v", act.Label, r.ID, err) + } +} + +// refusalWords is a verb's refusal as the operator reads it: the plans named as "the plan" (plainRefusal), and, where +// the words still carry an identifier the operator's channel withholds, one sentence of the answer's own. +func refusalWords(words string, act conditions.Action) string { + return plainRefusalOr(words, refusalFallback(act)) +} + +// refusalFallback is what an answer's refusal says when its own words cannot be said: one sentence for each answer +// to a plan's ask, never another answer's. +func refusalFallback(act conditions.Action) string { + switch { + case act.Arguments["retry"] != "": + return "a retry would not run now; the details say why" + case act.Arguments["go"] != "": + return "the delivery could not be started now; the details say why" + case act.Arguments["stop"] != "": + return "the delivery could not be stopped now; the details say why" + } + return "the controller refused it; the details say why" +} + // handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision, // never a repair (handActVerbs). const handActWarrant = "warrant" diff --git a/cmd/mesh-controller/asker_retry_integration_test.go b/cmd/mesh-controller/asker_retry_integration_test.go new file mode 100644 index 00000000..b3c6365b --- /dev/null +++ b/cmd/mesh-controller/asker_retry_integration_test.go @@ -0,0 +1,141 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own +// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1). +func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + ended := before.Add(time.Minute) + failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Delivery: &inventory.PlanDelivery{}, + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.KilledByHand}}, + Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // The controller's seat answers `plans` as it does: by running the verb. + sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) { + var args map[string]string + _ = json.Unmarshal(m.Data, &args) + answer := link.Answer{Result: json.RawMessage(`"ok"`)} + if len(args) != 1 || args["retry"] == "" { + answer = link.Answer{Error: "the retry was asked with more than the plan"} + } else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil { + answer = link.Answer{Error: err.Error()} + } + body, _ := json.Marshal(answer) + _ = m.Respond(body) + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + + // What S23 raises for it, read from the store as the watchdog reads it. + raised := func(context.Context) ([]conditions.Condition, error) { + facts, err := gatherFailedPlans(ctx, open.inventory, time.Now()) + if err != nil { + return nil, err + } + var out []conditions.Condition + for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) { + out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}) + } + return out, nil + } + now := time.Now() + store := memAskedStore{} + var sent []published + var acts []link.HandAct + a := &asker{open: raised, store: store, + silence: func(context.Context, string, time.Duration, string, string) error { return nil }, + publish: func(_ context.Context, subject string, body []byte, id string) error { + sent = append(sent, published{subject, id, body}) + return nil + }, + call: callAction(conn), + record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil }, + now: func() time.Time { return now }, + logf: t.Logf} + cs, _ := raised(ctx) + if len(cs) != 1 || len(cs[0].Actions) != 1 { + t.Fatalf("the failed plan raised %+v", cs) + } + if err := a.reconcile(ctx); err != nil { + t.Fatal(err) + } + var w asks.Warrant + for _, r := range store { + for _, o := range r.Ask.Options { + if o.Label == "Retry" { + w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen, + Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now, + AskDigest: r.Ask.Digest(), + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} + } + } + } + if w.Ask == "" { + t.Fatalf("no ask offers Retry: %+v", store) + } + body, _ := json.Marshal(w) + if err := a.Decided(ctx, body); err != nil { + t.Fatal(err) + } + if err := a.Decided(ctx, body); err != nil { // heard again: nothing more + t.Fatal(err) + } + + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + m := p.Modules["a"] + if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" { + t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m) + } + if len(*asked) != 1 { + t.Fatalf("the build was asked %d times: %v", len(*asked), *asked) + } + if got := store[w.Ask]; got.Acted != "done" { + t.Fatalf("the ask ended %q", got.Acted) + } + if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) { + t.Fatalf("the warrant was recorded as %+v", acts) + } + if cs, _ = raised(ctx); len(cs) != 0 { + t.Fatalf("the plan-failed condition did not end: %+v", cs) + } +} diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 554214d9..40604d7e 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -77,6 +77,15 @@ type askerRig struct { silenced []string acts []link.HandAct now time.Time + // refuses, when set, is what every verb the asker calls answers; told is every notice the operator was told. + refuses error + told []told +} + +// told is one plain notice the asker told the operator. +type told struct { + headline string + lines []string } func newAskerRig(t *testing.T) *askerRig { @@ -102,6 +111,10 @@ func newAskerRig(t *testing.T) *askerRig { call: func(_ context.Context, a conditions.Action, args map[string]string) error { raw, _ := json.Marshal(args) r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw)) + return r.refuses + }, + tell: func(_ context.Context, headline string, lines []string) error { + r.told = append(r.told, told{headline, lines}) return nil }, record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil }, @@ -1013,3 +1026,215 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) { t.Errorf("held %v (%q, %v) with nothing to read", held, why, err) } } + +// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it. +func failedPlanCondition(plan string) conditions.Condition { + p := aFailedPlan(plan, nil) + f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)) + f.failed = []failedPlanFacts{failedFactsFor(p)} + o := watchPlansFailed(f)[0] + return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, + Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} +} + +// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again". +func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) { + r := newAskerRig(t) + c := failedPlanCondition("plan-1792433168676870614") + r.open = []conditions.Condition{c} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + sent := r.asksSent(t) + if len(sent) != 1 || len(sent[0].Options) != 1 { + t.Fatalf("asked %+v", sent) + } + o := sent[0].Options[0] + if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" { + t.Errorf("the option is %+v", o) + } + if err := sent[0].Check(r.now); err != nil { + t.Errorf("the ask is refused as made: %v", err) + } + if strings.Contains(sent[0].Explanation, "mesh MCP server") { + t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation) + } +} + +// 1315/AC2: the warrant calls plans with retry= and nothing else, once, and is recorded as the hand act; +// the same warrant heard again does nothing. +func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) { + plan := "plan-1792433168676870614" + r := newAskerRig(t) + c := failedPlanCondition(plan) + r.open = []conditions.Condition{c} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, c.Key, "Retry") + answerWith(t, r, w) + answerWith(t, r, w) + if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want { + t.Fatalf("called %v, want exactly %s", r.called, want) + } + if len(r.acts) != 1 { + t.Fatalf("hand-acts %+v", r.acts) + } + act := r.acts[0] + if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan || + act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer { + t.Errorf("the hand-act is %+v", act) + } + if got := r.store[w.Ask]; got.Acted != "done" { + t.Errorf("kept %+v", got) + } +} + +// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is +// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before. +func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) { + plan := "plan-1792433168676870614" + waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent, + Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.", + Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)} + for _, tc := range []struct { + name string + c conditions.Condition + label string + }{ + {"retry", failedPlanCondition(plan), "Retry"}, + {"stop", waiting, "Stop"}, + } { + t.Run(tc.name, func(t *testing.T) { + r := newAskerRig(t) + r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"} + r.open = []conditions.Condition{tc.c} + _ = r.a.reconcile(context.Background()) + w := r.warrantFor(t, tc.c.Key, tc.label) + answerWith(t, r, w) + answerWith(t, r, w) // heard again: told once + if len(r.told) != 1 { + t.Fatalf("told %+v", r.told) + } + got := r.told[0] + if got.headline != tc.c.Headline { + t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline) + } + said := strings.Join(got.lines, " ") + if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) { + t.Errorf("the lines do not carry the answer and the refusal's words: %q", said) + } + if why, ok := conditions.Plain(said); !ok { + t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said) + } + if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") { + t.Errorf("the failure is not recorded as before: %+v", r.acts) + } + if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") { + t.Errorf("kept %+v", kept) + } + }) + } +} + +// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here. +func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) { + r := newAskerRig(t) + c := failedPlanCondition("plan-1792433168676870614") + r.open = []conditions.Condition{c} + _ = r.a.reconcile(context.Background()) + answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) + if len(r.told) != 0 { + t.Fatalf("a retry that was done told %+v", r.told) + } + + r = newAskerRig(t) + r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"} + module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning, + Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.", + Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart", + Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove, + Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}} + r.open = []conditions.Condition{module} + _ = r.a.reconcile(context.Background()) + answerWith(t, r, r.warrantFor(t, module.Key, "Restart")) + if len(r.told) != 0 || len(r.acts) != 1 { + t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts) + } +} + +// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was +// changed"; and the sentence a refusal falls back to fits the answer given. +func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) { + plan := "plan-1792433168676870614" + r := newAskerRig(t) + r.refuses = errors.New("mesh-controller did not answer plans within 30s") + c := failedPlanCondition(plan) + r.open = []conditions.Condition{c} + _ = r.a.reconcile(context.Background()) + answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) + if len(r.told) != 1 { + t.Fatalf("told %+v", r.told) + } + said := strings.Join(r.told[0].lines, " ") + if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") { + t.Errorf("a call that may have run is told as %q", said) + } + if why, ok := conditions.Plain(said); !ok { + t.Errorf("not plain: %s: %q", why, said) + } +} + +func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) { + plan := "plan-1792433168676870614" + for _, tc := range []struct { + act conditions.Action + want string + }{ + {retryAction(plan), "a retry would not run now"}, + {conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"}, + {conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"}, + } { + // Words the operator's channel withholds: a hash and a dotted name. + got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act) + if !strings.Contains(got, tc.want) { + t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want) + } + if _, ok := conditions.Plain(got); !ok { + t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got) + } + } +} + +// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the +// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not. +func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) { + r := newAskerRig(t) + c := failedPlanCondition("plan-1792433168676870614") + c.Raised = r.now.Add(-time.Minute) + r.open = []conditions.Condition{c} + _ = r.a.reconcile(context.Background()) + answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) + // Still open right after the answer: not asked again at once. + r.now = r.now.Add(time.Minute) + _ = r.a.reconcile(context.Background()) + if n := len(r.asksSent(t)); n != 1 { + t.Fatalf("asked again while the same condition stayed open: %d", n) + } + // The retry ran, the plan left failed and the condition ended. + r.now = r.now.Add(time.Minute) + r.open = nil + _ = r.a.reconcile(context.Background()) + // It fails again, ten minutes after the answer: a condition raised after the answered ask ended. + r.now = r.now.Add(8 * time.Minute) + again := failedPlanCondition("plan-1792433168676870614") + again.Raised = r.now + r.open = []conditions.Condition{again} + if err := r.a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if n := len(r.asksSent(t)); n != 2 { + t.Fatalf("the second failure was asked %d times in all, want 2", n) + } + if len(r.called) != 1 { + t.Fatalf("something retried without an answer: %v", r.called) + } +} diff --git a/cmd/mesh-controller/asker_wire.go b/cmd/mesh-controller/asker_wire.go index d6ba3be5..66651dc5 100644 --- a/cmd/mesh-controller/asker_wire.go +++ b/cmd/mesh-controller/asker_wire.go @@ -135,6 +135,12 @@ func (b busAsked) All(ctx context.Context) ([]asked, error) { return out, nil } +// verbRefused is a verb's own refusal of an act: the seat answered, and said no. Any other error of callAction (a bus +// that did not answer, a grant that is refused) is not one: the verb may have run. +type verbRefused struct{ verb, words string } + +func (e verbRefused) Error() string { return e.verb + " refused: " + e.words } + // callAction performs an action's verb as the controller, through the grant that names it. func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error { return func(ctx context.Context, a conditions.Action, args map[string]string) error { @@ -168,7 +174,23 @@ func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, return err } if answer.Error != "" { - return fmt.Errorf("%s refused: %s", a.Verb, answer.Error) + return verbRefused{verb: a.Verb, words: answer.Error} + } + return nil + } +} + +// tellThrough tells the operator one plain notice through the operator channel's `tell`, over the controller's own +// connection and grant (novox/hq ADR 0380); a holder's refusal is an error naming it. +func tellThrough(conn *nats.Conn) func(ctx context.Context, headline string, lines []string) error { + return func(ctx context.Context, headline string, lines []string) error { + answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell", + map[string]any{"headline": headline, "lines": lines}, tellWithin) + if err != nil { + return err + } + if answer.Error != "" { + return fmt.Errorf("operator-channel.tell refused: %s", answer.Error) } return nil } @@ -576,6 +598,7 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper _, err := link.RecordHandAct(ctx, conn, act) return err }, + tell: tellThrough(conn), routerRecord: routerRecordOf(conn, open.inventory), routerHere: routerHereIn(open.inventory), grantHeld: grantHeldIn(open, conn), diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 6b451ef7..8fb90f96 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -72,6 +72,10 @@ var handActVerbs = []handActVerb{ // An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a // channel that proved who answered, performed by the controller as itself. {Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"}, + // A seat passed by the controller to its only claimant when its holder on record left (novox/hq ADR 0449 + // §4): the mesh working as decided. A person's own `seat --to` is not recorded under this cause. + {Verb: "seat", Decision: "the controller passes a seat held once to its only claimant when its recorded " + + "holder is assigned nowhere (ADR 0449 §4)", DecidedFor: []string{"seat-passed"}}, {Verb: "broker consumer-reset"}, // Silencing the same condition twice says the condition, or what it watches, wants mending — unless // it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258). diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index f07fa940..965b8892 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -429,6 +429,11 @@ var plainWordings = map[string]func(conditions.Observation) words{ "out) has not let it start yet.", Resolved: "Resolved: the delivery is no longer waiting"} }), + kindPlanFailed: worded(func(o conditions.Observation) words { + return words{Headline: "A delivery failed", + Explanation: "A delivery did not finish. Nothing is tried again until you say so.", + Resolved: "Resolved: the delivery is no longer failed"} + }), "ask-lost": worded(func(o conditions.Observation) words { return words{Headline: "A build has no result", Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index a0efccf0..108c6d83 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -285,12 +285,28 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory, assigned []string } var others []candidate + // **Every machine's assignments, before the holdings are read by any resolution** (novox/hq ADR 0449 §4): a + // seat held once whose holder on record is assigned on no machine needs the whole mesh to be known, and it + // passes to its only claimant here, in the composition that finds it. + everyAssigned := map[string][]string{} + var assignedErr error + for _, n := range nodes { + theirs, err := inv.Assigned(ctx, n.Name) + if err != nil { + assignedErr = errors.Join(assignedErr, fmt.Errorf("%s: %w", n.Name, err)) + continue + } + if len(theirs) > 0 { + everyAssigned[n.Name] = theirs + } + } + holdings = theMeshPassesSeatsOn(ctx, inv, shelf, everyAssigned, assignedErr, holdings) for _, n := range nodes { if n.Name == exclude { continue } - theirs, err := inv.Assigned(ctx, n.Name) - if err != nil || len(theirs) == 0 { + theirs := everyAssigned[n.Name] + if len(theirs) == 0 { continue } caps, _ := inv.ProfileOf(ctx, n.Name) diff --git a/cmd/mesh-controller/plan_failed.go b/cmd/mesh-controller/plan_failed.go new file mode 100644 index 00000000..2e3011ec --- /dev/null +++ b/cmd/mesh-controller/plan_failed.go @@ -0,0 +1,227 @@ +package main + +import ( + "context" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// failedModule is one module of a failed plan that failed to build, where it runs and what it said. +type failedModule struct { + module string + machines []string + why string +} + +// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1). +type failedPlanFacts struct { + id, repository, commit string + modules []string + failed []failedModule + // note is where the plan stopped when no module failed to build (a rollout stopped at its first machine). + note string + // since is the plan's last change: when it failed. + since time.Time + // refusal is why `plans retry` would not run it, empty when it would. + refusal string + // passed is set when a newer plan of its repository and branch is done and the retry is refused for that + // (novox/hq ADR 0451): the condition asks nothing. + passed bool +} + +// machines are the machines the failed modules run on, sorted, each once. +func (p failedPlanFacts) machines() []string { + seen := map[string]bool{} + var out []string + for _, m := range p.failed { + for _, n := range m.machines { + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + } + sort.Strings(out) + return out +} + +// failureSaid is what failed, in the mesh's own words, for the condition's evidence. +func (p failedPlanFacts) failureSaid() string { + var parts []string + for _, m := range p.failed { + line := m.module + if len(m.machines) > 0 { + line += " on " + strings.Join(m.machines, ", ") + } + if m.why != "" { + line += ": " + oneLine(m.why) + } + parts = append(parts, line) + } + if len(parts) == 0 { + return "stopped: " + oneLine(p.note) + } + return strings.Join(parts, "; ") +} + +// refusalSaid is why a retry would not run, for the evidence. +func (p failedPlanFacts) refusalSaid() string { + if p.refusal == "" { + return "" + } + return "; a retry would not run: " + oneLine(p.refusal) +} + +// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan, +// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs). +func retryAction(plan string) conditions.Action { + return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, + Arguments: map[string]string{"retry": plan}} +} + +var ( + planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`) + walkWd = regexp.MustCompile(`\bwalks\b`) + walkOne = regexp.MustCompile(`\bwalk\b`) +) + +// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a +// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the +// words say only that a retry would not run. +func plainRefusal(refusal string) string { + return plainRefusalOr(refusal, "a retry would not run now; the details say why") +} + +// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly. +func plainRefusalOr(refusal, fallback string) string { + r := planIDWord.ReplaceAllString(refusal, "the plan") + r = walkWd.ReplaceAllString(r, "deliveries") + r = walkOne.ReplaceAllString(r, "delivery") + r = strings.TrimSpace(r) + if _, ok := conditions.Plain(r); !ok { + return fallback + } + return r +} + +// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that +// is plain, and Retry when a retry would run, else the refusal's words. +func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) { + name := deliveryName(p.modules, p.repository) + headline = conditions.Capital(name) + " failed" + what := deliveryWhat(p.modules, p.repository) + var parts []string + for i, m := range p.failed { + if i == 2 { + parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2)) + break + } + line := m.module + if len(m.machines) > 0 { + line += " for " + readableList(m.machines) + } + if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" { + if _, ok := conditions.Plain(w); ok { + line += " (" + w + ")" + } + } + parts = append(parts, line) + } + if len(parts) > 0 { + explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts)) + } else { + explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what) + } + switch { + case p.passed: + explanation += " A newer delivery has passed it, so a retry would send older code. Nothing is needed from you." + case p.refusal == "": + explanation += " Nothing is tried again until you say so." + needs = "try the failed builds again " + FromMeshMCPServer + actions = []conditions.Action{retryAction(p.id)} + default: + explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "." + needs = "look at why it failed; a retry would not run now." + } + resolved = conditions.Capital(name) + " is no longer failed" + return +} + +// cutWords is s cut to at most n bytes at a word. +func cutWords(s string, n int) string { + if len(s) <= n { + return s + } + t := s[:n] + if i := strings.LastIndexByte(t, ' '); i > n/2 { + t = t[:i] + } + return strings.TrimRight(t, " ,;:") + "…" +} + +// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator +// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not +// stopped by a person. Each carries why `plans retry` would not run it, when it would not. +func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) { + plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin)) + if err != nil { + return nil, fmt.Errorf("the plans cannot be read: %w", err) + } + var failed []inventory.Plan + for _, p := range plans { + if failedAndAsked(p) { + failed = append(failed, p) + } + } + if len(failed) == 0 { + return nil, nil + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, fmt.Errorf("the catalogue cannot be read: %w", err) + } + onPath := ownPathOf(entries) + runsOn := map[string][]string{} + for _, e := range entries { + runsOn[e.Manifest.Module] = e.On + } + var out []failedPlanFacts + for _, p := range failed { + out = append(out, failedFactsOf(p, plans, onPath, runsOn)) + } + return out, nil +} + +// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not +// stopped by a person. +func failedAndAsked(p inventory.Plan) bool { + return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == "" +} + +// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each +// module runs. +func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string, + runsOn map[string][]string) failedPlanFacts { + f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p), + note: p.Note, since: p.Updated} + for _, m := range f.modules { + if s := p.Modules[m]; s != nil && s.State == "failed" { + machines := append([]string(nil), runsOn[m]...) + sort.Strings(machines) + f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why}) + } + } + if err := retryRefusal(p, plans, onPath); err != nil { + f.refusal = err.Error() + var passed passedBy + f.passed = errors.As(err, &passed) + } + return f +} diff --git a/cmd/mesh-controller/plan_failed_test.go b/cmd/mesh-controller/plan_failed_test.go new file mode 100644 index 00000000..05691840 --- /dev/null +++ b/cmd/mesh-controller/plan_failed_test.go @@ -0,0 +1,338 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314). + +func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan { + p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed, + Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC), + Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}}, + Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} + if mutate != nil { + mutate(&p) + } + return p +} + +func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts { + return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{}, + map[string][]string{"openrazer": {"g14", "shanks"}}) +} + +// 1314/AC1 +func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + p := aFailedPlan("plan-1792433168676870614", nil) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(p)} + got := watchPlansFailed(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator || + o.Scope != conditions.ScopePlan || o.ID != p.ID { + t.Fatalf("kind, severity, resolver or subject: %+v", o) + } + if len(o.Actions) != 1 { + t.Fatalf("actions: %+v", o.Actions) + } + a := o.Actions[0] + if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove || + len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID { + t.Fatalf("the action is not Retry with only the plan: %+v", a) + } + if o.Needs == "" { + t.Fatal("no Needs: the asker wants an action with a need") + } + for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} { + if !strings.Contains(o.Headline+o.Explanation+o.Said, want) { + t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said) + } + } + if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") { + t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation) + } + words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions} + if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { + t.Errorf("the words are not plain: %s\n%+v", why, words) + } +} + +// 1314/AC2 +func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) { + for name, p := range map[string]inventory.Plan{ + "held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }), + "stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }), + "no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }), + "building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }), + } { + if failedAndAsked(p) { + t.Errorf("%s: a plan not asked of the operator is asked", name) + } + } + if !failedAndAsked(aFailedPlan("plan-5", nil)) { + t.Error("a failed delivery plan is not asked") + } +} + +// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it. +func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) }) + newer := aFailedPlan("plan-2", func(p *inventory.Plan) { + p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute) + }) + superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" }) + done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone }) + first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC) + gate := aFailedPlan("plan-5", func(p *inventory.Plan) { + p.Note = "openrazer stopped at its first machine" + p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"}, + Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"}, + Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}} + }) + for name, c := range map[string]struct { + p inventory.Plan + others []inventory.Plan + says string + }{ + "a newer plan": {older, []inventory.Plan{newer}, "supersedes it"}, + "superseded": {superseded, nil, "superseded"}, + "done": {done, nil, "nothing to retry"}, + "gate failed": {gate, nil, "failed its gate"}, + } { + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)} + got := watchPlansFailed(f) + if len(got) != 1 { + t.Fatalf("%s: %+v", name, got) + } + o := got[0] + if len(o.Actions) != 0 { + t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions) + } + if !strings.Contains(o.Said, c.says) { + t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said) + } + if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) { + t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation) + } + words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} + if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { + t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words) + } + } +} + +// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23. +func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + defer k.Close(t.Context()) + p := aFailedPlan("plan-1792433168676870614", nil) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(p)} + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + open, _ := k.Open(t.Context()) + if len(open) != 1 || open[0].Kind != "plan-failed" { + t.Fatalf("not raised: %+v", open) + } + // Retried: the plan is building, so it is no longer among the failed. + f = calm(now.Add(time.Minute)) + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + if open, _ = k.Open(t.Context()); len(open) != 0 { + t.Fatalf("the condition outlived the failure: %+v", open) + } + // Failed again: asked again, with Retry, and nothing retried it. + f = calm(now.Add(2 * time.Minute)) + again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) }) + f.failed = []failedPlanFacts{failedFactsFor(again)} + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + open, _ = k.Open(t.Context()) + if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" { + t.Fatalf("a second failure was not asked again with Retry: %+v", open) + } +} + +// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing +// S23 produces is an observation that offers an answer. +func TestNoClockRetriesAFailedPlan(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))} + for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} { + f.now = now.Add(later) + got := watchPlansFailed(f) + if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" { + t.Fatalf("after %s: %+v", later, got) + } + } + // A plan failed longer ago than the week it is kept is left to `plans`. + f.now = now.Add(planFailedWithin + time.Second) + if got := watchPlansFailed(f); len(got) != 0 { + t.Fatalf("a plan failed beyond the window is still raised: %+v", got) + } +} + +// A later plan of the same repository and branch, in a state, made after the failed one. +func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan { + q := aFailedPlan(id, func(p *inventory.Plan) { + p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour) + }) + return q +} + +// 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan. +func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) { + failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) + later := laterPlan("plan-2", "main", inventory.PlanDone, failed) + err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{}) + if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") { + t.Fatalf("the retry was not refused, naming the later plan: %v", err) + } +} + +// 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing. +func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) { + failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) + open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed) + err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{}) + if err == nil || !strings.Contains(err.Error(), "supersedes it") { + t.Fatalf("with the later plan open the refusal is not today's: %v", err) + } + other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed) + if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil { + t.Fatalf("a later done plan of another branch refused the retry: %v", err) + } + earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed) + earlier.Created = failed.Created.Add(-time.Hour) + if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil { + t.Fatalf("an earlier done plan refused the retry: %v", err) + } +} + +// 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery. +func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) }) + later := laterPlan("plan-2", "main", inventory.PlanDone, failed) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(failed, later)} + got := watchPlansFailed(f) + if len(got) != 1 || len(got[0].Actions) != 0 { + t.Fatalf("%+v", got) + } + o := got[0] + if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") { + t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation) + } + if strings.Contains(o.Explanation, "details say why") { + t.Errorf("the generic fallback was said: %q", o.Explanation) + } + words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} + if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { + t.Errorf("not plain: %s\n%+v", why, words) + } +} + +// plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since. +type plansBehind []inventory.Plan + +func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) { + var out []inventory.Plan + for _, p := range s { + if p.Open() { + out = append(out, p) + } + } + return out, nil +} + +func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) { + all := append([]inventory.Plan(nil), s...) + sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) }) + if len(all) > limit { + all = all[:limit] + } + return all, nil +} + +func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) { + var out []inventory.Plan + for _, p := range s { + if p.Created.After(since) || p.Open() { + out = append(out, p) + } + } + return out, nil +} + +// 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it. +func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) { + failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) }) + all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)} + for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50 + other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed) + other.Repository = "novox/other" + other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour) + all = append(all, other) + } + plans, err := plansForRetry(t.Context(), all, failed) + if err != nil { + t.Fatal(err) + } + if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") { + t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err) + } +} + +// ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its +// plan's last change passes 7 days. +func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) { + now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) + old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) }) + f := calm(now) + f.failed = []failedPlanFacts{failedFactsFor(old)} + if got := watchPlansFailed(f); len(got) != 0 { + t.Fatalf("a plan last changed 8 days ago raised %+v", got) + } + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + defer k.Close(t.Context()) + fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) }) + f.failed = []failedPlanFacts{failedFactsFor(fresh)} + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + if open, _ := k.Open(t.Context()); len(open) != 1 { + t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open) + } + f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old + if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { + t.Fatal(err) + } + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("the condition outlived the 7 days: %+v", open) + } +} diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index aa9adf7b..6e5e99ed 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -154,6 +154,31 @@ func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bo return inventory.Plan{}, false } +// newerDonePlan is a done plan of the same repository and branch made after this one: the newer commit it delivered +// runs now (novox/hq ADR 0451). +func newerDonePlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { + var found inventory.Plan + var ok bool + for _, q := range plans { + if q.ID == p.ID || q.State != inventory.PlanDone || !strings.EqualFold(q.Repository, p.Repository) || + (p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) { + continue + } + if !ok || q.Created.After(found.Created) { + found, ok = q, true + } + } + return found, ok +} + +// passedBy is the refusal of a retry of a plan that a newer done plan of its repository and branch has passed. +type passedBy struct{ plan, by, repository, commit string } + +func (e passedBy) Error() string { + return fmt.Sprintf("%s was passed by %s, a newer plan of %s (at %s) that is done: retrying it would build and "+ + "send the older commit over the newer one that runs", e.plan, e.by, e.repository, short(e.commit)) +} + // retryRefusal is why a plan cannot be retried, or nothing. func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error { switch { @@ -165,6 +190,11 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild ` asks one module again", p.ID, p.State) } + // **A newer delivery of the branch has passed it** (novox/hq ADR 0451): retrying would build and send the older + // commit over the newer one that runs. + if q, found := newerDonePlan(p, plans); found { + return passedBy{plan: p.ID, by: q.ID, repository: q.Repository, commit: q.Commit} + } if len(failedIn(p)) > 0 { if q, found := newerOpenPlan(p, plans); found { return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+ @@ -275,6 +305,32 @@ func goesOnAgain(p *inventory.Plan) { } } +// planReader is what a retry reads the mesh's plans through. +type planReader interface { + OpenPlans(ctx context.Context) ([]inventory.Plan, error) + RecentPlans(ctx context.Context, limit int) ([]inventory.Plan, error) + PlansSince(ctx context.Context, since time.Time) ([]inventory.Plan, error) +} + +// plansForRetry are the plans a retry of p judges itself against. +func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inventory.Plan, error) { + plans, err := inv.OpenPlans(ctx) + if err != nil { + return nil, err + } + recent, err := inv.RecentPlans(ctx, 50) + if err != nil { + return nil, err + } + // **Every plan made after this one** (novox/hq ADR 0451 §1): a plan that passed it is seen however many plans + // were made since, not only when it is among the newest. + later, err := inv.PlansSince(ctx, p.Created) + if err != nil { + return nil, err + } + return append(append(plans, recent...), later...), nil +} + // retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at // that tier: what follows is the plan going on as if they had built the first time. func retryPlan(ctx context.Context, open *stores, id string) (string, error) { @@ -288,15 +344,10 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { if err != nil { return "", err } - plans, err := inv.OpenPlans(ctx) + plans, err := plansForRetry(ctx, inv, p) if err != nil { return "", err } - recent, err := inv.RecentPlans(ctx, 50) - if err != nil { - return "", err - } - plans = append(plans, recent...) // **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for // the merge that brought the failure answers them now, and a retried walk would name them twice. // **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped, diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index fe1a4674..82834cec 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -79,11 +79,14 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) { if err := retryRefusal(stopped, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") { t.Errorf("a plan with nothing failed to build was retried: %v", err) } - // A failed or done plan does not supersede it. - if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour)), - plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}, ownPathOf(nil)); err != nil { + // A later failed plan does not supersede it; a later done one passes it (novox/hq ADR 0451). + if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}, ownPathOf(nil)); err != nil { t.Errorf("refused for a plan that does not supersede it: %v", err) } + if err := retryRefusal(failed, []inventory.Plan{plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}, ownPathOf(nil)); err == nil || + !strings.Contains(err.Error(), "plan-5") { + t.Errorf("a plan a later done plan passed was retried: %v", err) + } // Another branch's open walk, or an older one, does not supersede it, and is one walk open: one at a time // (novox/hq ADR 0276). other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour)) diff --git a/cmd/mesh-controller/seat_pass.go b/cmd/mesh-controller/seat_pass.go new file mode 100644 index 00000000..8995c530 --- /dev/null +++ b/cmd/mesh-controller/seat_pass.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// seatPassing is what passing seats on needs of the outside: the record's compare-and-set, and the hand act. +type seatPassing struct { + pass func(ctx context.Context, p catalogue.SeatPass) (bool, error) + act func(ctx context.Context, p catalogue.SeatPass) error + say io.Writer +} + +// passSeatsOn records, in the composition that finds it, the new holder of each seat held once whose holder on +// record is assigned nowhere and which has exactly one claimant able to hold it, and writes the controller's own +// hand act for it (novox/hq ADR 0449 §4, issue 1317). It answers the holdings as they are after it, so the +// resolution that follows reads the record it just moved. A failed write is said and leaves the record as it was: +// a composition is never refused for it. +func passSeatsOn(ctx context.Context, sp seatPassing, holdings []catalogue.Held, + assignments []catalogue.Assignment) []catalogue.Held { + passes, said := catalogue.SeatsToPassOn(holdings, assignments) + for _, line := range said { + fmt.Fprintln(sp.say, line) + } + for _, p := range passes { + moved, err := sp.pass(ctx, p) + if err != nil { + fmt.Fprintf(sp.say, "%s could not be passed from %s on %s to %s on %s: %v\n", p.Seat.Name, + p.From.Module, p.From.Node, p.To.Module, p.To.Node, err) + continue + } + if !moved { + continue // another composition passed it first, and wrote the act + } + if err := sp.act(ctx, p); err != nil { + fmt.Fprintf(sp.say, "%s passed from %s on %s to %s on %s, and the hand act could NOT be recorded: %v\n", + p.Seat.Name, p.From.Module, p.From.Node, p.To.Module, p.To.Node, err) + } + var next []catalogue.Held + for _, h := range holdings { + if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == p.Seat.Name { + continue + } + next = append(next, h) + } + p.To.Site = p.From.Site + holdings = append(next, p.To) + } + return holdings +} + +// seatPassHandAct is the controller's own entry for a pass: it is the act that puts the holder on record, +// where a person's `seat --to` would have, so the record of who held what stays complete (ADR 0449 §4). +func seatPassHandAct(p catalogue.SeatPass) link.HandAct { + return link.HandAct{ + By: "the controller", + Verb: "seat", + Args: []string{p.Seat.Name, "--to", p.To.Node + "/" + p.To.Module, "--was", p.From.Node + "/" + p.From.Module}, + Why: fmt.Sprintf("%s on %s, the holder on record of %s, is assigned on no machine, and %s on %s is the only "+ + "assignment that claims it and can hold it (ADR 0449 §4)", p.From.Module, p.From.Node, p.Seat.Name, + p.To.Module, p.To.Node), + Cause: "seat-passed", + Outcome: "done", + } +} + +// assignmentsOf are the assignments of every machine, as the passing of a seat reads them. **A module assigned and +// missing from the shelf is still assigned**: it is an assignment with no claims, so its holder is not taken for gone. +func assignmentsOf(shelf map[string]catalogue.Manifest, assigned map[string][]string) []catalogue.Assignment { + var out []catalogue.Assignment + for node, modules := range assigned { + for _, module := range modules { + m, ok := shelf[module] + if !ok { + m = catalogue.Manifest{Module: module} + } + out = append(out, catalogue.Assignment{Node: node, Manifest: m}) + } + } + return out +} + +// seatPassingOn is how a composition on this inventory passes a seat: against the store's record with the hand act +// on the bus, or, on a throwaway mesh (the merge gate's), in memory only, writing nothing anywhere. +func seatPassingOn(inv *inventory.Inventory, w io.Writer) seatPassing { + if inv.Throwaway() { + return seatPassing{say: w, + pass: func(context.Context, catalogue.SeatPass) (bool, error) { return true, nil }, + act: func(context.Context, catalogue.SeatPass) error { return nil }} + } + return seatPassing{ + say: w, + pass: func(ctx context.Context, p catalogue.SeatPass) (bool, error) { + return inv.PassSeat(ctx, p.Seat.Name, p.Seat.Scope, p.From, p.To) + }, + act: func(ctx context.Context, p catalogue.SeatPass) error { + return onTheBus(func(conn *nats.Conn) error { + _, err := link.RecordHandAct(ctx, conn, seatPassHandAct(p)) + return err + }) + }, + } +} + +// theMeshPassesSeatsOn is passSeatsOn as the composition runs it. **Only when every machine's assignments were +// read** (assignedErr nil): a machine whose read failed would look as if its modules were assigned nowhere, and +// its holder would lose the seat. Said to where the composition says its other lines (sayLeftOut). +func theMeshPassesSeatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest, + assigned map[string][]string, assignedErr error, holdings []catalogue.Held) []catalogue.Held { + var w io.Writer = os.Stderr + if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok { + w = lw + } + if assignedErr != nil { + fmt.Fprintf(w, "no seat is passed on in this composition: a machine's assignments could not be read, so a "+ + "holder on record might look assigned nowhere: %v\n", assignedErr) + return holdings + } + return passSeatsOn(ctx, seatPassingOn(inv, w), holdings, assignmentsOf(shelf, assigned)) +} diff --git a/cmd/mesh-controller/seat_pass_test.go b/cmd/mesh-controller/seat_pass_test.go new file mode 100644 index 00000000..5181964e --- /dev/null +++ b/cmd/mesh-controller/seat_pass_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func aBrokerClaiming(module string) catalogue.Manifest { + return catalogue.Manifest{Module: module, + Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}}, + Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}} +} + +func withBusSeat(t *testing.T) { + t.Helper() + was := catalogue.Seats() + t.Cleanup(func() { catalogue.UseSeats(was) }) + catalogue.UseSeats([]catalogue.Seat{{Name: "mesh-broker", Scope: catalogue.ScopeMesh, Delivers: "mesh-bus", Decision: "test"}}) +} + +// 1317/AC1: the composition records the claimant as the holder and writes one hand act by the controller naming +// the seat, the former holder and the new one; the holdings it answers name the new holder. +func TestACompositionPassesTheSeatAndWritesOneHandActByTheController(t *testing.T) { + withBusSeat(t) + var passed []catalogue.SeatPass + var acts []link.HandAct + var said bytes.Buffer + sp := seatPassing{say: &said, + pass: func(_ context.Context, p catalogue.SeatPass) (bool, error) { + passed = append(passed, p) + return true, nil + }, + act: func(_ context.Context, p catalogue.SeatPass) error { + acts = append(acts, seatPassHandAct(p)) + return nil + }, + } + holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}} + got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}}) + if len(got) != 1 || got[0].Module != "gate" || got[0].Node != "novox" { + t.Fatalf("the holdings after the composition do not name the gate: %+v", got) + } + if len(passed) != 1 || len(acts) != 1 { + t.Fatalf("passes=%d acts=%d, want one each", len(passed), len(acts)) + } + a := acts[0] + text := strings.Join(append([]string{a.By, a.Verb, a.Why}, a.Args...), " ") + for _, want := range []string{"the controller", "mesh-broker", "novox/old-broker", "novox/gate"} { + if !strings.Contains(text, want) { + t.Fatalf("the hand act does not name %q: %+v", want, a) + } + } +} + +// 1317/AC2: two claimants, nothing recorded, the refusal said. +func TestACompositionWithTwoClaimantsRecordsNoneAndSaysSo(t *testing.T) { + withBusSeat(t) + sp := seatPassing{say: &bytes.Buffer{}, + pass: func(context.Context, catalogue.SeatPass) (bool, error) { + t.Fatal("a seat with two claimants was passed") + return false, nil + }, + act: func(context.Context, catalogue.SeatPass) error { t.Fatal("an act was written"); return nil }, + } + var said bytes.Buffer + sp.say = &said + holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}} + got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{ + {Node: "novox", Manifest: aBrokerClaiming("gate")}, {Node: "ace", Manifest: aBrokerClaiming("other")}}) + if len(got) != 1 || got[0].Module != "old-broker" { + t.Fatalf("the record changed: %+v", got) + } + if !strings.Contains(said.String(), "gate") || !strings.Contains(said.String(), "other") || + !strings.Contains(said.String(), "seat mesh-broker --to") { + t.Fatalf("the refusal is not said with both and the remedy: %q", said.String()) + } +} + +// 1317/AC3 and AC4: nothing changes while the holder is assigned, nor with no claimant. +func TestACompositionChangesNothingWhileTheHolderIsAssignedOrNobodyClaims(t *testing.T) { + withBusSeat(t) + sp := seatPassing{say: &bytes.Buffer{}, + pass: func(context.Context, catalogue.SeatPass) (bool, error) { t.Fatal("passed"); return false, nil }, + act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil }, + } + holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}} + got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{ + {Node: "novox", Manifest: aBrokerClaiming("old-broker")}, {Node: "novox", Manifest: aBrokerClaiming("gate")}}) + if len(got) != 1 || got[0].Module != "old-broker" { + t.Fatalf("holder assigned: record changed: %+v", got) + } + got = passSeatsOn(t.Context(), sp, holdings, nil) + if len(got) != 1 || got[0].Module != "old-broker" { + t.Fatalf("no claimant: record changed: %+v", got) + } +} + +// A composition that lost the compare-and-set writes no act, and a failed write leaves the record. +func TestALostPassWritesNoActAndAFailedOneLeavesTheRecord(t *testing.T) { + withBusSeat(t) + holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}} + as := []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}} + lost := seatPassing{say: &bytes.Buffer{}, + pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, nil }, + act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act without a pass"); return nil }, + } + passSeatsOn(t.Context(), lost, holdings, as) + var said bytes.Buffer + sp := seatPassing{say: &said, + pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, context.DeadlineExceeded }, + act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil }, + } + got := passSeatsOn(t.Context(), sp, holdings, as) + if got[0].Module != "old-broker" || !strings.Contains(said.String(), "could not be passed") { + t.Fatalf("got %+v said %q", got, said.String()) + } +} + +// A pass is the mesh working as ADR 0449 §4 decided, never a repair: two of them raise no healer-wanted and +// count as no repair. +func TestTwoSeatPassesAreNoRepairAndRaiseNoHealerWanted(t *testing.T) { + withBusSeat(t) + now := time.Date(2026, 10, 19, 20, 0, 0, 0, time.UTC) + mk := func(at time.Time) link.HandAct { + a := seatPassHandAct(catalogue.SeatPass{Seat: catalogue.Seat{Name: "mesh-broker"}, + From: catalogue.Held{Node: "novox", Module: "old"}, To: catalogue.Held{Node: "novox", Module: "gate"}}) + a.At = at + return a + } + acts := []link.HandAct{mk(now.Add(-26 * time.Hour)), mk(now.Add(-time.Hour))} + if r := repairs(acts); len(r) != 0 { + t.Fatalf("a seat pass counts as a repair: %+v", r) + } + f := calm(now) + f.handActs = acts + if got := watchHandActs(f); len(got) != 0 { + t.Fatalf("two passes raised %+v", got) + } +} + +// A failed read of one machine's assignments passes no seat: its holder would look assigned nowhere. +func TestNoSeatIsPassedWhenAMachinesAssignmentsCouldNotBeRead(t *testing.T) { + withBusSeat(t) + var said bytes.Buffer + ctx := leftOutSaidBy(t.Context(), &said) + holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "ace", Module: "old-broker"}} + shelf := map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")} + got := theMeshPassesSeatsOn(ctx, nil, shelf, map[string][]string{"novox": {"gate"}}, + errors.New("ace: the bus did not answer"), holdings) + if len(got) != 1 || got[0].Module != "old-broker" { + t.Fatalf("the record moved: %+v", got) + } + if !strings.Contains(said.String(), "no seat is passed") || !strings.Contains(said.String(), "ace") { + t.Errorf("not said: %q", said.String()) + } +} + +// A holder that is assigned but missing from the shelf is still assigned. +func TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone(t *testing.T) { + withBusSeat(t) + as := assignmentsOf(map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")}, + map[string][]string{"novox": {"gate", "old-broker"}}) + passes, _ := catalogue.SeatsToPassOn( + []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as) + if len(passes) != 0 { + t.Fatalf("a holder missing from the shelf lost its seat: %+v", passes) + } +} + +// The merge gate's throwaway mesh passes the seat in memory and writes nothing: no record, no hand act. +func TestAThrowawayMeshPassesTheSeatInMemoryAndWritesNothing(t *testing.T) { + withBusSeat(t) + inv := &inventory.Inventory{} + inv.MarkThrowaway() + var said bytes.Buffer + holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}} + got := passSeatsOn(t.Context(), seatPassingOn(inv, &said), holdings, + []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}}) + if len(got) != 1 || got[0].Module != "gate" { + t.Fatalf("the gate's view does not hold the pass: %+v", got) + } + if said.Len() != 0 { + t.Errorf("the throwaway composition said %q (a write was tried)", said.String()) + } +} diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index c311efb4..33a49603 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -234,6 +234,16 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.batches, func(b batchFacts) time.Time { return b.created }) }}, + {Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan", + Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)", + Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " + + "words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " + + "from the plan's last change, and ended when the plan is no longer failed", + Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3, + needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed, + newest: func(f *signalFacts) time.Time { + return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since }) + }}, {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + @@ -404,6 +414,34 @@ func watchPlans(f *signalFacts) []conditions.Observation { return out } +// kindPlanFailed is S23's kind: plan..plan-failed. +const kindPlanFailed = "plan-failed" + +// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no +// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to +// the operator). A plan failed longer ago than this is left to `plans`. +const planFailedWithin = 7 * 24 * time.Hour + +// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when +// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2). +// The condition ends when the plan is no longer failed, because it is no longer among the facts. +func watchPlansFailed(f *signalFacts) []conditions.Observation { + var out []conditions.Observation + for _, p := range f.failed { + if f.now.Sub(p.since) > planFailedWithin { + continue + } + headline, explanation, needs, actions, resolved := planFailedWords(p) + out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed, + Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit), + p.failureSaid()), + Said: p.failureSaid() + p.refusalSaid(), + Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved}) + } + return out +} + // kindWalkWaiting is S16's kind: plan..waiting. const kindWalkWaiting = "waiting" diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 3fc0561c..488117da 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -160,6 +160,17 @@ var suppressions = map[string]suppression{ created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}} }, }, + // A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449). + "S23": { + inside: func(f *signalFacts) { + f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, + since: f.now.Add(-planFailedWithin - time.Second)}} + }, + past: func(f *signalFacts) { + f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, + since: f.now.Add(-time.Second)}} + }, + }, // A batch queued behind an open walk past that walk's bound, named. "S19": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 3ec0ff1c..88605563 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -55,6 +55,9 @@ type signalFacts struct { waits []waitFacts // batches are the batches not yet cut (S18, S19, novox/hq ADR 0276). batches []batchFacts + // failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449). + failed []failedPlanFacts + failedErr error // refusedSends are the sends a machine refused within the hour for the generation they were composed // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send @@ -353,6 +356,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { if f.plansErr == nil { f.batches, f.plansErr = gatherBatches(ctx, inv, now) } + f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now) f.loop, f.loopErr = w.gatherLoop() f.asks, f.asksErr = w.gatherAsks(ctx, inv) if open, err := w.keeper.Open(ctx); err != nil { diff --git a/internal/catalogue/seat_pass.go b/internal/catalogue/seat_pass.go new file mode 100644 index 00000000..336c8f67 --- /dev/null +++ b/internal/catalogue/seat_pass.go @@ -0,0 +1,85 @@ +package catalogue + +import ( + "fmt" + "sort" +) + +// An assignment as the passing of a seat reads it: a module on a machine, with its manifest. +type Assignment struct { + Node string + Manifest Manifest +} + +// SeatPass is one seat whose holder on record is assigned nowhere, and the only assignment that can take it. +type SeatPass struct { + Seat Seat + From Held + To Held +} + +// SeatsToPassOn reads the mesh for seats held once whose holder on record has left (novox/hq ADR 0449 §4). +// +// **A holder on record that is assigned on no machine holds nothing, and a seat that answers nothing is +// the day a delivery removed its module.** Of such a seat, exactly one assignment in the mesh able to hold +// it (it claims the seat at its scope and CanHold accepts it) takes it over: that is a pass, and the caller +// records it and writes the hand act. With no such assignment the record stays; with two or more nothing is +// recorded and the answer says so, naming both and the `seat --to` that settles it. A holder still assigned +// anywhere changes nothing: the other claimants stand beside it, eligible and silent (ADR 0131). Only mesh +// seats held once are read: a replicated seat's holders are added by an act (ADR 0223), a kinded bench is held +// by its claims, and a seat with several holders on record is a store that disagrees with the mesh (refused +// by the resolver). +func SeatsToPassOn(holdings []Held, assignments []Assignment) ([]SeatPass, []string) { + assigned := map[string]bool{} + for _, a := range assignments { + assigned[a.Manifest.Module] = true + } + bySeat := map[string][]Held{} + var order []string + for _, h := range holdings { + s, ok := SeatNamed(h.Claim) + if !ok || s.Replicated || s.Scope != ScopeMesh || h.Scope != ScopeMesh || KindedBenches[s.Name] { + continue + } + if _, seen := bySeat[s.Name]; !seen { + order = append(order, s.Name) + } + bySeat[s.Name] = append(bySeat[s.Name], h) + } + sort.Strings(order) + + var passes []SeatPass + var said []string + for _, name := range order { + recs := bySeat[name] + if len(recs) != 1 || assigned[recs[0].Module] { + continue + } + seat, _ := SeatNamed(name) + var claimants []Held + for _, a := range assignments { + if CanHold(a.Manifest, seat) != nil { + continue + } + claimants = append(claimants, Held{Claim: seat.Name, Scope: seat.Scope, Node: a.Node, Module: a.Manifest.Module}) + } + sort.Slice(claimants, func(i, j int) bool { + if claimants[i].Node != claimants[j].Node { + return claimants[i].Node < claimants[j].Node + } + return claimants[i].Module < claimants[j].Module + }) + switch len(claimants) { + case 0: + case 1: + passes = append(passes, SeatPass{Seat: seat, From: recs[0], To: claimants[0]}) + default: + said = append(said, fmt.Sprintf( + "%s on %s is on record as the holder of %q and is assigned nowhere, and %s on %s and %s on %s both claim it "+ + "— nothing is recorded; `seat %s --to /` records the one that holds it", + recs[0].Module, recs[0].Node, seat.Name, claimants[0].Module, claimants[0].Node, + claimants[1].Module, claimants[1].Node, seat.Name)) + } + } + return passes, said +} diff --git a/internal/catalogue/seat_pass_test.go b/internal/catalogue/seat_pass_test.go new file mode 100644 index 00000000..d1bc8e06 --- /dev/null +++ b/internal/catalogue/seat_pass_test.go @@ -0,0 +1,81 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// **A seat held once whose holder on record is assigned nowhere passes to its only claimant** (novox/hq ADR +// 0449 §4, issue 1317). + +func onRecord(node, module string) []Held { + return []Held{{Claim: "mesh-broker", Scope: ScopeMesh, Node: node, Module: module}} +} + +// 1317/AC1 +func TestASeatWhoseHolderIsAssignedNowhereIsPassedToItsOnlyClaimant(t *testing.T) { + busSeatDelivering(t, "mesh-bus") + passes, said := SeatsToPassOn(onRecord("novox", "old-broker"), + []Assignment{{Node: "novox", Manifest: newBroker()}}) + if len(said) != 0 || len(passes) != 1 { + t.Fatalf("the only claimant was not given the seat: passes=%v said=%v", passes, said) + } + p := passes[0] + if p.Seat.Name != "mesh-broker" || p.From.Module != "old-broker" || p.From.Node != "novox" || + p.To.Module != "new-broker" || p.To.Node != "novox" { + t.Fatalf("the pass does not name the seat, the former holder and the new one: %+v", p) + } +} + +// 1317/AC2 +func TestWithTwoClaimantsAndTheHolderGoneNothingIsPassedAndTheRefusalNamesBoth(t *testing.T) { + busSeatDelivering(t, "mesh-bus") + passes, said := SeatsToPassOn(onRecord("novox", "gone"), []Assignment{ + {Node: "novox", Manifest: newBroker()}, {Node: "ace", Manifest: oldBroker()}}) + if len(passes) != 0 || len(said) != 1 { + t.Fatalf("two claimants: passes=%v said=%v", passes, said) + } + for _, want := range []string{"old-broker", "new-broker", "seat mesh-broker --to"} { + if !strings.Contains(said[0], want) { + t.Fatalf("the refusal does not say %q: %s", want, said[0]) + } + } +} + +// 1317/AC3 +func TestAHolderStillAssignedSomewhereChangesNothing(t *testing.T) { + busSeatDelivering(t, "mesh-bus") + // Assigned on another machine than the record names: still assigned somewhere. + passes, said := SeatsToPassOn(onRecord("novox", "old-broker"), []Assignment{ + {Node: "ace", Manifest: oldBroker()}, {Node: "novox", Manifest: newBroker()}}) + if len(passes) != 0 || len(said) != 0 { + t.Fatalf("a holder still assigned lost its seat: passes=%v said=%v", passes, said) + } +} + +// 1317/AC4 +func TestWithNoClaimantTheRecordStays(t *testing.T) { + busSeatDelivering(t, "mesh-bus") + passes, said := SeatsToPassOn(onRecord("novox", "old-broker"), []Assignment{ + {Node: "novox", Manifest: Manifest{Module: "other"}}}) + if len(passes) != 0 || len(said) != 0 { + t.Fatalf("with no claimant: passes=%v said=%v", passes, said) + } + // A claimant that cannot hold the seat (it does not provide what the seat delivers) is no claimant. + bad := Manifest{Module: "no-bus", Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}} + passes, said = SeatsToPassOn(onRecord("novox", "old-broker"), []Assignment{{Node: "novox", Manifest: bad}}) + if len(passes) != 0 || len(said) != 0 { + t.Fatalf("a module that cannot hold the seat was passed it: passes=%v said=%v", passes, said) + } +} + +// A replicated seat is never passed: its holders are added by an act (ADR 0223). +func TestAReplicatedSeatIsNeverPassed(t *testing.T) { + record := []Held{{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "novox", Module: "gone"}} + resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}, + Claims: []Claim{{Name: "mesh-dns-resolver", Scope: ScopeMesh}}} + passes, _ := SeatsToPassOn(record, []Assignment{{Node: "novox", Manifest: resolver}}) + if len(passes) != 0 { + t.Fatalf("a replicated seat was passed: %v", passes) + } +} diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index df09bfa0..0d297639 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -85,6 +85,7 @@ func (i *Inventory) UseChain(c *chain.Chain) { // beside the real one, as the merge gate raises the facts snapshot's mesh in a throwaway store (issue 1121). Nothing of it // reaches the bus. func (i *Inventory) UseThrowawayChain() { + i.MarkThrowaway() i.UseChain(chain.InMemory(nil)) i.markLaidOut() } @@ -129,3 +130,9 @@ func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) { } return c, nil } + +// MarkThrowaway says this inventory is a mesh raised for a moment: a composition on it writes nothing outside it. +func (i *Inventory) MarkThrowaway() { i.throwaway = true } + +// Throwaway says whether this inventory is a mesh raised for a moment (UseThrowawayChain). +func (i *Inventory) Throwaway() bool { return i != nil && i.throwaway } diff --git a/internal/inventory/holdings_test.go b/internal/inventory/holdings_test.go index b298a59d..e129c141 100644 --- a/internal/inventory/holdings_test.go +++ b/internal/inventory/holdings_test.go @@ -153,3 +153,24 @@ func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) { t.Fatalf("a handover left other holders on record: %+v", held) } } + +// A pass is a compare-and-set: it moves the record only while the holder it was read with is still on it. +func TestPassingASeatMovesTheRecordOnceAndOnlyFromTheHolderRead(t *testing.T) { + inv, ctx := twoBrokersOnTwoNodes(t) + if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, "anchor", "old-broker"); err != nil { + t.Fatal(err) + } + from := catalogue.Held{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "old-broker"} + to := catalogue.Held{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "laptop", Module: "new-broker"} + passed, err := inv.PassSeat(ctx, "mesh-broker", catalogue.ScopeMesh, from, to) + if err != nil || !passed { + t.Fatalf("the seat was not passed: %v %v", passed, err) + } + held, _ := inv.Holdings(ctx) + if len(held) != 1 || held[0].Node != "laptop" || held[0].Module != "new-broker" { + t.Fatalf("the record does not name the new holder alone: %+v", held) + } + if passed, err = inv.PassSeat(ctx, "mesh-broker", catalogue.ScopeMesh, from, to); err != nil || passed { + t.Fatalf("a second pass from the same read moved the record again: %v %v", passed, err) + } +} diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 26f57483..08c95fbe 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -39,6 +39,9 @@ type Inventory struct { state *chainState // recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store. recorded func(context.Context, Duration) error + // throwaway is set on an inventory raised for a moment beside the real mesh (the merge gate's): nothing it + // composes is to be written to the store or the bus. + throwaway bool } // Open connects to the inventory store. diff --git a/internal/inventory/seats.go b/internal/inventory/seats.go index 33dc4706..972e3e2c 100644 --- a/internal/inventory/seats.go +++ b/internal/inventory/seats.go @@ -284,6 +284,48 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module return tx.Commit(ctx) } +// PassSeat records `to` as the holder of a seat held once in place of `from`, only while `from` is still the +// one on record: a compare-and-set in one transaction, so two compositions reading the same record cannot both +// hand the seat over (novox/hq ADR 0449 §4). It answers false, with no error, when the record no longer names +// `from` (another composition passed it first). The new holder must be an assignment. +func (i *Inventory) PassSeat(ctx context.Context, seat, scope string, from, to catalogue.Held) (bool, error) { + fromNode, err := i.NodeByName(ctx, from.Node) + if err != nil { + return false, err + } + toNode, err := i.NodeByName(ctx, to.Node) + if err != nil { + return false, err + } + if err := i.mustBeAssigned(ctx, seat, to.Node, to.Module); err != nil { + return false, err + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + gone, err := tx.Exec(ctx, + `delete from seat_holding where seat = $1 and node = $2 and module = $3`, + seat, fromNode.ID, from.Module) + if err != nil { + return false, fmt.Errorf("passing %s from %s on %s: %w", seat, from.Module, from.Node, err) + } + if gone.RowsAffected() == 0 { + return false, nil + } + if _, err := tx.Exec(ctx, + `delete from seat_holding where seat = $1`, seat); err != nil { + return false, fmt.Errorf("passing %s: %w", seat, err) + } + if _, err := tx.Exec(ctx, + `insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)`, + seat, scope, toNode.ID, to.Module); err != nil { + return false, fmt.Errorf("recording %s on %s as the holder of %s: %w", to.Module, to.Node, seat, err) + } + return true, tx.Commit(ctx) +} + // AddSeatHolder records one more assignment holding a replicated seat, beside those already on // record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge — // the seat's definition is compiled, and the store holds no column for it. Recording a holder