Merge pull request 'Ask a failed delivery's Retry on the phone, tell a refused answer, and pass a seat to its only claimant (ADR 0449, ADR 0451)' (#402) from feat/no-terminal into main
This commit was merged in pull request #402.
This commit is contained in:
@@ -208,8 +208,11 @@ type asker struct {
|
||||
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
||||
// asked, and why. Nil raises nothing (a test that does not look).
|
||||
raise func(ctx context.Context, obs []conditions.Observation) error
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// tell tells the operator one plain notice that needs no answer (operator-channel.tell, novox/hq ADR 0380):
|
||||
// the refusal of an answer to a plan's ask (ADR 0449 §3). Nil tells nothing (a test that does not look).
|
||||
tell func(ctx context.Context, headline string, lines []string) error
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
saidNoRouter bool
|
||||
saidNoGrant bool
|
||||
@@ -406,7 +409,9 @@ func (a *asker) reconcile(ctx context.Context) error {
|
||||
for _, r := range all {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||
answered[k] = r
|
||||
if prior, has := answered[k]; !has || r.Ended.After(prior.Ended) {
|
||||
answered[k] = r
|
||||
}
|
||||
}
|
||||
if r.State == string(asks.OutcomeRefused) {
|
||||
if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) {
|
||||
@@ -497,7 +502,9 @@ func (a *asker) reconcile(ctx context.Context) error {
|
||||
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
||||
}
|
||||
}
|
||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
||||
// **A condition raised after the answered ask ended is a new one** (novox/hq ADR 0449 §2): a retried
|
||||
// plan that fails again is asked again, not held back for the hour its earlier answer holds.
|
||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) && !c.Raised.After(r.Ended) {
|
||||
if _, held := byCondition[key]; !held {
|
||||
continue
|
||||
}
|
||||
@@ -636,6 +643,8 @@ func doesWords(act conditions.Action) string {
|
||||
return "the delivery starts"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||
return "the delivery is stopped"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "":
|
||||
return "the failed builds are asked again"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
@@ -884,6 +893,7 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
ended := a.now()
|
||||
if acted != nil {
|
||||
outcome = "failed: " + acted.Error()
|
||||
a.tellRefused(ctx, *r, act, acted)
|
||||
}
|
||||
r.Ended, r.Acted = ended, outcome
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
@@ -918,6 +928,55 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// tellRefused tells the operator what became of the answer they gave to a plan's ask, when it was not done (novox/hq
|
||||
// ADR 0449 §3): the controller's `plans` verb refuses a Retry, Start or Stop when the plan moved on between the ask and
|
||||
// the answer, and the refusal was only recorded, so the operator who pressed the answer heard nothing. **A refusal is
|
||||
// told as one, and nothing else is**: a verb that said no changed nothing, but a call that got no answer (a timeout, a
|
||||
// refused grant) may have run, and is told as not known. One plain notice with the ask's headline, beside the hand act
|
||||
// the caller still records. An answer to any other ask is not told here. A notice that cannot be told is said in the
|
||||
// log and changes nothing.
|
||||
func (a *asker) tellRefused(ctx context.Context, r asked, act conditions.Action, failure error) {
|
||||
if a.tell == nil || act.Verb != broker.ControllerSeat+".plans" {
|
||||
return
|
||||
}
|
||||
var lines []string
|
||||
var refused verbRefused
|
||||
if errors.As(failure, &refused) {
|
||||
lines = []string{
|
||||
fmt.Sprintf("Your answer, %s, could not be done: %s.", act.Label, strings.TrimRight(refusalWords(refused.words, act), ".")),
|
||||
"Nothing was changed.",
|
||||
}
|
||||
} else {
|
||||
lines = []string{
|
||||
fmt.Sprintf("Your answer, %s, was given, and it is not known whether it was done.", act.Label),
|
||||
"Look at the delivery before you answer again.",
|
||||
}
|
||||
}
|
||||
if err := a.tell(ctx, r.Ask.Headline, lines); err != nil {
|
||||
a.logf("the operator could NOT be told that %s was refused (ask %s): %v", act.Label, r.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusalWords is a verb's refusal as the operator reads it: the plans named as "the plan" (plainRefusal), and, where
|
||||
// the words still carry an identifier the operator's channel withholds, one sentence of the answer's own.
|
||||
func refusalWords(words string, act conditions.Action) string {
|
||||
return plainRefusalOr(words, refusalFallback(act))
|
||||
}
|
||||
|
||||
// refusalFallback is what an answer's refusal says when its own words cannot be said: one sentence for each answer
|
||||
// to a plan's ask, never another answer's.
|
||||
func refusalFallback(act conditions.Action) string {
|
||||
switch {
|
||||
case act.Arguments["retry"] != "":
|
||||
return "a retry would not run now; the details say why"
|
||||
case act.Arguments["go"] != "":
|
||||
return "the delivery could not be started now; the details say why"
|
||||
case act.Arguments["stop"] != "":
|
||||
return "the delivery could not be stopped now; the details say why"
|
||||
}
|
||||
return "the controller refused it; the details say why"
|
||||
}
|
||||
|
||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||
// never a repair (handActVerbs).
|
||||
const handActWarrant = "warrant"
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
|
||||
// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
|
||||
func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
ended := before.Add(time.Minute)
|
||||
failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Delivery: &inventory.PlanDelivery{},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}},
|
||||
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
// The controller's seat answers `plans` as it does: by running the verb.
|
||||
sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
|
||||
var args map[string]string
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
answer := link.Answer{Result: json.RawMessage(`"ok"`)}
|
||||
if len(args) != 1 || args["retry"] == "" {
|
||||
answer = link.Answer{Error: "the retry was asked with more than the plan"}
|
||||
} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
|
||||
answer = link.Answer{Error: err.Error()}
|
||||
}
|
||||
body, _ := json.Marshal(answer)
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
// What S23 raises for it, read from the store as the watchdog reads it.
|
||||
raised := func(context.Context) ([]conditions.Condition, error) {
|
||||
facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
|
||||
out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
now := time.Now()
|
||||
store := memAskedStore{}
|
||||
var sent []published
|
||||
var acts []link.HandAct
|
||||
a := &asker{open: raised, store: store,
|
||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
||||
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
||||
sent = append(sent, published{subject, id, body})
|
||||
return nil
|
||||
},
|
||||
call: callAction(conn),
|
||||
record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
|
||||
now: func() time.Time { return now },
|
||||
logf: t.Logf}
|
||||
cs, _ := raised(ctx)
|
||||
if len(cs) != 1 || len(cs[0].Actions) != 1 {
|
||||
t.Fatalf("the failed plan raised %+v", cs)
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w asks.Warrant
|
||||
for _, r := range store {
|
||||
for _, o := range r.Ask.Options {
|
||||
if o.Label == "Retry" {
|
||||
w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
|
||||
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
||||
AskDigest: r.Ask.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.Ask == "" {
|
||||
t.Fatalf("no ask offers Retry: %+v", store)
|
||||
}
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := p.Modules["a"]
|
||||
if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" {
|
||||
t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("the build was asked %d times: %v", len(*asked), *asked)
|
||||
}
|
||||
if got := store[w.Ask]; got.Acted != "done" {
|
||||
t.Fatalf("the ask ended %q", got.Acted)
|
||||
}
|
||||
if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
|
||||
t.Fatalf("the warrant was recorded as %+v", acts)
|
||||
}
|
||||
if cs, _ = raised(ctx); len(cs) != 0 {
|
||||
t.Fatalf("the plan-failed condition did not end: %+v", cs)
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,15 @@ type askerRig struct {
|
||||
silenced []string
|
||||
acts []link.HandAct
|
||||
now time.Time
|
||||
// refuses, when set, is what every verb the asker calls answers; told is every notice the operator was told.
|
||||
refuses error
|
||||
told []told
|
||||
}
|
||||
|
||||
// told is one plain notice the asker told the operator.
|
||||
type told struct {
|
||||
headline string
|
||||
lines []string
|
||||
}
|
||||
|
||||
func newAskerRig(t *testing.T) *askerRig {
|
||||
@@ -102,6 +111,10 @@ func newAskerRig(t *testing.T) *askerRig {
|
||||
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
||||
raw, _ := json.Marshal(args)
|
||||
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
||||
return r.refuses
|
||||
},
|
||||
tell: func(_ context.Context, headline string, lines []string) error {
|
||||
r.told = append(r.told, told{headline, lines})
|
||||
return nil
|
||||
},
|
||||
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
||||
@@ -1013,3 +1026,215 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
|
||||
func failedPlanCondition(plan string) conditions.Condition {
|
||||
p := aFailedPlan(plan, nil)
|
||||
f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
o := watchPlansFailed(f)[0]
|
||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
}
|
||||
|
||||
// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
|
||||
func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
r.open = []conditions.Condition{c}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
||||
t.Fatalf("asked %+v", sent)
|
||||
}
|
||||
o := sent[0].Options[0]
|
||||
if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
|
||||
t.Errorf("the option is %+v", o)
|
||||
}
|
||||
if err := sent[0].Check(r.now); err != nil {
|
||||
t.Errorf("the ask is refused as made: %v", err)
|
||||
}
|
||||
if strings.Contains(sent[0].Explanation, "mesh MCP server") {
|
||||
t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
|
||||
// the same warrant heard again does nothing.
|
||||
func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition(plan)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, c.Key, "Retry")
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w)
|
||||
if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
|
||||
t.Fatalf("called %v, want exactly %s", r.called, want)
|
||||
}
|
||||
if len(r.acts) != 1 {
|
||||
t.Fatalf("hand-acts %+v", r.acts)
|
||||
}
|
||||
act := r.acts[0]
|
||||
if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
|
||||
act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
|
||||
t.Errorf("the hand-act is %+v", act)
|
||||
}
|
||||
if got := r.store[w.Ask]; got.Acted != "done" {
|
||||
t.Errorf("kept %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is
|
||||
// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before.
|
||||
func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
c conditions.Condition
|
||||
label string
|
||||
}{
|
||||
{"retry", failedPlanCondition(plan), "Retry"},
|
||||
{"stop", waiting, "Stop"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"}
|
||||
r.open = []conditions.Condition{tc.c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, tc.c.Key, tc.label)
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w) // heard again: told once
|
||||
if len(r.told) != 1 {
|
||||
t.Fatalf("told %+v", r.told)
|
||||
}
|
||||
got := r.told[0]
|
||||
if got.headline != tc.c.Headline {
|
||||
t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline)
|
||||
}
|
||||
said := strings.Join(got.lines, " ")
|
||||
if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) {
|
||||
t.Errorf("the lines do not carry the answer and the refusal's words: %q", said)
|
||||
}
|
||||
if why, ok := conditions.Plain(said); !ok {
|
||||
t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said)
|
||||
}
|
||||
if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") {
|
||||
t.Errorf("the failure is not recorded as before: %+v", r.acts)
|
||||
}
|
||||
if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") {
|
||||
t.Errorf("kept %+v", kept)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here.
|
||||
func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
if len(r.told) != 0 {
|
||||
t.Fatalf("a retry that was done told %+v", r.told)
|
||||
}
|
||||
|
||||
r = newAskerRig(t)
|
||||
r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"}
|
||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
||||
r.open = []conditions.Condition{module}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, module.Key, "Restart"))
|
||||
if len(r.told) != 0 || len(r.acts) != 1 {
|
||||
t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts)
|
||||
}
|
||||
}
|
||||
|
||||
// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was
|
||||
// changed"; and the sentence a refusal falls back to fits the answer given.
|
||||
func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
r := newAskerRig(t)
|
||||
r.refuses = errors.New("mesh-controller did not answer plans within 30s")
|
||||
c := failedPlanCondition(plan)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
if len(r.told) != 1 {
|
||||
t.Fatalf("told %+v", r.told)
|
||||
}
|
||||
said := strings.Join(r.told[0].lines, " ")
|
||||
if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") {
|
||||
t.Errorf("a call that may have run is told as %q", said)
|
||||
}
|
||||
if why, ok := conditions.Plain(said); !ok {
|
||||
t.Errorf("not plain: %s: %q", why, said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
for _, tc := range []struct {
|
||||
act conditions.Action
|
||||
want string
|
||||
}{
|
||||
{retryAction(plan), "a retry would not run now"},
|
||||
{conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"},
|
||||
{conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"},
|
||||
} {
|
||||
// Words the operator's channel withholds: a hash and a dotted name.
|
||||
got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act)
|
||||
if !strings.Contains(got, tc.want) {
|
||||
t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want)
|
||||
}
|
||||
if _, ok := conditions.Plain(got); !ok {
|
||||
t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the
|
||||
// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not.
|
||||
func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
c.Raised = r.now.Add(-time.Minute)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
// Still open right after the answer: not asked again at once.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != 1 {
|
||||
t.Fatalf("asked again while the same condition stayed open: %d", n)
|
||||
}
|
||||
// The retry ran, the plan left failed and the condition ended.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
r.open = nil
|
||||
_ = r.a.reconcile(context.Background())
|
||||
// It fails again, ten minutes after the answer: a condition raised after the answered ask ended.
|
||||
r.now = r.now.Add(8 * time.Minute)
|
||||
again := failedPlanCondition("plan-1792433168676870614")
|
||||
again.Raised = r.now
|
||||
r.open = []conditions.Condition{again}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(r.asksSent(t)); n != 2 {
|
||||
t.Fatalf("the second failure was asked %d times in all, want 2", n)
|
||||
}
|
||||
if len(r.called) != 1 {
|
||||
t.Fatalf("something retried without an answer: %v", r.called)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +135,12 @@ func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// verbRefused is a verb's own refusal of an act: the seat answered, and said no. Any other error of callAction (a bus
|
||||
// that did not answer, a grant that is refused) is not one: the verb may have run.
|
||||
type verbRefused struct{ verb, words string }
|
||||
|
||||
func (e verbRefused) Error() string { return e.verb + " refused: " + e.words }
|
||||
|
||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
@@ -168,7 +174,23 @@ func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action,
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s refused: %s", a.Verb, answer.Error)
|
||||
return verbRefused{verb: a.Verb, words: answer.Error}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// tellThrough tells the operator one plain notice through the operator channel's `tell`, over the controller's own
|
||||
// connection and grant (novox/hq ADR 0380); a holder's refusal is an error naming it.
|
||||
func tellThrough(conn *nats.Conn) func(ctx context.Context, headline string, lines []string) error {
|
||||
return func(ctx context.Context, headline string, lines []string) error {
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell",
|
||||
map[string]any{"headline": headline, "lines": lines}, tellWithin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("operator-channel.tell refused: %s", answer.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -576,6 +598,7 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
},
|
||||
tell: tellThrough(conn),
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
|
||||
@@ -72,6 +72,10 @@ var handActVerbs = []handActVerb{
|
||||
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
|
||||
// channel that proved who answered, performed by the controller as itself.
|
||||
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
|
||||
// A seat passed by the controller to its only claimant when its holder on record left (novox/hq ADR 0449
|
||||
// §4): the mesh working as decided. A person's own `seat --to` is not recorded under this cause.
|
||||
{Verb: "seat", Decision: "the controller passes a seat held once to its only claimant when its recorded " +
|
||||
"holder is assigned nowhere (ADR 0449 §4)", DecidedFor: []string{"seat-passed"}},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||
|
||||
@@ -429,6 +429,11 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"out) has not let it start yet.",
|
||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||
}),
|
||||
kindPlanFailed: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery failed",
|
||||
Explanation: "A delivery did not finish. Nothing is tried again until you say so.",
|
||||
Resolved: "Resolved: the delivery is no longer failed"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||
|
||||
@@ -285,12 +285,28 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
assigned []string
|
||||
}
|
||||
var others []candidate
|
||||
// **Every machine's assignments, before the holdings are read by any resolution** (novox/hq ADR 0449 §4): a
|
||||
// seat held once whose holder on record is assigned on no machine needs the whole mesh to be known, and it
|
||||
// passes to its only claimant here, in the composition that finds it.
|
||||
everyAssigned := map[string][]string{}
|
||||
var assignedErr error
|
||||
for _, n := range nodes {
|
||||
theirs, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
assignedErr = errors.Join(assignedErr, fmt.Errorf("%s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
if len(theirs) > 0 {
|
||||
everyAssigned[n.Name] = theirs
|
||||
}
|
||||
}
|
||||
holdings = theMeshPassesSeatsOn(ctx, inv, shelf, everyAssigned, assignedErr, holdings)
|
||||
for _, n := range nodes {
|
||||
if n.Name == exclude {
|
||||
continue
|
||||
}
|
||||
theirs, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil || len(theirs) == 0 {
|
||||
theirs := everyAssigned[n.Name]
|
||||
if len(theirs) == 0 {
|
||||
continue
|
||||
}
|
||||
caps, _ := inv.ProfileOf(ctx, n.Name)
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// failedModule is one module of a failed plan that failed to build, where it runs and what it said.
|
||||
type failedModule struct {
|
||||
module string
|
||||
machines []string
|
||||
why string
|
||||
}
|
||||
|
||||
// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1).
|
||||
type failedPlanFacts struct {
|
||||
id, repository, commit string
|
||||
modules []string
|
||||
failed []failedModule
|
||||
// note is where the plan stopped when no module failed to build (a rollout stopped at its first machine).
|
||||
note string
|
||||
// since is the plan's last change: when it failed.
|
||||
since time.Time
|
||||
// refusal is why `plans retry` would not run it, empty when it would.
|
||||
refusal string
|
||||
// passed is set when a newer plan of its repository and branch is done and the retry is refused for that
|
||||
// (novox/hq ADR 0451): the condition asks nothing.
|
||||
passed bool
|
||||
}
|
||||
|
||||
// machines are the machines the failed modules run on, sorted, each once.
|
||||
func (p failedPlanFacts) machines() []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, m := range p.failed {
|
||||
for _, n := range m.machines {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// failureSaid is what failed, in the mesh's own words, for the condition's evidence.
|
||||
func (p failedPlanFacts) failureSaid() string {
|
||||
var parts []string
|
||||
for _, m := range p.failed {
|
||||
line := m.module
|
||||
if len(m.machines) > 0 {
|
||||
line += " on " + strings.Join(m.machines, ", ")
|
||||
}
|
||||
if m.why != "" {
|
||||
line += ": " + oneLine(m.why)
|
||||
}
|
||||
parts = append(parts, line)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "stopped: " + oneLine(p.note)
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// refusalSaid is why a retry would not run, for the evidence.
|
||||
func (p failedPlanFacts) refusalSaid() string {
|
||||
if p.refusal == "" {
|
||||
return ""
|
||||
}
|
||||
return "; a retry would not run: " + oneLine(p.refusal)
|
||||
}
|
||||
|
||||
// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan,
|
||||
// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs).
|
||||
func retryAction(plan string) conditions.Action {
|
||||
return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"retry": plan}}
|
||||
}
|
||||
|
||||
var (
|
||||
planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`)
|
||||
walkWd = regexp.MustCompile(`\bwalks\b`)
|
||||
walkOne = regexp.MustCompile(`\bwalk\b`)
|
||||
)
|
||||
|
||||
// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a
|
||||
// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the
|
||||
// words say only that a retry would not run.
|
||||
func plainRefusal(refusal string) string {
|
||||
return plainRefusalOr(refusal, "a retry would not run now; the details say why")
|
||||
}
|
||||
|
||||
// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly.
|
||||
func plainRefusalOr(refusal, fallback string) string {
|
||||
r := planIDWord.ReplaceAllString(refusal, "the plan")
|
||||
r = walkWd.ReplaceAllString(r, "deliveries")
|
||||
r = walkOne.ReplaceAllString(r, "delivery")
|
||||
r = strings.TrimSpace(r)
|
||||
if _, ok := conditions.Plain(r); !ok {
|
||||
return fallback
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that
|
||||
// is plain, and Retry when a retry would run, else the refusal's words.
|
||||
func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) {
|
||||
name := deliveryName(p.modules, p.repository)
|
||||
headline = conditions.Capital(name) + " failed"
|
||||
what := deliveryWhat(p.modules, p.repository)
|
||||
var parts []string
|
||||
for i, m := range p.failed {
|
||||
if i == 2 {
|
||||
parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2))
|
||||
break
|
||||
}
|
||||
line := m.module
|
||||
if len(m.machines) > 0 {
|
||||
line += " for " + readableList(m.machines)
|
||||
}
|
||||
if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" {
|
||||
if _, ok := conditions.Plain(w); ok {
|
||||
line += " (" + w + ")"
|
||||
}
|
||||
}
|
||||
parts = append(parts, line)
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts))
|
||||
} else {
|
||||
explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what)
|
||||
}
|
||||
switch {
|
||||
case p.passed:
|
||||
explanation += " A newer delivery has passed it, so a retry would send older code. Nothing is needed from you."
|
||||
case p.refusal == "":
|
||||
explanation += " Nothing is tried again until you say so."
|
||||
needs = "try the failed builds again " + FromMeshMCPServer
|
||||
actions = []conditions.Action{retryAction(p.id)}
|
||||
default:
|
||||
explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "."
|
||||
needs = "look at why it failed; a retry would not run now."
|
||||
}
|
||||
resolved = conditions.Capital(name) + " is no longer failed"
|
||||
return
|
||||
}
|
||||
|
||||
// cutWords is s cut to at most n bytes at a word.
|
||||
func cutWords(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
t := s[:n]
|
||||
if i := strings.LastIndexByte(t, ' '); i > n/2 {
|
||||
t = t[:i]
|
||||
}
|
||||
return strings.TrimRight(t, " ,;:") + "…"
|
||||
}
|
||||
|
||||
// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator
|
||||
// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not
|
||||
// stopped by a person. Each carries why `plans retry` would not run it, when it would not.
|
||||
func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) {
|
||||
plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans cannot be read: %w", err)
|
||||
}
|
||||
var failed []inventory.Plan
|
||||
for _, p := range plans {
|
||||
if failedAndAsked(p) {
|
||||
failed = append(failed, p)
|
||||
}
|
||||
}
|
||||
if len(failed) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||
}
|
||||
onPath := ownPathOf(entries)
|
||||
runsOn := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
runsOn[e.Manifest.Module] = e.On
|
||||
}
|
||||
var out []failedPlanFacts
|
||||
for _, p := range failed {
|
||||
out = append(out, failedFactsOf(p, plans, onPath, runsOn))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not
|
||||
// stopped by a person.
|
||||
func failedAndAsked(p inventory.Plan) bool {
|
||||
return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == ""
|
||||
}
|
||||
|
||||
// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each
|
||||
// module runs.
|
||||
func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string,
|
||||
runsOn map[string][]string) failedPlanFacts {
|
||||
f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p),
|
||||
note: p.Note, since: p.Updated}
|
||||
for _, m := range f.modules {
|
||||
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||
machines := append([]string(nil), runsOn[m]...)
|
||||
sort.Strings(machines)
|
||||
f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why})
|
||||
}
|
||||
}
|
||||
if err := retryRefusal(p, plans, onPath); err != nil {
|
||||
f.refusal = err.Error()
|
||||
var passed passedBy
|
||||
f.passed = errors.As(err, &passed)
|
||||
}
|
||||
return f
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314).
|
||||
|
||||
func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan {
|
||||
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed,
|
||||
Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC),
|
||||
Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
||||
if mutate != nil {
|
||||
mutate(&p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts {
|
||||
return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{},
|
||||
map[string][]string{"openrazer": {"g14", "shanks"}})
|
||||
}
|
||||
|
||||
// 1314/AC1
|
||||
func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator ||
|
||||
o.Scope != conditions.ScopePlan || o.ID != p.ID {
|
||||
t.Fatalf("kind, severity, resolver or subject: %+v", o)
|
||||
}
|
||||
if len(o.Actions) != 1 {
|
||||
t.Fatalf("actions: %+v", o.Actions)
|
||||
}
|
||||
a := o.Actions[0]
|
||||
if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove ||
|
||||
len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID {
|
||||
t.Fatalf("the action is not Retry with only the plan: %+v", a)
|
||||
}
|
||||
if o.Needs == "" {
|
||||
t.Fatal("no Needs: the asker wants an action with a need")
|
||||
}
|
||||
for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} {
|
||||
if !strings.Contains(o.Headline+o.Explanation+o.Said, want) {
|
||||
t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") {
|
||||
t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("the words are not plain: %s\n%+v", why, words)
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC2
|
||||
func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) {
|
||||
for name, p := range map[string]inventory.Plan{
|
||||
"held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }),
|
||||
"stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }),
|
||||
"no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }),
|
||||
"building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }),
|
||||
} {
|
||||
if failedAndAsked(p) {
|
||||
t.Errorf("%s: a plan not asked of the operator is asked", name)
|
||||
}
|
||||
}
|
||||
if !failedAndAsked(aFailedPlan("plan-5", nil)) {
|
||||
t.Error("a failed delivery plan is not asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it.
|
||||
func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) })
|
||||
newer := aFailedPlan("plan-2", func(p *inventory.Plan) {
|
||||
p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute)
|
||||
})
|
||||
superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" })
|
||||
done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone })
|
||||
first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC)
|
||||
gate := aFailedPlan("plan-5", func(p *inventory.Plan) {
|
||||
p.Note = "openrazer stopped at its first machine"
|
||||
p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"},
|
||||
Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"},
|
||||
Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}}
|
||||
})
|
||||
for name, c := range map[string]struct {
|
||||
p inventory.Plan
|
||||
others []inventory.Plan
|
||||
says string
|
||||
}{
|
||||
"a newer plan": {older, []inventory.Plan{newer}, "supersedes it"},
|
||||
"superseded": {superseded, nil, "superseded"},
|
||||
"done": {done, nil, "nothing to retry"},
|
||||
"gate failed": {gate, nil, "failed its gate"},
|
||||
} {
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%s: %+v", name, got)
|
||||
}
|
||||
o := got[0]
|
||||
if len(o.Actions) != 0 {
|
||||
t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions)
|
||||
}
|
||||
if !strings.Contains(o.Said, c.says) {
|
||||
t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said)
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) {
|
||||
t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23.
|
||||
func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(t.Context())
|
||||
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ := k.Open(t.Context())
|
||||
if len(open) != 1 || open[0].Kind != "plan-failed" {
|
||||
t.Fatalf("not raised: %+v", open)
|
||||
}
|
||||
// Retried: the plan is building, so it is no longer among the failed.
|
||||
f = calm(now.Add(time.Minute))
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ = k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition outlived the failure: %+v", open)
|
||||
}
|
||||
// Failed again: asked again, with Retry, and nothing retried it.
|
||||
f = calm(now.Add(2 * time.Minute))
|
||||
again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) })
|
||||
f.failed = []failedPlanFacts{failedFactsFor(again)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(t.Context())
|
||||
if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" {
|
||||
t.Fatalf("a second failure was not asked again with Retry: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing
|
||||
// S23 produces is an observation that offers an answer.
|
||||
func TestNoClockRetriesAFailedPlan(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))}
|
||||
for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} {
|
||||
f.now = now.Add(later)
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" {
|
||||
t.Fatalf("after %s: %+v", later, got)
|
||||
}
|
||||
}
|
||||
// A plan failed longer ago than the week it is kept is left to `plans`.
|
||||
f.now = now.Add(planFailedWithin + time.Second)
|
||||
if got := watchPlansFailed(f); len(got) != 0 {
|
||||
t.Fatalf("a plan failed beyond the window is still raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A later plan of the same repository and branch, in a state, made after the failed one.
|
||||
func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan {
|
||||
q := aFailedPlan(id, func(p *inventory.Plan) {
|
||||
p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour)
|
||||
})
|
||||
return q
|
||||
}
|
||||
|
||||
// 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan.
|
||||
func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) {
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
||||
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
||||
err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{})
|
||||
if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") {
|
||||
t.Fatalf("the retry was not refused, naming the later plan: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing.
|
||||
func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) {
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
||||
open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed)
|
||||
err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{})
|
||||
if err == nil || !strings.Contains(err.Error(), "supersedes it") {
|
||||
t.Fatalf("with the later plan open the refusal is not today's: %v", err)
|
||||
}
|
||||
other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed)
|
||||
if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil {
|
||||
t.Fatalf("a later done plan of another branch refused the retry: %v", err)
|
||||
}
|
||||
earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed)
|
||||
earlier.Created = failed.Created.Add(-time.Hour)
|
||||
if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil {
|
||||
t.Fatalf("an earlier done plan refused the retry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery.
|
||||
func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) })
|
||||
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(failed, later)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 || len(got[0].Actions) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") {
|
||||
t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation)
|
||||
}
|
||||
if strings.Contains(o.Explanation, "details say why") {
|
||||
t.Errorf("the generic fallback was said: %q", o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("not plain: %s\n%+v", why, words)
|
||||
}
|
||||
}
|
||||
|
||||
// plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since.
|
||||
type plansBehind []inventory.Plan
|
||||
|
||||
func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) {
|
||||
var out []inventory.Plan
|
||||
for _, p := range s {
|
||||
if p.Open() {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) {
|
||||
all := append([]inventory.Plan(nil), s...)
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) })
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) {
|
||||
var out []inventory.Plan
|
||||
for _, p := range s {
|
||||
if p.Created.After(since) || p.Open() {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it.
|
||||
func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) {
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) })
|
||||
all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)}
|
||||
for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50
|
||||
other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed)
|
||||
other.Repository = "novox/other"
|
||||
other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour)
|
||||
all = append(all, other)
|
||||
}
|
||||
plans, err := plansForRetry(t.Context(), all, failed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") {
|
||||
t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its
|
||||
// plan's last change passes 7 days.
|
||||
func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) })
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(old)}
|
||||
if got := watchPlansFailed(f); len(got) != 0 {
|
||||
t.Fatalf("a plan last changed 8 days ago raised %+v", got)
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(t.Context())
|
||||
fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) })
|
||||
f.failed = []failedPlanFacts{failedFactsFor(fresh)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
||||
t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open)
|
||||
}
|
||||
f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition outlived the 7 days: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -154,6 +154,31 @@ func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bo
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// newerDonePlan is a done plan of the same repository and branch made after this one: the newer commit it delivered
|
||||
// runs now (novox/hq ADR 0451).
|
||||
func newerDonePlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
var found inventory.Plan
|
||||
var ok bool
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || q.State != inventory.PlanDone || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
if !ok || q.Created.After(found.Created) {
|
||||
found, ok = q, true
|
||||
}
|
||||
}
|
||||
return found, ok
|
||||
}
|
||||
|
||||
// passedBy is the refusal of a retry of a plan that a newer done plan of its repository and branch has passed.
|
||||
type passedBy struct{ plan, by, repository, commit string }
|
||||
|
||||
func (e passedBy) Error() string {
|
||||
return fmt.Sprintf("%s was passed by %s, a newer plan of %s (at %s) that is done: retrying it would build and "+
|
||||
"send the older commit over the newer one that runs", e.plan, e.by, e.repository, short(e.commit))
|
||||
}
|
||||
|
||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error {
|
||||
switch {
|
||||
@@ -165,6 +190,11 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]st
|
||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||
}
|
||||
|
||||
// **A newer delivery of the branch has passed it** (novox/hq ADR 0451): retrying would build and send the older
|
||||
// commit over the newer one that runs.
|
||||
if q, found := newerDonePlan(p, plans); found {
|
||||
return passedBy{plan: p.ID, by: q.ID, repository: q.Repository, commit: q.Commit}
|
||||
}
|
||||
if len(failedIn(p)) > 0 {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
@@ -275,6 +305,32 @@ func goesOnAgain(p *inventory.Plan) {
|
||||
}
|
||||
}
|
||||
|
||||
// planReader is what a retry reads the mesh's plans through.
|
||||
type planReader interface {
|
||||
OpenPlans(ctx context.Context) ([]inventory.Plan, error)
|
||||
RecentPlans(ctx context.Context, limit int) ([]inventory.Plan, error)
|
||||
PlansSince(ctx context.Context, since time.Time) ([]inventory.Plan, error)
|
||||
}
|
||||
|
||||
// plansForRetry are the plans a retry of p judges itself against.
|
||||
func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inventory.Plan, error) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **Every plan made after this one** (novox/hq ADR 0451 §1): a plan that passed it is seen however many plans
|
||||
// were made since, not only when it is among the newest.
|
||||
later, err := inv.PlansSince(ctx, p.Created)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(append(plans, recent...), later...), nil
|
||||
}
|
||||
|
||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||
// that tier: what follows is the plan going on as if they had built the first time.
|
||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
@@ -288,15 +344,10 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
plans, err := plansForRetry(ctx, inv, p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans = append(plans, recent...)
|
||||
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
||||
|
||||
@@ -79,11 +79,14 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
if err := retryRefusal(stopped, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||
}
|
||||
// A failed or done plan does not supersede it.
|
||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour)),
|
||||
plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}, ownPathOf(nil)); err != nil {
|
||||
// A later failed plan does not supersede it; a later done one passes it (novox/hq ADR 0451).
|
||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}, ownPathOf(nil)); err != nil {
|
||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||
}
|
||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "plan-5") {
|
||||
t.Errorf("a plan a later done plan passed was retried: %v", err)
|
||||
}
|
||||
// Another branch's open walk, or an older one, does not supersede it, and is one walk open: one at a time
|
||||
// (novox/hq ADR 0276).
|
||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// seatPassing is what passing seats on needs of the outside: the record's compare-and-set, and the hand act.
|
||||
type seatPassing struct {
|
||||
pass func(ctx context.Context, p catalogue.SeatPass) (bool, error)
|
||||
act func(ctx context.Context, p catalogue.SeatPass) error
|
||||
say io.Writer
|
||||
}
|
||||
|
||||
// passSeatsOn records, in the composition that finds it, the new holder of each seat held once whose holder on
|
||||
// record is assigned nowhere and which has exactly one claimant able to hold it, and writes the controller's own
|
||||
// hand act for it (novox/hq ADR 0449 §4, issue 1317). It answers the holdings as they are after it, so the
|
||||
// resolution that follows reads the record it just moved. A failed write is said and leaves the record as it was:
|
||||
// a composition is never refused for it.
|
||||
func passSeatsOn(ctx context.Context, sp seatPassing, holdings []catalogue.Held,
|
||||
assignments []catalogue.Assignment) []catalogue.Held {
|
||||
passes, said := catalogue.SeatsToPassOn(holdings, assignments)
|
||||
for _, line := range said {
|
||||
fmt.Fprintln(sp.say, line)
|
||||
}
|
||||
for _, p := range passes {
|
||||
moved, err := sp.pass(ctx, p)
|
||||
if err != nil {
|
||||
fmt.Fprintf(sp.say, "%s could not be passed from %s on %s to %s on %s: %v\n", p.Seat.Name,
|
||||
p.From.Module, p.From.Node, p.To.Module, p.To.Node, err)
|
||||
continue
|
||||
}
|
||||
if !moved {
|
||||
continue // another composition passed it first, and wrote the act
|
||||
}
|
||||
if err := sp.act(ctx, p); err != nil {
|
||||
fmt.Fprintf(sp.say, "%s passed from %s on %s to %s on %s, and the hand act could NOT be recorded: %v\n",
|
||||
p.Seat.Name, p.From.Module, p.From.Node, p.To.Module, p.To.Node, err)
|
||||
}
|
||||
var next []catalogue.Held
|
||||
for _, h := range holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == p.Seat.Name {
|
||||
continue
|
||||
}
|
||||
next = append(next, h)
|
||||
}
|
||||
p.To.Site = p.From.Site
|
||||
holdings = append(next, p.To)
|
||||
}
|
||||
return holdings
|
||||
}
|
||||
|
||||
// seatPassHandAct is the controller's own entry for a pass: it is the act that puts the holder on record,
|
||||
// where a person's `seat --to` would have, so the record of who held what stays complete (ADR 0449 §4).
|
||||
func seatPassHandAct(p catalogue.SeatPass) link.HandAct {
|
||||
return link.HandAct{
|
||||
By: "the controller",
|
||||
Verb: "seat",
|
||||
Args: []string{p.Seat.Name, "--to", p.To.Node + "/" + p.To.Module, "--was", p.From.Node + "/" + p.From.Module},
|
||||
Why: fmt.Sprintf("%s on %s, the holder on record of %s, is assigned on no machine, and %s on %s is the only "+
|
||||
"assignment that claims it and can hold it (ADR 0449 §4)", p.From.Module, p.From.Node, p.Seat.Name,
|
||||
p.To.Module, p.To.Node),
|
||||
Cause: "seat-passed",
|
||||
Outcome: "done",
|
||||
}
|
||||
}
|
||||
|
||||
// assignmentsOf are the assignments of every machine, as the passing of a seat reads them. **A module assigned and
|
||||
// missing from the shelf is still assigned**: it is an assignment with no claims, so its holder is not taken for gone.
|
||||
func assignmentsOf(shelf map[string]catalogue.Manifest, assigned map[string][]string) []catalogue.Assignment {
|
||||
var out []catalogue.Assignment
|
||||
for node, modules := range assigned {
|
||||
for _, module := range modules {
|
||||
m, ok := shelf[module]
|
||||
if !ok {
|
||||
m = catalogue.Manifest{Module: module}
|
||||
}
|
||||
out = append(out, catalogue.Assignment{Node: node, Manifest: m})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatPassingOn is how a composition on this inventory passes a seat: against the store's record with the hand act
|
||||
// on the bus, or, on a throwaway mesh (the merge gate's), in memory only, writing nothing anywhere.
|
||||
func seatPassingOn(inv *inventory.Inventory, w io.Writer) seatPassing {
|
||||
if inv.Throwaway() {
|
||||
return seatPassing{say: w,
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return true, nil },
|
||||
act: func(context.Context, catalogue.SeatPass) error { return nil }}
|
||||
}
|
||||
return seatPassing{
|
||||
say: w,
|
||||
pass: func(ctx context.Context, p catalogue.SeatPass) (bool, error) {
|
||||
return inv.PassSeat(ctx, p.Seat.Name, p.Seat.Scope, p.From, p.To)
|
||||
},
|
||||
act: func(ctx context.Context, p catalogue.SeatPass) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, seatPassHandAct(p))
|
||||
return err
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// theMeshPassesSeatsOn is passSeatsOn as the composition runs it. **Only when every machine's assignments were
|
||||
// read** (assignedErr nil): a machine whose read failed would look as if its modules were assigned nowhere, and
|
||||
// its holder would lose the seat. Said to where the composition says its other lines (sayLeftOut).
|
||||
func theMeshPassesSeatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
||||
assigned map[string][]string, assignedErr error, holdings []catalogue.Held) []catalogue.Held {
|
||||
var w io.Writer = os.Stderr
|
||||
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
||||
w = lw
|
||||
}
|
||||
if assignedErr != nil {
|
||||
fmt.Fprintf(w, "no seat is passed on in this composition: a machine's assignments could not be read, so a "+
|
||||
"holder on record might look assigned nowhere: %v\n", assignedErr)
|
||||
return holdings
|
||||
}
|
||||
return passSeatsOn(ctx, seatPassingOn(inv, w), holdings, assignmentsOf(shelf, assigned))
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aBrokerClaiming(module string) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: module,
|
||||
Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}
|
||||
}
|
||||
|
||||
func withBusSeat(t *testing.T) {
|
||||
t.Helper()
|
||||
was := catalogue.Seats()
|
||||
t.Cleanup(func() { catalogue.UseSeats(was) })
|
||||
catalogue.UseSeats([]catalogue.Seat{{Name: "mesh-broker", Scope: catalogue.ScopeMesh, Delivers: "mesh-bus", Decision: "test"}})
|
||||
}
|
||||
|
||||
// 1317/AC1: the composition records the claimant as the holder and writes one hand act by the controller naming
|
||||
// the seat, the former holder and the new one; the holdings it answers name the new holder.
|
||||
func TestACompositionPassesTheSeatAndWritesOneHandActByTheController(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
var passed []catalogue.SeatPass
|
||||
var acts []link.HandAct
|
||||
var said bytes.Buffer
|
||||
sp := seatPassing{say: &said,
|
||||
pass: func(_ context.Context, p catalogue.SeatPass) (bool, error) {
|
||||
passed = append(passed, p)
|
||||
return true, nil
|
||||
},
|
||||
act: func(_ context.Context, p catalogue.SeatPass) error {
|
||||
acts = append(acts, seatPassHandAct(p))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||
if len(got) != 1 || got[0].Module != "gate" || got[0].Node != "novox" {
|
||||
t.Fatalf("the holdings after the composition do not name the gate: %+v", got)
|
||||
}
|
||||
if len(passed) != 1 || len(acts) != 1 {
|
||||
t.Fatalf("passes=%d acts=%d, want one each", len(passed), len(acts))
|
||||
}
|
||||
a := acts[0]
|
||||
text := strings.Join(append([]string{a.By, a.Verb, a.Why}, a.Args...), " ")
|
||||
for _, want := range []string{"the controller", "mesh-broker", "novox/old-broker", "novox/gate"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("the hand act does not name %q: %+v", want, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1317/AC2: two claimants, nothing recorded, the refusal said.
|
||||
func TestACompositionWithTwoClaimantsRecordsNoneAndSaysSo(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
sp := seatPassing{say: &bytes.Buffer{},
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) {
|
||||
t.Fatal("a seat with two claimants was passed")
|
||||
return false, nil
|
||||
},
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("an act was written"); return nil },
|
||||
}
|
||||
var said bytes.Buffer
|
||||
sp.say = &said
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{
|
||||
{Node: "novox", Manifest: aBrokerClaiming("gate")}, {Node: "ace", Manifest: aBrokerClaiming("other")}})
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("the record changed: %+v", got)
|
||||
}
|
||||
if !strings.Contains(said.String(), "gate") || !strings.Contains(said.String(), "other") ||
|
||||
!strings.Contains(said.String(), "seat mesh-broker --to") {
|
||||
t.Fatalf("the refusal is not said with both and the remedy: %q", said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 1317/AC3 and AC4: nothing changes while the holder is assigned, nor with no claimant.
|
||||
func TestACompositionChangesNothingWhileTheHolderIsAssignedOrNobodyClaims(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
sp := seatPassing{say: &bytes.Buffer{},
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { t.Fatal("passed"); return false, nil },
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil },
|
||||
}
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{
|
||||
{Node: "novox", Manifest: aBrokerClaiming("old-broker")}, {Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("holder assigned: record changed: %+v", got)
|
||||
}
|
||||
got = passSeatsOn(t.Context(), sp, holdings, nil)
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("no claimant: record changed: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A composition that lost the compare-and-set writes no act, and a failed write leaves the record.
|
||||
func TestALostPassWritesNoActAndAFailedOneLeavesTheRecord(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
as := []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}}
|
||||
lost := seatPassing{say: &bytes.Buffer{},
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, nil },
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act without a pass"); return nil },
|
||||
}
|
||||
passSeatsOn(t.Context(), lost, holdings, as)
|
||||
var said bytes.Buffer
|
||||
sp := seatPassing{say: &said,
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, context.DeadlineExceeded },
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil },
|
||||
}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, as)
|
||||
if got[0].Module != "old-broker" || !strings.Contains(said.String(), "could not be passed") {
|
||||
t.Fatalf("got %+v said %q", got, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A pass is the mesh working as ADR 0449 §4 decided, never a repair: two of them raise no healer-wanted and
|
||||
// count as no repair.
|
||||
func TestTwoSeatPassesAreNoRepairAndRaiseNoHealerWanted(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
now := time.Date(2026, 10, 19, 20, 0, 0, 0, time.UTC)
|
||||
mk := func(at time.Time) link.HandAct {
|
||||
a := seatPassHandAct(catalogue.SeatPass{Seat: catalogue.Seat{Name: "mesh-broker"},
|
||||
From: catalogue.Held{Node: "novox", Module: "old"}, To: catalogue.Held{Node: "novox", Module: "gate"}})
|
||||
a.At = at
|
||||
return a
|
||||
}
|
||||
acts := []link.HandAct{mk(now.Add(-26 * time.Hour)), mk(now.Add(-time.Hour))}
|
||||
if r := repairs(acts); len(r) != 0 {
|
||||
t.Fatalf("a seat pass counts as a repair: %+v", r)
|
||||
}
|
||||
f := calm(now)
|
||||
f.handActs = acts
|
||||
if got := watchHandActs(f); len(got) != 0 {
|
||||
t.Fatalf("two passes raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed read of one machine's assignments passes no seat: its holder would look assigned nowhere.
|
||||
func TestNoSeatIsPassedWhenAMachinesAssignmentsCouldNotBeRead(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
var said bytes.Buffer
|
||||
ctx := leftOutSaidBy(t.Context(), &said)
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "ace", Module: "old-broker"}}
|
||||
shelf := map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")}
|
||||
got := theMeshPassesSeatsOn(ctx, nil, shelf, map[string][]string{"novox": {"gate"}},
|
||||
errors.New("ace: the bus did not answer"), holdings)
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("the record moved: %+v", got)
|
||||
}
|
||||
if !strings.Contains(said.String(), "no seat is passed") || !strings.Contains(said.String(), "ace") {
|
||||
t.Errorf("not said: %q", said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A holder that is assigned but missing from the shelf is still assigned.
|
||||
func TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
as := assignmentsOf(map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")},
|
||||
map[string][]string{"novox": {"gate", "old-broker"}})
|
||||
passes, _ := catalogue.SeatsToPassOn(
|
||||
[]catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as)
|
||||
if len(passes) != 0 {
|
||||
t.Fatalf("a holder missing from the shelf lost its seat: %+v", passes)
|
||||
}
|
||||
}
|
||||
|
||||
// The merge gate's throwaway mesh passes the seat in memory and writes nothing: no record, no hand act.
|
||||
func TestAThrowawayMeshPassesTheSeatInMemoryAndWritesNothing(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
inv := &inventory.Inventory{}
|
||||
inv.MarkThrowaway()
|
||||
var said bytes.Buffer
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), seatPassingOn(inv, &said), holdings,
|
||||
[]catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||
if len(got) != 1 || got[0].Module != "gate" {
|
||||
t.Fatalf("the gate's view does not hold the pass: %+v", got)
|
||||
}
|
||||
if said.Len() != 0 {
|
||||
t.Errorf("the throwaway composition said %q (a write was tried)", said.String())
|
||||
}
|
||||
}
|
||||
@@ -234,6 +234,16 @@ var signalsTable = []signalRow{
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.batches, func(b batchFacts) time.Time { return b.created })
|
||||
}},
|
||||
{Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan",
|
||||
Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)",
|
||||
Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " +
|
||||
"words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " +
|
||||
"from the plan's last change, and ended when the plan is no longer failed",
|
||||
Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3,
|
||||
needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
|
||||
}},
|
||||
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
||||
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
||||
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
||||
@@ -404,6 +414,34 @@ func watchPlans(f *signalFacts) []conditions.Observation {
|
||||
return out
|
||||
}
|
||||
|
||||
// kindPlanFailed is S23's kind: plan.<id>.plan-failed.
|
||||
const kindPlanFailed = "plan-failed"
|
||||
|
||||
// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no
|
||||
// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to
|
||||
// the operator). A plan failed longer ago than this is left to `plans`.
|
||||
const planFailedWithin = 7 * 24 * time.Hour
|
||||
|
||||
// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when
|
||||
// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2).
|
||||
// The condition ends when the plan is no longer failed, because it is no longer among the facts.
|
||||
func watchPlansFailed(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, p := range f.failed {
|
||||
if f.now.Sub(p.since) > planFailedWithin {
|
||||
continue
|
||||
}
|
||||
headline, explanation, needs, actions, resolved := planFailedWords(p)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed,
|
||||
Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit),
|
||||
p.failureSaid()),
|
||||
Said: p.failureSaid() + p.refusalSaid(),
|
||||
Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// kindWalkWaiting is S16's kind: plan.<id>.waiting.
|
||||
const kindWalkWaiting = "waiting"
|
||||
|
||||
|
||||
@@ -160,6 +160,17 @@ var suppressions = map[string]suppression{
|
||||
created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}}
|
||||
},
|
||||
},
|
||||
// A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449).
|
||||
"S23": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
|
||||
since: f.now.Add(-planFailedWithin - time.Second)}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
|
||||
since: f.now.Add(-time.Second)}}
|
||||
},
|
||||
},
|
||||
// A batch queued behind an open walk past that walk's bound, named.
|
||||
"S19": {
|
||||
inside: func(f *signalFacts) {
|
||||
|
||||
@@ -55,6 +55,9 @@ type signalFacts struct {
|
||||
waits []waitFacts
|
||||
// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
|
||||
batches []batchFacts
|
||||
// failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449).
|
||||
failed []failedPlanFacts
|
||||
failedErr error
|
||||
// refusedSends are the sends a machine refused within the hour for the generation they were composed
|
||||
// from, each naming its sender (S20, novox/hq issue 234).
|
||||
refusedSends []inventory.Send
|
||||
@@ -353,6 +356,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
if f.plansErr == nil {
|
||||
f.batches, f.plansErr = gatherBatches(ctx, inv, now)
|
||||
}
|
||||
f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now)
|
||||
f.loop, f.loopErr = w.gatherLoop()
|
||||
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||
if open, err := w.keeper.Open(ctx); err != nil {
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// An assignment as the passing of a seat reads it: a module on a machine, with its manifest.
|
||||
type Assignment struct {
|
||||
Node string
|
||||
Manifest Manifest
|
||||
}
|
||||
|
||||
// SeatPass is one seat whose holder on record is assigned nowhere, and the only assignment that can take it.
|
||||
type SeatPass struct {
|
||||
Seat Seat
|
||||
From Held
|
||||
To Held
|
||||
}
|
||||
|
||||
// SeatsToPassOn reads the mesh for seats held once whose holder on record has left (novox/hq ADR 0449 §4).
|
||||
//
|
||||
// **A holder on record that is assigned on no machine holds nothing, and a seat that answers nothing is
|
||||
// the day a delivery removed its module.** Of such a seat, exactly one assignment in the mesh able to hold
|
||||
// it (it claims the seat at its scope and CanHold accepts it) takes it over: that is a pass, and the caller
|
||||
// records it and writes the hand act. With no such assignment the record stays; with two or more nothing is
|
||||
// recorded and the answer says so, naming both and the `seat --to` that settles it. A holder still assigned
|
||||
// anywhere changes nothing: the other claimants stand beside it, eligible and silent (ADR 0131). Only mesh
|
||||
// seats held once are read: a replicated seat's holders are added by an act (ADR 0223), a kinded bench is held
|
||||
// by its claims, and a seat with several holders on record is a store that disagrees with the mesh (refused
|
||||
// by the resolver).
|
||||
func SeatsToPassOn(holdings []Held, assignments []Assignment) ([]SeatPass, []string) {
|
||||
assigned := map[string]bool{}
|
||||
for _, a := range assignments {
|
||||
assigned[a.Manifest.Module] = true
|
||||
}
|
||||
bySeat := map[string][]Held{}
|
||||
var order []string
|
||||
for _, h := range holdings {
|
||||
s, ok := SeatNamed(h.Claim)
|
||||
if !ok || s.Replicated || s.Scope != ScopeMesh || h.Scope != ScopeMesh || KindedBenches[s.Name] {
|
||||
continue
|
||||
}
|
||||
if _, seen := bySeat[s.Name]; !seen {
|
||||
order = append(order, s.Name)
|
||||
}
|
||||
bySeat[s.Name] = append(bySeat[s.Name], h)
|
||||
}
|
||||
sort.Strings(order)
|
||||
|
||||
var passes []SeatPass
|
||||
var said []string
|
||||
for _, name := range order {
|
||||
recs := bySeat[name]
|
||||
if len(recs) != 1 || assigned[recs[0].Module] {
|
||||
continue
|
||||
}
|
||||
seat, _ := SeatNamed(name)
|
||||
var claimants []Held
|
||||
for _, a := range assignments {
|
||||
if CanHold(a.Manifest, seat) != nil {
|
||||
continue
|
||||
}
|
||||
claimants = append(claimants, Held{Claim: seat.Name, Scope: seat.Scope, Node: a.Node, Module: a.Manifest.Module})
|
||||
}
|
||||
sort.Slice(claimants, func(i, j int) bool {
|
||||
if claimants[i].Node != claimants[j].Node {
|
||||
return claimants[i].Node < claimants[j].Node
|
||||
}
|
||||
return claimants[i].Module < claimants[j].Module
|
||||
})
|
||||
switch len(claimants) {
|
||||
case 0:
|
||||
case 1:
|
||||
passes = append(passes, SeatPass{Seat: seat, From: recs[0], To: claimants[0]})
|
||||
default:
|
||||
said = append(said, fmt.Sprintf(
|
||||
"%s on %s is on record as the holder of %q and is assigned nowhere, and %s on %s and %s on %s both claim it "+
|
||||
"— nothing is recorded; `seat %s --to <node>/<module>` records the one that holds it",
|
||||
recs[0].Module, recs[0].Node, seat.Name, claimants[0].Module, claimants[0].Node,
|
||||
claimants[1].Module, claimants[1].Node, seat.Name))
|
||||
}
|
||||
}
|
||||
return passes, said
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **A seat held once whose holder on record is assigned nowhere passes to its only claimant** (novox/hq ADR
|
||||
// 0449 §4, issue 1317).
|
||||
|
||||
func onRecord(node, module string) []Held {
|
||||
return []Held{{Claim: "mesh-broker", Scope: ScopeMesh, Node: node, Module: module}}
|
||||
}
|
||||
|
||||
// 1317/AC1
|
||||
func TestASeatWhoseHolderIsAssignedNowhereIsPassedToItsOnlyClaimant(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
passes, said := SeatsToPassOn(onRecord("novox", "old-broker"),
|
||||
[]Assignment{{Node: "novox", Manifest: newBroker()}})
|
||||
if len(said) != 0 || len(passes) != 1 {
|
||||
t.Fatalf("the only claimant was not given the seat: passes=%v said=%v", passes, said)
|
||||
}
|
||||
p := passes[0]
|
||||
if p.Seat.Name != "mesh-broker" || p.From.Module != "old-broker" || p.From.Node != "novox" ||
|
||||
p.To.Module != "new-broker" || p.To.Node != "novox" {
|
||||
t.Fatalf("the pass does not name the seat, the former holder and the new one: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// 1317/AC2
|
||||
func TestWithTwoClaimantsAndTheHolderGoneNothingIsPassedAndTheRefusalNamesBoth(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
passes, said := SeatsToPassOn(onRecord("novox", "gone"), []Assignment{
|
||||
{Node: "novox", Manifest: newBroker()}, {Node: "ace", Manifest: oldBroker()}})
|
||||
if len(passes) != 0 || len(said) != 1 {
|
||||
t.Fatalf("two claimants: passes=%v said=%v", passes, said)
|
||||
}
|
||||
for _, want := range []string{"old-broker", "new-broker", "seat mesh-broker --to"} {
|
||||
if !strings.Contains(said[0], want) {
|
||||
t.Fatalf("the refusal does not say %q: %s", want, said[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1317/AC3
|
||||
func TestAHolderStillAssignedSomewhereChangesNothing(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
// Assigned on another machine than the record names: still assigned somewhere.
|
||||
passes, said := SeatsToPassOn(onRecord("novox", "old-broker"), []Assignment{
|
||||
{Node: "ace", Manifest: oldBroker()}, {Node: "novox", Manifest: newBroker()}})
|
||||
if len(passes) != 0 || len(said) != 0 {
|
||||
t.Fatalf("a holder still assigned lost its seat: passes=%v said=%v", passes, said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1317/AC4
|
||||
func TestWithNoClaimantTheRecordStays(t *testing.T) {
|
||||
busSeatDelivering(t, "mesh-bus")
|
||||
passes, said := SeatsToPassOn(onRecord("novox", "old-broker"), []Assignment{
|
||||
{Node: "novox", Manifest: Manifest{Module: "other"}}})
|
||||
if len(passes) != 0 || len(said) != 0 {
|
||||
t.Fatalf("with no claimant: passes=%v said=%v", passes, said)
|
||||
}
|
||||
// A claimant that cannot hold the seat (it does not provide what the seat delivers) is no claimant.
|
||||
bad := Manifest{Module: "no-bus", Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}}
|
||||
passes, said = SeatsToPassOn(onRecord("novox", "old-broker"), []Assignment{{Node: "novox", Manifest: bad}})
|
||||
if len(passes) != 0 || len(said) != 0 {
|
||||
t.Fatalf("a module that cannot hold the seat was passed it: passes=%v said=%v", passes, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A replicated seat is never passed: its holders are added by an act (ADR 0223).
|
||||
func TestAReplicatedSeatIsNeverPassed(t *testing.T) {
|
||||
record := []Held{{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "novox", Module: "gone"}}
|
||||
resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}},
|
||||
Claims: []Claim{{Name: "mesh-dns-resolver", Scope: ScopeMesh}}}
|
||||
passes, _ := SeatsToPassOn(record, []Assignment{{Node: "novox", Manifest: resolver}})
|
||||
if len(passes) != 0 {
|
||||
t.Fatalf("a replicated seat was passed: %v", passes)
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,7 @@ func (i *Inventory) UseChain(c *chain.Chain) {
|
||||
// beside the real one, as the merge gate raises the facts snapshot's mesh in a throwaway store (issue 1121). Nothing of it
|
||||
// reaches the bus.
|
||||
func (i *Inventory) UseThrowawayChain() {
|
||||
i.MarkThrowaway()
|
||||
i.UseChain(chain.InMemory(nil))
|
||||
i.markLaidOut()
|
||||
}
|
||||
@@ -129,3 +130,9 @@ func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) {
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// MarkThrowaway says this inventory is a mesh raised for a moment: a composition on it writes nothing outside it.
|
||||
func (i *Inventory) MarkThrowaway() { i.throwaway = true }
|
||||
|
||||
// Throwaway says whether this inventory is a mesh raised for a moment (UseThrowawayChain).
|
||||
func (i *Inventory) Throwaway() bool { return i != nil && i.throwaway }
|
||||
|
||||
@@ -153,3 +153,24 @@ func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
|
||||
t.Fatalf("a handover left other holders on record: %+v", held)
|
||||
}
|
||||
}
|
||||
|
||||
// A pass is a compare-and-set: it moves the record only while the holder it was read with is still on it.
|
||||
func TestPassingASeatMovesTheRecordOnceAndOnlyFromTheHolderRead(t *testing.T) {
|
||||
inv, ctx := twoBrokersOnTwoNodes(t)
|
||||
if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, "anchor", "old-broker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
from := catalogue.Held{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "old-broker"}
|
||||
to := catalogue.Held{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "laptop", Module: "new-broker"}
|
||||
passed, err := inv.PassSeat(ctx, "mesh-broker", catalogue.ScopeMesh, from, to)
|
||||
if err != nil || !passed {
|
||||
t.Fatalf("the seat was not passed: %v %v", passed, err)
|
||||
}
|
||||
held, _ := inv.Holdings(ctx)
|
||||
if len(held) != 1 || held[0].Node != "laptop" || held[0].Module != "new-broker" {
|
||||
t.Fatalf("the record does not name the new holder alone: %+v", held)
|
||||
}
|
||||
if passed, err = inv.PassSeat(ctx, "mesh-broker", catalogue.ScopeMesh, from, to); err != nil || passed {
|
||||
t.Fatalf("a second pass from the same read moved the record again: %v %v", passed, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,9 @@ type Inventory struct {
|
||||
state *chainState
|
||||
// recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store.
|
||||
recorded func(context.Context, Duration) error
|
||||
// throwaway is set on an inventory raised for a moment beside the real mesh (the merge gate's): nothing it
|
||||
// composes is to be written to the store or the bus.
|
||||
throwaway bool
|
||||
}
|
||||
|
||||
// Open connects to the inventory store.
|
||||
|
||||
@@ -284,6 +284,48 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// PassSeat records `to` as the holder of a seat held once in place of `from`, only while `from` is still the
|
||||
// one on record: a compare-and-set in one transaction, so two compositions reading the same record cannot both
|
||||
// hand the seat over (novox/hq ADR 0449 §4). It answers false, with no error, when the record no longer names
|
||||
// `from` (another composition passed it first). The new holder must be an assignment.
|
||||
func (i *Inventory) PassSeat(ctx context.Context, seat, scope string, from, to catalogue.Held) (bool, error) {
|
||||
fromNode, err := i.NodeByName(ctx, from.Node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
toNode, err := i.NodeByName(ctx, to.Node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := i.mustBeAssigned(ctx, seat, to.Node, to.Module); err != nil {
|
||||
return false, err
|
||||
}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
gone, err := tx.Exec(ctx,
|
||||
`delete from seat_holding where seat = $1 and node = $2 and module = $3`,
|
||||
seat, fromNode.ID, from.Module)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("passing %s from %s on %s: %w", seat, from.Module, from.Node, err)
|
||||
}
|
||||
if gone.RowsAffected() == 0 {
|
||||
return false, nil
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`delete from seat_holding where seat = $1`, seat); err != nil {
|
||||
return false, fmt.Errorf("passing %s: %w", seat, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)`,
|
||||
seat, scope, toNode.ID, to.Module); err != nil {
|
||||
return false, fmt.Errorf("recording %s on %s as the holder of %s: %w", to.Module, to.Node, seat, err)
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
|
||||
// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
|
||||
// the seat's definition is compiled, and the store holds no column for it. Recording a holder
|
||||
|
||||
Reference in New Issue
Block a user