Archived
Compare commits
12
Commits
218b378db6
...
f996cf7b6b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f996cf7b6b | ||
|
|
bcfcfaaaf3 | ||
|
|
a3181e356b | ||
|
|
dd2fd28c92 | ||
|
|
9d165c7500 | ||
|
|
7630072e63 | ||
|
|
1e9dcf0667 | ||
|
|
cd6610cdf7 | ||
|
|
facc551ff2 | ||
|
|
fafa3fd9ef | ||
|
|
61d00a234a | ||
|
|
0a0f5b4a5a |
@@ -213,6 +213,8 @@ type asker struct {
|
||||
|
||||
saidNoRouter bool
|
||||
saidNoGrant bool
|
||||
// saidGrantCaveat is whether a grant held with a caveat was said, so it is said once.
|
||||
saidGrantCaveat bool
|
||||
|
||||
mu sync.Mutex
|
||||
nudged chan struct{}
|
||||
@@ -317,6 +319,13 @@ func (a *asker) reconcile(ctx context.Context) error {
|
||||
}
|
||||
return a.sayUnasked(ctx, unasked, "the bus does not hold the controller's grant to ask yet: "+why)
|
||||
}
|
||||
if why != "" && !a.saidGrantCaveat {
|
||||
// Held, with a caveat worth one line (an older bus module cannot confirm the reload).
|
||||
a.logf("the controller asks: %s", why)
|
||||
a.saidGrantCaveat = true
|
||||
} else if why == "" {
|
||||
a.saidGrantCaveat = false
|
||||
}
|
||||
a.saidNoGrant = false
|
||||
}
|
||||
channels := ""
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -808,3 +809,207 @@ func TestARefusalIsAskedAgainAndTheUndeliveredConditionClearsOnceTaken(t *testin
|
||||
t.Errorf("the explanation says its verdict twice: %q", e)
|
||||
}
|
||||
}
|
||||
|
||||
// hq issue 353, the review: the bus holds the controller's grant to ask only when both facts hold — the mesh
|
||||
// composes the controller's own grant, and the bus's module says it is in force on the running server — and
|
||||
// each fact not known is not held, said with why.
|
||||
func TestTheGrantIsHeldOnlyWhenComposedAndInForceOnTheBus(t *testing.T) {
|
||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
||||
no := func(why string) grantJudge {
|
||||
return func(context.Context) (bool, string, error) { return false, why, nil }
|
||||
}
|
||||
broken := func(context.Context) (bool, string, error) { return false, "", errors.New("the store is away") }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
composed, inForce grantJudge
|
||||
held bool
|
||||
says string
|
||||
}{
|
||||
{"composed and in force", yes, yes, true, ""},
|
||||
{"not composed", no("no router is assigned"), yes, false, "no router is assigned"},
|
||||
{"composed, written, not reloaded", yes, no("the server has not reloaded"), false, "has not reloaded"},
|
||||
{"the composition unknown", broken, yes, false, "could not be worked out: the store is away"},
|
||||
{"the bus unknown", yes, broken, false, "could not be worked out: the store is away"},
|
||||
{"nothing judges the bus", yes, nil, false, "not judged"},
|
||||
} {
|
||||
held, why, err := grantHeldBy(c.composed, c.inForce, time.Now)(context.Background())
|
||||
if err != nil || held != c.held || !strings.Contains(why, c.says) {
|
||||
t.Errorf("%s: held %v (%q, %v)", c.name, held, why, err)
|
||||
}
|
||||
}
|
||||
// Remembered for grantLookEvery, then judged again.
|
||||
now := time.Date(2026, 10, 9, 18, 0, 0, 0, time.UTC)
|
||||
inForce := false
|
||||
judge := grantHeldBy(yes, func(context.Context) (bool, string, error) { return inForce, "not yet", nil },
|
||||
func() time.Time { return now })
|
||||
if held, _, _ := judge(context.Background()); held {
|
||||
t.Fatal("held before the bus enforces it")
|
||||
}
|
||||
inForce = true
|
||||
if held, _, _ := judge(context.Background()); held {
|
||||
t.Error("judged again inside grantLookEvery")
|
||||
}
|
||||
now = now.Add(grantLookEvery)
|
||||
if held, _, _ := judge(context.Background()); !held {
|
||||
t.Error("not judged again after grantLookEvery")
|
||||
}
|
||||
}
|
||||
|
||||
// The controller's own grant, as composed: present while a router that takes asks is assigned, absent
|
||||
// otherwise — whatever else in the user list changes.
|
||||
func TestTheControllersOwnGrantToAskIsReadFromTheComposition(t *testing.T) {
|
||||
router := broker.Declared{Module: "messenger", Holds: []broker.Seat{{Name: broker.AsksSeat, Scope: "mesh",
|
||||
Accepts: []string{"ask", "cancel"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "decided"}}}}
|
||||
other := broker.Declared{Module: "dunst"}
|
||||
with := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {router, other}}}
|
||||
without := broker.Records{Nodes: []string{"anchor"}, Assigned: map[string][]broker.Declared{"anchor": {other}}}
|
||||
if may, err := controllerMayAsk(with); err != nil || !may {
|
||||
t.Errorf("with a router assigned: %v %v", may, err)
|
||||
}
|
||||
if may, err := controllerMayAsk(without); err != nil || may {
|
||||
t.Errorf("with no router assigned: %v %v", may, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The bus module's answer, with the record's word on the list sent as the check only where the answer does
|
||||
// not settle it (reviews 22 and 31 of PR 188); each way it is not held names its own remedy.
|
||||
func TestTheBusModulesAnswerNamesTheRemedyForEachCase(t *testing.T) {
|
||||
const (
|
||||
reloaded = `{"allowed":true,"in_force":true,"file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:56:05Z"}`
|
||||
notLoaded = `{"allowed":true,"in_force":false,"why":"the user list on the machine allows it, and the server has not reloaded since it was written","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`
|
||||
noVarz = `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`
|
||||
noStat = `{"allowed":true,"in_force":false,"why":"when the bus's user list was written could not be read inside mesh-broker-nats: exit 1"}`
|
||||
notAllowed = `{"allowed":false,"in_force":false,"why":"the user list on the machine does not allow it"}`
|
||||
)
|
||||
current := busListState{Machine: "anchor"}
|
||||
behind := busListState{Machine: "anchor", Behind: true}
|
||||
stepped := busListState{Machine: "anchor", Behind: true, BusStep: "sending anchor would replace the bus; `bus upgrade --why …`"}
|
||||
uncomposed := busListState{Machine: "anchor", Behind: true, Uncomposed: errors.New("module foo requires a provision nobody provides")}
|
||||
for _, c := range []struct {
|
||||
name, answer string
|
||||
askErr error
|
||||
list busListState
|
||||
listErr error
|
||||
held bool
|
||||
says, not string
|
||||
}{
|
||||
{"in force", reloaded, nil, behind, nil, true, "", "push"},
|
||||
{"written, not reloaded", notLoaded, nil, current, nil, false, "within about 5 s", "push"},
|
||||
{"in force not known: varz", noVarz, nil, current, nil, false, "is not known: the server's last configuration load", "wait"},
|
||||
{"in force not known: stat", noStat, nil, current, nil, false, "is not known: when the bus's user list was written", "wait"},
|
||||
{"not allowed, list behind", notAllowed, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
||||
{"not allowed, bus step pending", notAllowed, nil, stepped, nil, false, "`bus upgrade", "`push anchor`"},
|
||||
{"not allowed, list uncomposed", notAllowed, nil, uncomposed, nil, false, "`plan anchor`", "`push anchor`"},
|
||||
{"older module, allowed, list current", `{"allowed":true}`, nil, current, nil, true, "not confirmed", "push"},
|
||||
{"older module, allowed, list behind", `{"allowed":true}`, nil, behind, nil, true, "not confirmed", "push"},
|
||||
{"older module, allowed, list unknown", `{"allowed":true}`, nil, current, errors.New("the store is away"), true, "`bus upgrade`", "push"},
|
||||
{"older module, not allowed, list behind", `{"allowed":false}`, nil, behind, nil, false, "`push anchor` carries it", "5 s"},
|
||||
{"module cannot answer, list current", "", errors.New("no responders"), current, nil, true, "could not answer", "push"},
|
||||
{"module cannot answer, list behind", "", errors.New("no responders"), behind, nil, false, "`push anchor` carries it", ""},
|
||||
{"module cannot answer, list unknown", "", errors.New("no responders"), current, errors.New("the store is away"), false, "could not be worked out", ""},
|
||||
{"answer unreadable, list behind", `not json`, nil, behind, nil, false, "could not be read", ""},
|
||||
} {
|
||||
judge := busGrantBy(
|
||||
func(context.Context) (string, json.RawMessage, error) {
|
||||
return "anchor", json.RawMessage(c.answer), c.askErr
|
||||
},
|
||||
func(context.Context) (busListState, error) { return c.list, c.listErr })
|
||||
held, why, _ := judgeGrantOnce(judge)
|
||||
if held != c.held || !strings.Contains(why, c.says) || (c.not != "" && strings.Contains(why, c.not)) {
|
||||
t.Errorf("%s: held %v (%q)", c.name, held, why)
|
||||
}
|
||||
}
|
||||
none := busGrantBy(func(context.Context) (string, json.RawMessage, error) { return "", nil, nil }, nil)
|
||||
if held, why, _ := judgeGrantOnce(none); held || !strings.Contains(why, "no module holds the bus") {
|
||||
t.Errorf("no bus: held %v (%q)", held, why)
|
||||
}
|
||||
}
|
||||
|
||||
// judgeGrantOnce is the whole judgement with the composed grant held, as the asker makes it.
|
||||
func judgeGrantOnce(bus grantJudge) (bool, string, error) {
|
||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
||||
return grantHeldBy(yes, bus, time.Now)(context.Background())
|
||||
}
|
||||
|
||||
// The orders reviews 22 and 31 of PR 188 found: this controller live before the bus module that answers
|
||||
// in_force, with the bus's machine behind on its list — and, after mesh-catalog #166 merged, unpushable until a
|
||||
// person's bus upgrade. An older module that allows the ask asks (once, with the caveat said); a new module that
|
||||
// has not reloaded names the wait, not a push; one that cannot say whether it reloaded names what is not known.
|
||||
func TestAnOlderBusModuleNeitherSilencesTheAsksNorNamesTheWrongRemedy(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, answer string
|
||||
list busListState
|
||||
asks bool
|
||||
says, not string
|
||||
}{
|
||||
{"older module, list current", `{"allowed":true}`, busListState{Machine: "anchor"}, true, "not confirmed", ""},
|
||||
{"older module, list behind, bus step pending", `{"allowed":true}`,
|
||||
busListState{Machine: "anchor", Behind: true, BusStep: "a planned step"}, true, "not confirmed", ""},
|
||||
{"new module, not reloaded",
|
||||
`{"allowed":true,"in_force":false,"why":"not reloaded","file_written":"2026-10-09T15:55:56Z","server_loaded":"2026-10-09T15:50:00Z"}`,
|
||||
busListState{Machine: "anchor"}, false, "within about 5 s", "push"},
|
||||
{"new module, reload not known", `{"allowed":true,"in_force":false,"why":"the server's last configuration load could not be read: refused"}`,
|
||||
busListState{Machine: "anchor"}, false, "is not known", "wait"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
var said []string
|
||||
r.a.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
var raised [][]conditions.Observation
|
||||
r.a.raise = func(_ context.Context, obs []conditions.Observation) error {
|
||||
raised = append(raised, obs)
|
||||
return nil
|
||||
}
|
||||
yes := func(context.Context) (bool, string, error) { return true, "", nil }
|
||||
r.a.grantHeld = grantHeldBy(yes, busGrantBy(
|
||||
func(context.Context) (string, json.RawMessage, error) {
|
||||
return "anchor", json.RawMessage(c.answer), nil
|
||||
},
|
||||
func(context.Context) (busListState, error) { return c.list, nil }),
|
||||
func() time.Time { return r.now })
|
||||
r.open = []conditions.Condition{heldCondition()}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
_ = r.a.reconcile(context.Background())
|
||||
sent := r.asksSent(t)
|
||||
if c.asks {
|
||||
if len(sent) != 1 || sent[0].About != heldCondition().Key {
|
||||
t.Fatalf("not asked: %+v", sent)
|
||||
}
|
||||
n := 0
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, c.says) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("the caveat said %d times: %q", n, said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(sent) != 0 {
|
||||
t.Fatalf("asked while the bus does not hold the grant: %+v", sent)
|
||||
}
|
||||
if len(raised) == 0 || len(raised[len(raised)-1]) != 1 ||
|
||||
raised[len(raised)-1][0].Kind != "asks-undelivered" {
|
||||
t.Fatalf("not said as undelivered: %+v", raised)
|
||||
}
|
||||
summary := raised[len(raised)-1][0].Summary
|
||||
if !strings.Contains(summary, c.says) || (c.not != "" && strings.Contains(summary, c.not)) {
|
||||
t.Errorf("the condition names the wrong remedy: %q", summary)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The serving asker is wired to the real judgement: with records and a bus it cannot read, the grant is not
|
||||
// held, and so nothing is asked (a stub saying yes, or no judgement at all, would ask).
|
||||
func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
||||
a := servingAsker(&stores{}, nil, nil, nil)
|
||||
if a.grantHeld == nil {
|
||||
t.Fatal("the serving asker does not judge the bus's grant")
|
||||
}
|
||||
held, why, err := a.grantHeld(context.Background())
|
||||
if err != nil || held || !strings.Contains(why, "could not be worked out") {
|
||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,12 +235,27 @@ func routerHereIn(inv *inventory.Inventory) func(ctx context.Context) (bool, err
|
||||
}
|
||||
}
|
||||
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353): the user list the
|
||||
// machine holding the bus was last sent is the one the mesh composes now (brokerBehind, the same judgement a
|
||||
// push makes to send that machine first). While it is behind, the controller's ask is refused by the bus,
|
||||
// whatever the record says of the router, so nothing is asked and the operator is told to push that machine.
|
||||
// Judged at most every grantLookEvery: composing the list resolves the bus's machine whole.
|
||||
func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||
// grantHeldIn says whether the bus holds the controller's grant to ask (novox/hq issue 353), judged on the two
|
||||
// facts that make it so, and failing closed on each:
|
||||
//
|
||||
// 1. **composed**: the user list the mesh composes now grants the controller itself to publish its ask
|
||||
// (controllerMayAsk) — the controller's own grant, not whether any user of the list changed;
|
||||
// 2. **in force**: the bus's own module, on the machine holding the bus, says the user list there allows it
|
||||
// and the server reloaded after that list was written (the nats module's `nats_user_can`, `in_force`) — a
|
||||
// list sent is not a list the bus enforces until it reloads.
|
||||
//
|
||||
// Anything that cannot be worked out is not held, with why, so the conditions that need the operator are said
|
||||
// as undelivered rather than asked into a refusal.
|
||||
func grantHeldIn(open *stores, conn *nats.Conn) func(ctx context.Context) (bool, string, error) {
|
||||
return grantHeldBy(composedGrantIn(open), busGrantIn(open, conn), time.Now)
|
||||
}
|
||||
|
||||
// grantJudge is one of the two facts: whether it holds, and why not.
|
||||
type grantJudge func(ctx context.Context) (bool, string, error)
|
||||
|
||||
// grantHeldBy is the judgement over the two facts, remembered for grantLookEvery: composing the list resolves
|
||||
// the bus's machine whole, and asking the bus's module is a call.
|
||||
func grantHeldBy(composed, inForce grantJudge, now func() time.Time) func(ctx context.Context) (bool, string, error) {
|
||||
var mu sync.Mutex
|
||||
var at time.Time
|
||||
var held bool
|
||||
@@ -248,22 +263,251 @@ func grantHeldIn(open *stores) func(ctx context.Context) (bool, string, error) {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !at.IsZero() && time.Since(at) < grantLookEvery {
|
||||
if !at.IsZero() && now().Sub(at) < grantLookEvery {
|
||||
return held, why, nil
|
||||
}
|
||||
machine, behind, err := brokerBehind(ctx, open, nil)
|
||||
held, why = judgeGrant(ctx, composed, inForce)
|
||||
at = now()
|
||||
return held, why, nil
|
||||
}
|
||||
}
|
||||
|
||||
// judgeGrant is held only when both facts hold; an error is a fact not known, never a yes. A fact that holds
|
||||
// with a caveat (a reload not confirmed) gives that caveat as the why of a yes.
|
||||
func judgeGrant(ctx context.Context, composed, inForce grantJudge) (bool, string) {
|
||||
var caveat []string
|
||||
for _, fact := range []struct {
|
||||
judge grantJudge
|
||||
what string
|
||||
}{{composed, "whether the mesh composes the controller a grant to ask"}, {inForce, "whether the bus enforces it"}} {
|
||||
if fact.judge == nil {
|
||||
return false, fact.what + " is not judged here"
|
||||
}
|
||||
ok, why, err := fact.judge(ctx)
|
||||
if err != nil {
|
||||
return false, fact.what + " could not be worked out: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return false, why
|
||||
}
|
||||
if why != "" {
|
||||
caveat = append(caveat, why)
|
||||
}
|
||||
}
|
||||
return true, strings.Join(caveat, "; ")
|
||||
}
|
||||
|
||||
// controllerMayAsk says whether the controller's own grant, as composed from these records, lets it publish its
|
||||
// ask (ADR 0259 §3: composed while a router that takes asks under its asker's name is assigned).
|
||||
func controllerMayAsk(records broker.Records) (bool, error) {
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Kind != broker.KindController {
|
||||
continue
|
||||
}
|
||||
perms, err := broker.PermissionsFor(u)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return broker.MayPublish(perms, asks.AskSubject(askerName)), nil
|
||||
}
|
||||
return false, errors.New("the composition holds no controller")
|
||||
}
|
||||
|
||||
// composedGrantIn is the first fact, from the mesh's records.
|
||||
func composedGrantIn(open *stores) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return false, "", errors.New("the mesh's records are not open")
|
||||
}
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
at, held, why = time.Now(), !behind, ""
|
||||
if behind {
|
||||
why = fmt.Sprintf("the bus's user list on %s is behind what the mesh composes, so the bus has not been "+
|
||||
"given the controller's grant to ask; `push %s` carries it", machine, machine)
|
||||
may, err := controllerMayAsk(records)
|
||||
if err != nil || may {
|
||||
return may, "", err
|
||||
}
|
||||
return held, why, nil
|
||||
return false, "the mesh composes the controller no grant to ask: no router that takes asks under its " +
|
||||
"asker's name is assigned", nil
|
||||
}
|
||||
}
|
||||
|
||||
// busGrantIn is the second fact, asked of the bus's own module on the machine holding the bus, with the check
|
||||
// main made before it (novox/hq issue 353, PR 186: is the user list sent to the bus's machine the one composed
|
||||
// now) kept for the cases the module's word does not settle.
|
||||
func busGrantIn(open *stores, conn *nats.Conn) grantJudge {
|
||||
ask := func(ctx context.Context) (string, json.RawMessage, error) {
|
||||
if open == nil || open.inventory == nil || conn == nil {
|
||||
return "", nil, errors.New("the mesh's records or the bus are not open")
|
||||
}
|
||||
holders, err := seatHolders(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
h, held := holders[theBrokerSeat]
|
||||
if !held || h.Node == "" || h.Module == "" {
|
||||
return "", nil, nil
|
||||
}
|
||||
a, err := link.AskModuleToolOn(ctx, conn, h.Module, busUserCanTool, h.Node, map[string]any{
|
||||
"user": broker.ControllerName, "action": "publish", "subject": asks.AskSubject(askerName)}, 15*time.Second)
|
||||
if err != nil {
|
||||
return h.Node, nil, err
|
||||
}
|
||||
if a.Error != "" {
|
||||
return h.Node, nil, errors.New(a.Error)
|
||||
}
|
||||
return h.Node, a.Result, nil
|
||||
}
|
||||
list := func(ctx context.Context) (busListState, error) {
|
||||
if open == nil || open.inventory == nil {
|
||||
return busListState{}, errors.New("the mesh's records are not open")
|
||||
}
|
||||
s, err := brokerListState(ctx, open, nil)
|
||||
if err != nil || s.Machine == "" || !s.Behind {
|
||||
return s, err
|
||||
}
|
||||
step, err := busHeld(ctx, open.inventory, []string{s.Machine})
|
||||
if err != nil {
|
||||
return busListState{}, err
|
||||
}
|
||||
s.BusStep = step[s.Machine]
|
||||
return s, nil
|
||||
}
|
||||
return busGrantBy(ask, list)
|
||||
}
|
||||
|
||||
// busListState is the user list on the bus's machine as the controller's record knows it: whether the list
|
||||
// sent there is behind the one composed now, why it could not be composed, and the bus step a send there waits
|
||||
// on (that machine cannot be pushed until a person's `bus upgrade`).
|
||||
type busListState struct {
|
||||
Machine string
|
||||
Behind bool
|
||||
Uncomposed error
|
||||
BusStep string
|
||||
}
|
||||
|
||||
// remedy is the one thing that sends the bus's machine the list composed now: the plan's own reason when it
|
||||
// cannot be composed, the bus upgrade while sends there wait on it, and otherwise a push of that machine.
|
||||
func (s busListState) remedy(node string) string {
|
||||
if s.Machine != "" {
|
||||
node = s.Machine
|
||||
}
|
||||
switch {
|
||||
case s.Uncomposed != nil:
|
||||
return fmt.Sprintf("the user list for %s cannot be composed (%v); `plan %s` says what to fix first",
|
||||
node, s.Uncomposed, node)
|
||||
case s.BusStep != "":
|
||||
return fmt.Sprintf("sends to %s wait on the bus's planned step, so a push will not carry it: %s", node, s.BusStep)
|
||||
}
|
||||
return fmt.Sprintf("`push %s` carries it", node)
|
||||
}
|
||||
|
||||
// busGrantBy judges the bus's word, and names for each way it is not held the one thing that makes it held:
|
||||
//
|
||||
// - **in force**: held;
|
||||
// - **allowed, from a module older than `in_force`**: held, with the caveat that the reload is not confirmed.
|
||||
// The file allows the publish, and the bus reloads within about 5 s of a change to it, so the worst case is
|
||||
// one ask refused in those seconds, which is asked again (issue 369). The list check is not consulted:
|
||||
// while a bus step is pending the bus's machine cannot be pushed, so any change to the composed list would
|
||||
// read as behind for good and silence every ask until a person's bus upgrade (the review of PR 188);
|
||||
// - **allowed, written, not reloaded** (the module gives both times): wait about 5 s;
|
||||
// - **allowed, in force not known** (the module could not read a time): what is not known, said;
|
||||
// - **not allowed, or the module cannot answer**: the list check decides — the remedy that sends the list
|
||||
// when it is behind (busListState.remedy), and, for a module that cannot answer, held while the list sent
|
||||
// is current, as main judged it.
|
||||
//
|
||||
// ask answers the machine holding the bus and its module's answer ("" when nothing holds the bus); list is the
|
||||
// record's word on the list sent to that machine.
|
||||
func busGrantBy(ask func(ctx context.Context) (string, json.RawMessage, error),
|
||||
list func(ctx context.Context) (busListState, error)) grantJudge {
|
||||
return func(ctx context.Context) (bool, string, error) {
|
||||
node, raw, askErr := ask(ctx)
|
||||
if askErr == nil && node == "" {
|
||||
return false, "no module holds the bus, so nothing says what it enforces", nil
|
||||
}
|
||||
var answer busAnswer
|
||||
if askErr == nil {
|
||||
if answer, askErr = readBusAnswer(raw); askErr != nil {
|
||||
askErr = fmt.Errorf("its answer could not be read: %w", askErr)
|
||||
}
|
||||
}
|
||||
if askErr != nil {
|
||||
if node == "" {
|
||||
node = "the bus's machine"
|
||||
}
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the bus module on %s could not answer (%v), and whether the list sent "+
|
||||
"there is the one composed now could not be worked out: %w", node, askErr, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the bus module on %s could not answer (%v), and the user list sent there is "+
|
||||
"behind what the mesh composes; %s", node, askErr, s.remedy(node)), nil
|
||||
}
|
||||
return true, fmt.Sprintf("the bus module on %s could not answer (%v); judged by the user list sent there, "+
|
||||
"which is the one composed now", node, askErr), nil
|
||||
}
|
||||
notAllowed := func(older string) (bool, string, error) {
|
||||
s, err := list(ctx)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the user list on %s does not allow the controller to ask%s, and whether "+
|
||||
"the list sent there is the one composed now could not be worked out: %w", node, older, err)
|
||||
}
|
||||
if s.Behind {
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s: it is behind "+
|
||||
"what the mesh composes; %s", node, older, s.remedy(node)), nil
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s does not allow the controller to ask%s, though the list "+
|
||||
"sent there is the one composed now; %s again", node, older, s.remedy(node)), nil
|
||||
}
|
||||
switch {
|
||||
case answer.InForce == nil && answer.Allowed:
|
||||
return true, fmt.Sprintf("the user list on %s allows the controller to ask, but its reload is not "+
|
||||
"confirmed: the bus module there is older than the answer whether a grant is in force, and the bus "+
|
||||
"upgrade (`bus upgrade`) carries one that confirms it", node), nil
|
||||
case answer.InForce == nil:
|
||||
return notAllowed(" (the bus module there is older than the answer whether a grant is in force)")
|
||||
case !answer.Allowed:
|
||||
return notAllowed("")
|
||||
case *answer.InForce:
|
||||
return true, "", nil
|
||||
case answer.FileWritten != "" && answer.ServerLoaded != "":
|
||||
return false, fmt.Sprintf("the bus on %s has the controller's grant to ask written (%s) but has not "+
|
||||
"reloaded since (last load %s); it reloads in place within about 5 s, so nothing is to be done but wait",
|
||||
node, answer.FileWritten, answer.ServerLoaded), nil
|
||||
}
|
||||
why := answer.Why
|
||||
if why == "" {
|
||||
why = "it gives no reason"
|
||||
}
|
||||
return false, fmt.Sprintf("the user list on %s allows the controller to ask, but whether the bus enforces it "+
|
||||
"is not known: %s", node, why), nil
|
||||
}
|
||||
}
|
||||
|
||||
// busUserCanTool is the bus module's answer to whether a user may do something, and whether it is in force.
|
||||
const busUserCanTool = "nats_user_can"
|
||||
|
||||
// busAnswer is the bus module's answer to busUserCanTool; InForce is nil from a module older than that field,
|
||||
// and FileWritten and ServerLoaded are given only when it could read both times (mesh-catalog's nats InForce).
|
||||
type busAnswer struct {
|
||||
Allowed bool `json:"allowed"`
|
||||
InForce *bool `json:"in_force"`
|
||||
Why string `json:"why"`
|
||||
FileWritten string `json:"file_written"`
|
||||
ServerLoaded string `json:"server_loaded"`
|
||||
}
|
||||
|
||||
func readBusAnswer(raw json.RawMessage) (busAnswer, error) {
|
||||
var answer busAnswer
|
||||
err := json.Unmarshal(raw, &answer)
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// grantLookEvery is how often the bus's user list is judged against the one its machine was last sent.
|
||||
const grantLookEvery = 30 * time.Second
|
||||
|
||||
@@ -302,7 +546,19 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
||||
fmt.Printf("the operator cannot be asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
a := &asker{
|
||||
a := servingAsker(open, conn, js, keeper)
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
||||
// records, and the judgements it asks by — the bus's grant to ask among them (novox/hq issue 353).
|
||||
func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper *conditions.Keeper) *asker {
|
||||
return &asker{
|
||||
open: keeper.Open,
|
||||
silence: func(ctx context.Context, key string, d time.Duration, by, why string) error {
|
||||
_, err := keeper.Silence(ctx, key, d, by, why)
|
||||
@@ -322,7 +578,7 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
||||
},
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
channels: channelsIn(open.inventory),
|
||||
raise: func(ctx context.Context, obs []conditions.Observation) error {
|
||||
return keeper.Reconcile(ctx, sourceAsker, obs)
|
||||
@@ -330,10 +586,4 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
||||
now: time.Now,
|
||||
logf: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
|
||||
}
|
||||
if err := server.Decides(a); err != nil {
|
||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||
return
|
||||
}
|
||||
askerFrom = a
|
||||
go a.keep(ctx)
|
||||
}
|
||||
|
||||
@@ -226,7 +226,7 @@ func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(p inventory.Plan) { said = append(said, p) }
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -940,7 +941,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
@@ -958,7 +959,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||
}
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
|
||||
return err
|
||||
}); err != nil {
|
||||
@@ -973,7 +974,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
var p inventory.Plan
|
||||
var said stopAnswer
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without),
|
||||
time.Now().UTC())
|
||||
return err
|
||||
@@ -1215,15 +1216,19 @@ func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||
// comparison. Records arriving out of order are told apart by the plan's revision.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, p)
|
||||
//
|
||||
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
||||
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
||||
// the walk recorded.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, facts, p)
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
// Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are
|
||||
// read by whoever asks for the walk (`delivery walks`).
|
||||
p.Phases = p.WalkPhases(time.Now(), nil)
|
||||
body, err := json.Marshal(p)
|
||||
body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(ctx, facts, p)})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -1238,7 +1243,7 @@ func publishPlanMoved(ctx context.Context, bus link.Bus, p inventory.Plan) {
|
||||
// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept
|
||||
// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command
|
||||
// still runs; what it did is found by comparison.
|
||||
func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error {
|
||||
if checkEvents != nil {
|
||||
return f()
|
||||
}
|
||||
@@ -1249,9 +1254,10 @@ func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
return err
|
||||
}
|
||||
bus := link.OverNATS{Conn: conn, JS: js}
|
||||
was := inventory.PlanSaved
|
||||
checkEvents = bus
|
||||
inventory.PlanSaved = func(p inventory.Plan) { publishPlanMoved(ctx, bus, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, nil }()
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { publishPlanMoved(ctx, bus, facts, p) }
|
||||
defer func() { checkEvents, inventory.PlanSaved = nil, was }()
|
||||
ran = true
|
||||
return f()
|
||||
})
|
||||
@@ -1261,3 +1267,65 @@ func sayingOnTheBus(ctx context.Context, f func() error) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sayingWhatAVerbKept is PlanSaved for a process that is not the serving controller: each walk the verb keeps is
|
||||
// said as `plan-moved` before the save returns, with the facts of the inventory that kept it, on the bus the verb
|
||||
// reaches. It reaches the bus on the first save and never for a verb that keeps no walk. When the bus cannot be
|
||||
// reached, a save that started or ended a walk says so on the verb's error output: its mark is kept and no later
|
||||
// save says it, so the router holds what the walk touched until its bound, or not at all for a start. done lets
|
||||
// the bus go and takes the hook back.
|
||||
func sayingWhatAVerbKept(ctx context.Context) (said func(*inventory.Inventory, inventory.Plan), done func()) {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
reached bool
|
||||
bus link.Bus
|
||||
closing func()
|
||||
failed error
|
||||
)
|
||||
said = func(inv *inventory.Inventory, p inventory.Plan) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !reached {
|
||||
reached = true
|
||||
if handActConn != nil {
|
||||
bus, failed = overNATS(handActConn)
|
||||
} else if js, err := aBus(); err != nil {
|
||||
failed = err
|
||||
} else if bus, failed = overNATS(js.Conn()); failed == nil {
|
||||
closing = js.Close
|
||||
} else {
|
||||
js.Close()
|
||||
}
|
||||
}
|
||||
if bus == nil {
|
||||
switch p.WalkSaid {
|
||||
case inventory.WalkStarted:
|
||||
fmt.Fprintf(os.Stderr, "%s: its start is kept and could NOT be said as plan-moved (%v): the router "+
|
||||
"does not hold what it touches\n", p.ID, failed)
|
||||
case inventory.WalkEnded:
|
||||
fmt.Fprintf(os.Stderr, "%s: its end is kept and could NOT be said as plan-moved (%v): the router holds "+
|
||||
"what it touched until its bound\n", p.ID, failed)
|
||||
}
|
||||
return
|
||||
}
|
||||
publishPlanMoved(ctx, bus, inv, p)
|
||||
}
|
||||
done = func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
inventory.PlanSaved = nil
|
||||
if closing != nil {
|
||||
closing()
|
||||
}
|
||||
}
|
||||
return said, done
|
||||
}
|
||||
|
||||
// overNATS is the bus a connection carries, with its JetStream.
|
||||
func overNATS(conn *nats.Conn) (link.Bus, error) {
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.OverNATS{Conn: conn, JS: js}, nil
|
||||
}
|
||||
|
||||
@@ -119,6 +119,13 @@ func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
// A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353).
|
||||
if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) {
|
||||
t.Fatal("a list nobody could compose reads as current")
|
||||
}
|
||||
if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") {
|
||||
t.Fatal("listBehind does not read the digest as userListBehind does")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
|
||||
@@ -63,6 +63,15 @@ func run() error {
|
||||
// 45 §6), so the next controller takes it at once rather than after its age.
|
||||
defer theLease.release()
|
||||
|
||||
// Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save
|
||||
// that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its
|
||||
// own saves.
|
||||
if args[0] != "serve" {
|
||||
said, done := sayingWhatAVerbKept(ctx)
|
||||
inventory.PlanSaved = said
|
||||
defer done()
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
return buildCommand(ctx, args[1:])
|
||||
|
||||
@@ -251,6 +251,16 @@ func resumed(p *inventory.Plan, why string) {
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = why
|
||||
goesOnAgain(p)
|
||||
}
|
||||
}
|
||||
|
||||
// goesOnAgain clears a stop from a walk that walks again: a stop said why the walk ended, and a walk resumed has
|
||||
// not ended. Kept, it would end a retried walk that fails again as stopped, not failed (novox/hq ADR 0308
|
||||
// decision 1), and keep the search for the merge that brought the failure from running (searchAfterFailures).
|
||||
func goesOnAgain(p *inventory.Plan) {
|
||||
if p.Delivery != nil {
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = "", ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,6 +470,7 @@ func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string
|
||||
said = append(said, m+" to "+strings.Join(again, ", "))
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
goesOnAgain(p)
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
@@ -525,6 +536,7 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
||||
}
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
goesOnAgain(p)
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
|
||||
strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
|
||||
}
|
||||
|
||||
@@ -591,7 +591,7 @@ func proposeCommand(ctx context.Context, module, node, valuesArg string, clear,
|
||||
return err
|
||||
},
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
routerRecord: func(ctx context.Context, id string) (string, error) {
|
||||
bucket, err := asksRecords(ctx, open.inventory)
|
||||
if err != nil || bucket == "" {
|
||||
|
||||
+30
-14
@@ -249,7 +249,7 @@ func serve(ctx context.Context) (err error) {
|
||||
// (novox/hq to-be 45 §9).
|
||||
checkEvents = bus
|
||||
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
|
||||
inventory.PlanSaved = func(p inventory.Plan) { sayPlanMoved(ctx, bus, p) }
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) }
|
||||
if err := server.Checks(following{open}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1084,42 +1084,58 @@ func brokerFirst(names []string, holder string, behind bool) []string {
|
||||
// send anywhere, and a module running there always put it in its first wave. A digest of the list
|
||||
// last sent is kept for this (ADR 0043: the list is composed on each push, never kept itself).
|
||||
func brokerBehind(ctx context.Context, open *stores, names []string) (string, bool, error) {
|
||||
s, err := brokerListState(ctx, open, names)
|
||||
return s.Machine, s.Behind, err
|
||||
}
|
||||
|
||||
// brokerListState is brokerBehind with why the list could not be composed, when it could not: the plan's own
|
||||
// reason, which a remedy names rather than a push that would fail on it (the review of hq issue 353).
|
||||
func brokerListState(ctx context.Context, open *stores, names []string) (busListState, error) {
|
||||
inv := open.inventory
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
return busListState{}, err
|
||||
}
|
||||
h, held := holders[theBrokerSeat]
|
||||
if !held || h.Node == "" {
|
||||
return "", false, nil
|
||||
return busListState{}, nil
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
return busListState{}, err
|
||||
}
|
||||
if m, known := shelf[h.Module]; !known || m.BusUsers == "" {
|
||||
// A holder that is sent no user list carries no grant: nothing to send first.
|
||||
return "", false, nil
|
||||
return busListState{}, nil
|
||||
}
|
||||
for _, n := range names {
|
||||
if n == h.Node {
|
||||
return h.Node, false, nil
|
||||
return busListState{Machine: h.Node}, nil
|
||||
}
|
||||
}
|
||||
// **What cannot be worked out is behind** (the review of hq issue 353). The list it would be sent cannot be
|
||||
// composed, so whether it carries the grants a send relies on is not known; it is taken as behind, so a push
|
||||
// sends that machine first and says why it cannot (`plan` says it too), and a rollout that needs the list
|
||||
// refuses rather than sending code whose grants the bus may not hold. It said "not behind" until then: a
|
||||
// send went ahead as if the bus held a list nobody could compose.
|
||||
list, composeErr := "", error(nil)
|
||||
plan, _, err := planFor(ctx, open, h.Node)
|
||||
if err != nil {
|
||||
// It cannot be worked out: sending it would refuse the whole send, and `plan` says why.
|
||||
return h.Node, false, nil
|
||||
}
|
||||
list, _, err := busUserList(ctx, inv, plan.Modules)
|
||||
if err != nil {
|
||||
return h.Node, false, nil
|
||||
composeErr = err
|
||||
} else if list, _, err = busUserList(ctx, inv, plan.Modules); err != nil {
|
||||
composeErr = err
|
||||
}
|
||||
sent, err := inv.SentBusUsers(ctx, h.Node)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
return busListState{}, err
|
||||
}
|
||||
return h.Node, userListBehind(list, sent), nil
|
||||
return busListState{Machine: h.Node, Behind: listBehind(list, composeErr, sent), Uncomposed: composeErr}, nil
|
||||
}
|
||||
|
||||
// listBehind is whether the bus's machine is behind: the list composed now is not the one last sent, or it
|
||||
// could not be composed at all — not known is behind, never "not behind".
|
||||
func listBehind(list string, composeErr error, sentDigest string) bool {
|
||||
return composeErr != nil || userListBehind(list, sentDigest)
|
||||
}
|
||||
|
||||
// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
|
||||
|
||||
@@ -80,7 +80,7 @@ func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) {
|
||||
// A silence acts, so it is still its own process; it dials, and finds no such condition.
|
||||
_, _ = handlers["conditions"](context.Background(),
|
||||
json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`))
|
||||
names := closedNames(t, bus)
|
||||
names := closedNames(t, bus, "mesh-controller verb conditions")
|
||||
found := false
|
||||
for _, n := range names {
|
||||
found = found || n == "mesh-controller verb conditions"
|
||||
|
||||
@@ -812,14 +812,19 @@ func backlogCommand(ctx context.Context, sub string, args []string) error {
|
||||
}
|
||||
defer open.Close()
|
||||
if sub == "release-backlog" {
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
// Said on the bus (novox/hq ADR 0308 decision 1): the release plan starts walking here, and its start is
|
||||
// said by the verb that opens it.
|
||||
var p *inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, open.inventory, func() error {
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
why.record(ctx, "upgrade release-backlog", nil)
|
||||
p, err = releaseBacklog(ctx, open, link.Caller())
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
why.record(ctx, "upgrade release-backlog", nil)
|
||||
p, err := releaseBacklog(ctx, open, link.Caller())
|
||||
if err != nil {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if p == nil {
|
||||
|
||||
@@ -1559,7 +1559,7 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if len(positionals) == 2 && positionals[0] == "go" {
|
||||
why.record(ctx, "plans go", positionals[1:])
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, func() (err error) {
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
p, err = letGo(ctx, inv, positionals[1], "a person ("+whoAsked()+")", strings.TrimSpace(*why.why))
|
||||
return err
|
||||
}); err != nil {
|
||||
@@ -1571,8 +1571,13 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
// `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on.
|
||||
if len(positionals) == 2 && positionals[0] == "retry" {
|
||||
said, err := retryPlan(ctx, open, positionals[1])
|
||||
if err != nil {
|
||||
// Said on the bus (novox/hq ADR 0308 decision 1): a retried walk starts again, and its start is said
|
||||
// by the verb that starts it.
|
||||
var said string
|
||||
if err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
said, err = retryPlan(ctx, open, positionals[1])
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
@@ -1586,26 +1591,29 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if positionals[0] == "close" {
|
||||
how = "closed"
|
||||
}
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, positionals[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.Open() {
|
||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
// Said on the bus (novox/hq ADR 0308 decision 1): the walk's end, outcome stopped, is said by the verb
|
||||
// that ends it, so what it touched is not held until its bound.
|
||||
var p inventory.Plan
|
||||
if err := sayingOnTheBus(ctx, inv, func() error {
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
if p, err = inv.PlanByID(ctx, positionals[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.Open() {
|
||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||
stopByHand(&p, how, "a person ("+whoAsked()+")", strings.TrimSpace(*why.why))
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
})
|
||||
return inv.SavePlan(ctx, &p)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n",
|
||||
@@ -1924,3 +1932,15 @@ func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string,
|
||||
"`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error()))
|
||||
}
|
||||
}
|
||||
|
||||
// stopByHand ends a walk a person stopped or closed (`plans stop|close`): failed, with who stopped it and why
|
||||
// kept as the delivery's owner keeps its own stop, so the walk ends with outcome stopped, not failed (novox/hq
|
||||
// ADR 0308 decision 1).
|
||||
func stopByHand(p *inventory.Plan, how, by, why string) {
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + why
|
||||
if p.Delivery == nil {
|
||||
p.Delivery = &inventory.PlanDelivery{}
|
||||
}
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||
}
|
||||
|
||||
@@ -84,8 +84,12 @@ func asAProcess(t *testing.T, url string) {
|
||||
t.Setenv(broker.CertificateVar, "")
|
||||
}
|
||||
|
||||
// closedNames are the names of the connections the bus saw closed.
|
||||
func closedNames(t *testing.T, bus *server.Server) []string {
|
||||
// closedNames are the names of the connections the bus saw closed, waited for by want's own name: the
|
||||
// bus can register another connection's close first (the test's own setup connection, say), so returning
|
||||
// on the first non-empty list names whichever closed first, not necessarily the one being waited for
|
||||
// (novox/hq issue 543). closedNames instead waits until want itself appears among the closed connections,
|
||||
// or the deadline passes, and returns whatever names it last saw either way.
|
||||
func closedNames(t *testing.T, bus *server.Server, want string) []string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var names []string
|
||||
@@ -95,10 +99,14 @@ func closedNames(t *testing.T, bus *server.Server) []string {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names = names[:0]
|
||||
found := false
|
||||
for _, c := range connz.Conns {
|
||||
names = append(names, c.Name)
|
||||
if c.Name == want {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
if found {
|
||||
return names
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
func serviceManagerHolder() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "systemd", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
|
||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
|
||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal",
|
||||
"reset-failed", "wanted-by", "unlink-dangling"}}},
|
||||
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a walk touches, its bound and its outcome (novox/hq ADR 0308 decision 1, decision 5): said in
|
||||
// `plan-moved` on the save that starts the walk and the save that ends it, beside the walk whole, for the holder
|
||||
// of the operator's channel to hold what the walk touches while it runs and tell its result once.
|
||||
//
|
||||
// **What a walk touches** is every module it moves on every machine it sends it to, and every module a moved
|
||||
// tool runner (node-tools) or node-engine (mesh-host) carries on that machine: what restarts there. At the start
|
||||
// that is what the walk will send — each module of its tiers whose policy rolls it out, on every machine running
|
||||
// it; at the end it is what it reached — the machines each module was sent to, first and rest, and what a gated
|
||||
// send carried with it.
|
||||
//
|
||||
// **Its bound** is twice the largest delivery budget among the walk's deliveries (deliveryBudgets: a core
|
||||
// module's ten minutes, a leaf module's budget in force), and fifteen minutes, not doubled, for a walk none of
|
||||
// whose deliveries has a budget: a release plan (builds waiting for a person's gate), a walk whose every module's
|
||||
// policy records rather than rolls out (held for a person, ADR 0282 decision 1), or a walk kept without a class.
|
||||
|
||||
// walkNoBudgetBound is the bound of a walk with no budget (ADR 0308 decision 5): not doubled.
|
||||
const walkNoBudgetBound = 15 * time.Minute
|
||||
|
||||
// budgetNone is the budget class of a walk none of whose deliveries has a budget.
|
||||
const budgetNone = "none"
|
||||
|
||||
// walkSaid is what `plan-moved` adds on a walk's start and end, beside the walk whole.
|
||||
type walkSaid struct {
|
||||
Walk string `json:"walk"`
|
||||
Phase string `json:"phase"`
|
||||
// Touches is, per machine, the modules the walk moves or reaches there and what a moved tool runner or
|
||||
// node-engine carries there; never null.
|
||||
Touches []walkTouch `json:"touches"`
|
||||
// Outcome is on the end only: delivered, failed or stopped.
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
// BoundSeconds and BudgetClass are said on both, read from the walk's class at the start.
|
||||
BoundSeconds int `json:"bound_seconds"`
|
||||
BudgetClass string `json:"budget_class"`
|
||||
// StartedAt is when the walk was said started, on both; EndedAt when it was said ended, on the end: the
|
||||
// bound runs from StartedAt, whenever the event is heard.
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
EndedAt *time.Time `json:"ended_at,omitempty"`
|
||||
}
|
||||
|
||||
// walkTouch is what a walk touches on one machine.
|
||||
type walkTouch struct {
|
||||
Machine string `json:"machine"`
|
||||
Modules []string `json:"modules"`
|
||||
// Carried names those of Modules that are there only because a moved tool runner or node-engine carries
|
||||
// them: what restarts with it, not what the walk builds.
|
||||
Carried []string `json:"carried,omitempty"`
|
||||
}
|
||||
|
||||
// planMovedBody is `plan-moved`'s body: the walk whole, as before, and on its start and end what it touches.
|
||||
type planMovedBody struct {
|
||||
inventory.Plan
|
||||
*walkSaid
|
||||
}
|
||||
|
||||
// walkFacts is what the touches are read from: the controller's inventory.
|
||||
type walkFacts interface {
|
||||
Running(ctx context.Context, module string) ([]string, error)
|
||||
UpgradeOf(ctx context.Context, module string) (inventory.Upgrade, error)
|
||||
Assigned(ctx context.Context, node string) ([]string, error)
|
||||
SentBuilds(ctx context.Context, node string) (map[string]string, bool, error)
|
||||
Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error)
|
||||
}
|
||||
|
||||
// sayWalk is what `plan-moved` adds to the save that started or ended the walk; nil on every other save. A fact
|
||||
// that cannot be read is said on the controller's log and left out, never in the way of the event: a walk said
|
||||
// with fewer touches is held less, never longer.
|
||||
func sayWalk(ctx context.Context, facts walkFacts, p inventory.Plan) *walkSaid {
|
||||
if p.WalkSaid == "" {
|
||||
return nil
|
||||
}
|
||||
w := &walkSaid{Walk: p.ID, Phase: p.WalkSaid}
|
||||
if p.Times != nil {
|
||||
w.StartedAt = p.Times.Started
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
w.EndedAt, w.Outcome = p.Times.Ended, p.Outcome()
|
||||
}
|
||||
}
|
||||
read := readWalkFacts(ctx, facts, p)
|
||||
w.BudgetClass = walkBudgetClass(p, read.rollsOut)
|
||||
w.BoundSeconds = int(walkBound(w.BudgetClass) / time.Second)
|
||||
var moved map[string][]string
|
||||
if p.WalkSaid == inventory.WalkStarted {
|
||||
moved = walkSends(p, read.running, read.rollsOut)
|
||||
} else {
|
||||
moved = walkReached(p, read.running)
|
||||
}
|
||||
var shelf map[string]catalogue.Manifest
|
||||
if len(moved[broker.RuntimeModule]) > 0 {
|
||||
// Which modules the tool runner serves is the catalogue's: read only when the walk moves it.
|
||||
shelf = read.shelf()
|
||||
}
|
||||
w.Touches = touchesOf(moved, read.onMachine(moved), shelf)
|
||||
return w
|
||||
}
|
||||
|
||||
// walkBudgetClass is the largest budget class among the walk's deliveries: its class read at the cut when one of
|
||||
// its modules rolls out, and none when it has no budget.
|
||||
func walkBudgetClass(p inventory.Plan, rollsOut func(string) bool) string {
|
||||
if p.Release != nil || p.Times == nil {
|
||||
return budgetNone
|
||||
}
|
||||
if _, budgeted := deliveryBudgets[p.Times.Class]; !budgeted {
|
||||
return budgetNone
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
if rollsOut(m) {
|
||||
return p.Times.Class
|
||||
}
|
||||
}
|
||||
}
|
||||
return budgetNone
|
||||
}
|
||||
|
||||
// walkBound is a walk's bound by its budget class: twice its budget, or fifteen minutes with none.
|
||||
func walkBound(class string) time.Duration {
|
||||
if b, ok := deliveryBudgets[class]; ok {
|
||||
return 2 * b
|
||||
}
|
||||
return walkNoBudgetBound
|
||||
}
|
||||
|
||||
// walkSends is what a walk will send at its start, per module the machines it goes to: each module of its tiers
|
||||
// whose policy rolls it out, on every machine running it (for a release plan, those of its order).
|
||||
func walkSends(p inventory.Plan, running func(string) []string, rollsOut func(string) bool) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s != nil && s.State == planDeleted {
|
||||
continue
|
||||
}
|
||||
if p.Release == nil && !rollsOut(m) {
|
||||
continue
|
||||
}
|
||||
for _, n := range running(m) {
|
||||
if p.Release != nil && !slices.Contains(p.Release.Order, n) {
|
||||
continue
|
||||
}
|
||||
out[m] = appendNew(out[m], n)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// walkReached is what a walk sent by its end, per module the machines it reached: its first machines, its rest,
|
||||
// and every module a gated send carried with it. A release plan reached the machines it walked: those done and
|
||||
// the one being judged.
|
||||
func walkReached(p inventory.Plan, running func(string) []string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
carried := func(g *inventory.PlanGate) {
|
||||
if g == nil {
|
||||
return
|
||||
}
|
||||
for _, c := range g.Carried {
|
||||
out[c.Module] = appendNew(out[c.Module], c.Node)
|
||||
}
|
||||
}
|
||||
if r := p.Release; r != nil {
|
||||
reached := append([]string{}, r.Done...)
|
||||
if r.Gate != nil {
|
||||
reached = appendNew(reached, r.Gate.Machines...)
|
||||
carried(r.Gate)
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
for _, n := range running(m) {
|
||||
if slices.Contains(reached, n) {
|
||||
out[m] = appendNew(out[m], n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
for m, s := range p.Modules {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
out[m] = appendNew(out[m], s.First...)
|
||||
for n := range s.Rest {
|
||||
out[m] = appendNew(out[m], n)
|
||||
}
|
||||
carried(s.Gate)
|
||||
if len(out[m]) == 0 {
|
||||
delete(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// touchesOf turns what a walk moves, per module, into what it touches, per machine: the modules it moves there,
|
||||
// and, where it moves the tool runner, every module there whose tools it serves, and where it moves the
|
||||
// node-engine, every module there. Sorted by machine and module.
|
||||
func touchesOf(moved map[string][]string, on map[string][]string, shelf map[string]catalogue.Manifest) []walkTouch {
|
||||
byMachine := map[string][]string{}
|
||||
for m, machines := range moved {
|
||||
for _, n := range machines {
|
||||
byMachine[n] = appendNew(byMachine[n], m)
|
||||
}
|
||||
}
|
||||
out := make([]walkTouch, 0, len(byMachine))
|
||||
for n, modules := range byMachine {
|
||||
t := walkTouch{Machine: n, Modules: modules}
|
||||
engine, runner := slices.Contains(modules, hostModule), slices.Contains(modules, broker.RuntimeModule)
|
||||
if engine || runner {
|
||||
for _, m := range on[n] {
|
||||
if slices.Contains(t.Modules, m) || (!engine && !servesTools(shelf[m])) {
|
||||
continue
|
||||
}
|
||||
t.Modules = append(t.Modules, m)
|
||||
t.Carried = append(t.Carried, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(t.Modules)
|
||||
sort.Strings(t.Carried)
|
||||
out = append(out, t)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine })
|
||||
return out
|
||||
}
|
||||
|
||||
// servesTools says the tool runner serves something of this module: a bundle it loads entrypoints from, or the
|
||||
// module's own tools.
|
||||
func servesTools(m catalogue.Manifest) bool {
|
||||
if len(m.Tools) > 0 {
|
||||
return true
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// walkRead is the facts a walk's touches are worked out from, read once per event.
|
||||
type walkRead struct {
|
||||
running func(string) []string
|
||||
rollsOut func(string) bool
|
||||
onMachine func(moved map[string][]string) map[string][]string
|
||||
shelf func() map[string]catalogue.Manifest
|
||||
}
|
||||
|
||||
// readWalkFacts reads what the walk's touches need. Without facts (a test, a command with no store), nothing runs
|
||||
// anywhere and every module rolls out: the touches are then what the walk itself recorded.
|
||||
func readWalkFacts(ctx context.Context, facts walkFacts, p inventory.Plan) walkRead {
|
||||
cantRead := func(what string, err error) {
|
||||
fmt.Fprintf(os.Stderr, "%s: %s cannot be read for what the walk touches: %v\n", p.ID, what, err)
|
||||
}
|
||||
r := walkRead{
|
||||
running: func(string) []string { return nil },
|
||||
rollsOut: func(string) bool { return true },
|
||||
onMachine: func(map[string][]string) map[string][]string { return nil },
|
||||
shelf: func() map[string]catalogue.Manifest { return nil },
|
||||
}
|
||||
if facts == nil {
|
||||
return r
|
||||
}
|
||||
running := map[string][]string{}
|
||||
r.running = func(m string) []string {
|
||||
if on, read := running[m]; read {
|
||||
return on
|
||||
}
|
||||
on, err := facts.Running(ctx, m)
|
||||
if err != nil {
|
||||
cantRead("where "+m+" runs", err)
|
||||
}
|
||||
running[m] = on
|
||||
return on
|
||||
}
|
||||
rolls := map[string]bool{}
|
||||
r.rollsOut = func(m string) bool {
|
||||
if yes, read := rolls[m]; read {
|
||||
return yes
|
||||
}
|
||||
u, err := facts.UpgradeOf(ctx, m)
|
||||
if err != nil {
|
||||
cantRead(m+"'s policy", err)
|
||||
}
|
||||
// A policy that cannot be read is taken to roll out: the walk is then held more, never less.
|
||||
rolls[m] = err != nil || u.RollOut
|
||||
return rolls[m]
|
||||
}
|
||||
r.onMachine = func(moved map[string][]string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, core := range []string{hostModule, broker.RuntimeModule} {
|
||||
for _, n := range moved[core] {
|
||||
if _, read := out[n]; read {
|
||||
continue
|
||||
}
|
||||
var there []string
|
||||
if sent, known, err := facts.SentBuilds(ctx, n); err != nil {
|
||||
cantRead("what "+n+" was sent", err)
|
||||
} else if known {
|
||||
for m := range sent {
|
||||
there = append(there, m)
|
||||
}
|
||||
}
|
||||
if assigned, err := facts.Assigned(ctx, n); err != nil {
|
||||
cantRead("what "+n+" is assigned", err)
|
||||
} else {
|
||||
there = appendNew(there, assigned...)
|
||||
}
|
||||
sort.Strings(there)
|
||||
out[n] = there
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
r.shelf = func() map[string]catalogue.Manifest {
|
||||
shelf, err := facts.Catalogue(ctx)
|
||||
if err != nil {
|
||||
cantRead("the catalogue", err)
|
||||
}
|
||||
return shelf
|
||||
}
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// fakeWalkFacts is the inventory as far as a walk's touches read it.
|
||||
type fakeWalkFacts struct {
|
||||
running map[string][]string
|
||||
records map[string]bool // modules whose policy records rather than rolls out
|
||||
sent map[string]map[string]string
|
||||
assigned map[string][]string
|
||||
shelf map[string]catalogue.Manifest
|
||||
}
|
||||
|
||||
func (f fakeWalkFacts) Running(_ context.Context, m string) ([]string, error) {
|
||||
return f.running[m], nil
|
||||
}
|
||||
func (f fakeWalkFacts) UpgradeOf(_ context.Context, m string) (inventory.Upgrade, error) {
|
||||
return inventory.Upgrade{RollOut: !f.records[m]}, nil
|
||||
}
|
||||
func (f fakeWalkFacts) Assigned(_ context.Context, n string) ([]string, error) {
|
||||
return f.assigned[n], nil
|
||||
}
|
||||
func (f fakeWalkFacts) SentBuilds(_ context.Context, n string) (map[string]string, bool, error) {
|
||||
s, ok := f.sent[n]
|
||||
return s, ok, nil
|
||||
}
|
||||
func (f fakeWalkFacts) Catalogue(context.Context) (map[string]catalogue.Manifest, error) {
|
||||
return f.shelf, nil
|
||||
}
|
||||
|
||||
// theMesh is four machines: the tool runner on each, messenger and mesh-issues on novox, gitea on ace.
|
||||
func theMesh() fakeWalkFacts {
|
||||
return fakeWalkFacts{
|
||||
running: map[string][]string{
|
||||
broker.RuntimeModule: {"ace", "g14", "novox", "shanks"},
|
||||
"messenger": {"novox"},
|
||||
"gitea": {"ace"},
|
||||
"plex": {"ace"},
|
||||
},
|
||||
sent: map[string]map[string]string{
|
||||
"novox": {broker.RuntimeModule: "c0", "messenger": "c1", "mesh-issues": "c2", "postgres": "c3"},
|
||||
"ace": {broker.RuntimeModule: "c0", "gitea": "c4"},
|
||||
},
|
||||
assigned: map[string][]string{"g14": {broker.RuntimeModule, "desktop"}, "shanks": {broker.RuntimeModule}},
|
||||
shelf: map[string]catalogue.Manifest{
|
||||
"messenger": {Module: "messenger", Tools: []string{"notify"}},
|
||||
"mesh-issues": {Module: "mesh-issues", Bundles: []catalogue.Bundle{{Name: "tools", Loads: []string{"main"}}}},
|
||||
"postgres": {Module: "postgres"},
|
||||
"gitea": {Module: "gitea", Tools: []string{"gitea_api"}},
|
||||
"desktop": {Module: "desktop", Tools: []string{"notify_send"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func walkOfTiers(class string, tiers ...[]string) inventory.Plan {
|
||||
cut := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC)
|
||||
started := cut.Add(time.Second)
|
||||
p := inventory.Plan{ID: "plan-308", State: inventory.PlanBuilding, Tiers: tiers,
|
||||
Modules: map[string]*inventory.PlanModule{}, Times: &inventory.PlanTimes{Cut: &cut, Class: class, Started: &started}}
|
||||
for _, tier := range tiers {
|
||||
for _, m := range tier {
|
||||
p.Modules[m] = &inventory.PlanModule{}
|
||||
}
|
||||
}
|
||||
p.WalkSaid = inventory.WalkStarted
|
||||
return p
|
||||
}
|
||||
|
||||
// novox/hq ADR 0308 How it is checked §1: the start names the modules the walk moves and the machines it sends
|
||||
// them to, and every module a moved tool runner carries there.
|
||||
func TestAWalksStartNamesWhatItTouchesWithWhatTheToolRunnerCarries(t *testing.T) {
|
||||
p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"})
|
||||
w := sayWalk(t.Context(), theMesh(), p)
|
||||
if w == nil || w.Walk != "plan-308" || w.Phase != inventory.WalkStarted || w.Outcome != "" {
|
||||
t.Fatalf("the start is said: %+v", w)
|
||||
}
|
||||
want := []walkTouch{
|
||||
{Machine: "ace", Modules: []string{"gitea", broker.RuntimeModule}, Carried: []string{"gitea"}},
|
||||
{Machine: "g14", Modules: []string{"desktop", broker.RuntimeModule}, Carried: []string{"desktop"}},
|
||||
// postgres serves no tools: the tool runner does not carry it.
|
||||
{Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule}, Carried: []string{"mesh-issues"}},
|
||||
{Machine: "shanks", Modules: []string{broker.RuntimeModule}},
|
||||
}
|
||||
if !reflect.DeepEqual(w.Touches, want) {
|
||||
t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want)
|
||||
}
|
||||
if w.BudgetClass != inventory.ClassCore || w.BoundSeconds != 1200 || w.StartedAt == nil {
|
||||
t.Fatalf("a core walk is bound at 20 minutes: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// A moved node-engine carries every module on its machine; a module whose policy records is not sent, and not
|
||||
// touched.
|
||||
func TestAMovedNodeEngineCarriesEverythingOnItsMachine(t *testing.T) {
|
||||
f := theMesh()
|
||||
f.running[hostModule] = []string{"novox"}
|
||||
f.records = map[string]bool{"plex": true}
|
||||
p := walkOfTiers(inventory.ClassCore, []string{hostModule, "plex"})
|
||||
w := sayWalk(t.Context(), f, p)
|
||||
want := []walkTouch{{Machine: "novox",
|
||||
Modules: []string{"mesh-host", "mesh-issues", "messenger", "node-tools", "postgres"},
|
||||
Carried: []string{"mesh-issues", "messenger", "node-tools", "postgres"}}}
|
||||
if !reflect.DeepEqual(w.Touches, want) {
|
||||
t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The end names what the walk reached — its first machines, its rest, what a gated send carried — and its
|
||||
// outcome; the modules a reached tool runner carries are touched there too.
|
||||
func TestAWalksEndNamesWhatItReachedAndItsOutcome(t *testing.T) {
|
||||
p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"})
|
||||
ended := p.Times.Started.Add(4 * time.Minute)
|
||||
p.Times.Ended = &ended
|
||||
p.State, p.WalkSaid = inventory.PlanDone, inventory.WalkEnded
|
||||
p.Modules[broker.RuntimeModule] = &inventory.PlanModule{First: []string{"shanks"},
|
||||
Rest: map[string]inventory.SentDeclaration{"novox": {}},
|
||||
Gate: &inventory.PlanGate{Carried: []inventory.CarriedMove{{Module: "gitea", Node: "shanks"}}}}
|
||||
// messenger was built and found unchanged: no move, not reached.
|
||||
p.Modules["messenger"] = &inventory.PlanModule{Why: "no move"}
|
||||
w := sayWalk(t.Context(), theMesh(), p)
|
||||
if w.Phase != inventory.WalkEnded || w.Outcome != inventory.WalkDelivered || w.EndedAt == nil {
|
||||
t.Fatalf("the end is said with its outcome: %+v", w)
|
||||
}
|
||||
want := []walkTouch{
|
||||
{Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule},
|
||||
Carried: []string{"mesh-issues", "messenger"}},
|
||||
{Machine: "shanks", Modules: []string{"gitea", broker.RuntimeModule}},
|
||||
}
|
||||
if !reflect.DeepEqual(w.Touches, want) {
|
||||
t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want)
|
||||
}
|
||||
for outcome, end := range map[string]func(*inventory.Plan){
|
||||
inventory.WalkFailed: func(p *inventory.Plan) { p.State = inventory.PlanFailed },
|
||||
inventory.WalkStopped: func(p *inventory.Plan) {
|
||||
p.State, p.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
|
||||
},
|
||||
} {
|
||||
q := p
|
||||
end(&q)
|
||||
if got := sayWalk(t.Context(), theMesh(), q).Outcome; got != outcome {
|
||||
t.Errorf("outcome %q, want %q", got, outcome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0308 How it is checked §1 and decision 5: the bound is twice the largest budget in force among
|
||||
// the walk's deliveries — 20 minutes for a core walk, twice the leaf budget for a leaf walk — and 15 minutes, not
|
||||
// doubled, for a walk none of whose deliveries has a budget.
|
||||
func TestAWalksBoundIsTwiceItsBudgetOrFifteenMinutesWithNone(t *testing.T) {
|
||||
f := theMesh()
|
||||
f.records = map[string]bool{"plex": true}
|
||||
release := walkOfTiers("", []string{"gitea"})
|
||||
release.Times, release.Release = &inventory.PlanTimes{}, &inventory.PlanRelease{Order: []string{"ace"}}
|
||||
for name, c := range map[string]struct {
|
||||
p inventory.Plan
|
||||
class string
|
||||
bound int
|
||||
}{
|
||||
"core": {walkOfTiers(inventory.ClassCore, []string{"messenger"}), inventory.ClassCore, 1200},
|
||||
"leaf": {walkOfTiers(inventory.ClassLeaf, []string{"messenger"}), inventory.ClassLeaf, int(2 * deliveryBudgets[inventory.ClassLeaf] / time.Second)},
|
||||
"held for a person": {walkOfTiers(inventory.ClassLeaf, []string{"plex"}), budgetNone, 900},
|
||||
"a release": {release, budgetNone, 900},
|
||||
"kept without a class": {walkOfTiers("", []string{"messenger"}), budgetNone, 900},
|
||||
} {
|
||||
w := sayWalk(t.Context(), f, c.p)
|
||||
if w.BudgetClass != c.class || w.BoundSeconds != c.bound {
|
||||
t.Errorf("%s: %s bound %ds, want %s %ds", name, w.BudgetClass, w.BoundSeconds, c.class, c.bound)
|
||||
}
|
||||
}
|
||||
// The leaf budget in force is three minutes (ADR 0304 decision 4): a leaf walk is bound at six.
|
||||
if deliveryBudgets[inventory.ClassLeaf] == 3*time.Minute {
|
||||
if w := sayWalk(t.Context(), f, walkOfTiers(inventory.ClassLeaf, []string{"messenger"})); w.BoundSeconds != 360 {
|
||||
t.Errorf("a leaf walk at a three-minute budget is bound at 360 s: %d", w.BoundSeconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// plan-moved's body: the walk whole as before, and on its start the walk's fields beside it; on any other save
|
||||
// none of them.
|
||||
func TestPlanMovedCarriesTheWalkWholeAndOnItsStartWhatItTouches(t *testing.T) {
|
||||
p := walkOfTiers(inventory.ClassLeaf, []string{"messenger"})
|
||||
body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said map[string]any
|
||||
if err := json.Unmarshal(body, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"id", "state", "tiers", "modules", "times", "walk", "phase", "touches", "bound_seconds", "budget_class", "started_at"} {
|
||||
if _, ok := said[k]; !ok {
|
||||
t.Errorf("plan-moved lacks %q: %s", k, body)
|
||||
}
|
||||
}
|
||||
if _, ok := said["outcome"]; ok {
|
||||
t.Errorf("a start says no outcome: %s", body)
|
||||
}
|
||||
if said["walk"] != "plan-308" || said["phase"] != "started" || said["budget_class"] != "leaf" {
|
||||
t.Errorf("the walk's fields: %s", body)
|
||||
}
|
||||
p.WalkSaid = ""
|
||||
body, _ = json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)})
|
||||
said = map[string]any{}
|
||||
_ = json.Unmarshal(body, &said)
|
||||
for _, k := range []string{"walk", "phase", "touches", "bound_seconds"} {
|
||||
if _, ok := said[k]; ok {
|
||||
t.Errorf("a save neither starting nor ending the walk says %q: %s", k, body)
|
||||
}
|
||||
}
|
||||
// A walk that touches nothing says an empty list, never null.
|
||||
q := walkOfTiers(inventory.ClassLeaf, []string{"nobody-runs-this"})
|
||||
body, _ = json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), theMesh(), q)})
|
||||
said = map[string]any{}
|
||||
_ = json.Unmarshal(body, &said)
|
||||
if touches, ok := said["touches"].([]any); !ok || len(touches) != 0 {
|
||||
t.Errorf("touches nothing: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store and a bus: a walk let go by the delivery's verb is said started with what it touches and
|
||||
// its bound, and stopped by the verb is said ended, stopped (novox/hq ADR 0308 How it is checked §1).
|
||||
func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
next := planMovedHeard(t)
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cut := time.Now().UTC()
|
||||
waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8",
|
||||
Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat},
|
||||
Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := next(inventory.WalkStarted)
|
||||
touches, _ := json.Marshal(started["touches"])
|
||||
if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` ||
|
||||
started["budget_class"] != inventory.ClassLeaf ||
|
||||
started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) {
|
||||
t.Fatalf("the start: %v", started)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended := next(inventory.WalkEnded)
|
||||
touches, _ = json.Marshal(ended["touches"])
|
||||
if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` {
|
||||
t.Fatalf("the end, which sent nothing: %v", ended)
|
||||
}
|
||||
}
|
||||
|
||||
// planMovedHeard raises a bus of the test's own, points the verbs' hand-act connection at it, and returns what
|
||||
// waits for the next plan-moved said with the given phase, failing the test when none comes within 5 s.
|
||||
func planMovedHeard(t *testing.T) func(phase string) map[string]any {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 64)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
return func(phase string) map[string]any {
|
||||
t.Helper()
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said map[string]any
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said["phase"] == phase {
|
||||
return said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("no walk was said %s", phase)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store and a bus: a walk a person stops with `plans stop` is said ended, outcome stopped, by the
|
||||
// verb itself, and is kept stopped by that person; `plans close` ends one the same way (novox/hq ADR 0308
|
||||
// decision 1, review 15 on mesh-controller#247).
|
||||
func TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
next := planMovedHeard(t)
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, verb := range []string{"stop", "close"} {
|
||||
cut := time.Now().UTC()
|
||||
walk := inventory.Plan{ID: "plan-308-" + verb, Repository: "novox/mesh-catalog", Branch: "main",
|
||||
Commit: "c8c8c8c8", Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Times: &inventory.PlanTimes{Cut: &cut, Started: &cut, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{verb, walk.ID, "--why", "a test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended := next(inventory.WalkEnded)
|
||||
if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkStopped {
|
||||
t.Fatalf("plans %s: the end is %v", verb, ended)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, walk.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Outcome() != inventory.WalkStopped || p.Delivery == nil || p.Delivery.StoppedWhy != "a test" {
|
||||
t.Fatalf("plans %s kept the walk %s, %+v", verb, p.Outcome(), p.Delivery)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store and a bus: a failed walk retried with `plans retry` starts again, and the verb says its
|
||||
// start (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247).
|
||||
func TestAWalkRetriedByAPersonIsSaidStartedAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asksRecorded(t)
|
||||
next := planMovedHeard(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
ended := before.Add(time.Minute)
|
||||
failed := inventory.Plan{ID: "plan-308-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}},
|
||||
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"retry", failed.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := next(inventory.WalkStarted)
|
||||
if started["walk"] != failed.ID || started["budget_class"] != inventory.ClassLeaf {
|
||||
t.Fatalf("the retried start: %v", started)
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store and a bus: the release plan `upgrade release-backlog` opens is said started by the verb
|
||||
// (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247).
|
||||
func TestABacklogReleasedByAPersonIsSaidStarted(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
next := planMovedHeard(t)
|
||||
if err := backlogCommand(t.Context(), "release-backlog", []string{"--why", "a test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := next(inventory.WalkStarted)
|
||||
if p := b.release(t); started["walk"] != p.ID {
|
||||
t.Fatalf("the release's start: %v", started)
|
||||
}
|
||||
}
|
||||
|
||||
// asTyped runs a verb as the controller's command line runs it, through run, as a person's terminal would.
|
||||
func asTyped(t *testing.T, args ...string) error {
|
||||
t.Helper()
|
||||
was := os.Args
|
||||
os.Args = append([]string{"mesh-controller"}, args...)
|
||||
t.Cleanup(func() { os.Args = was })
|
||||
err := run()
|
||||
// run gives back the lease as the process would end; the test's process goes on, and takes it again.
|
||||
theLease.mu.Lock()
|
||||
theLease.held, theLease.kv, theLease.noBus = nil, nil, false
|
||||
theLease.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
// aStartedWalkAsking keeps a live walk, said started, whose first tier asked a as build.
|
||||
func aStartedWalkAsking(t *testing.T, open *stores, id, build string, asked time.Time) inventory.Plan {
|
||||
t.Helper()
|
||||
walk := inventory.Plan{ID: id, Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: build}},
|
||||
Times: &inventory.PlanTimes{Cut: &asked, Started: &asked, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(t.Context(), &walk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return walk
|
||||
}
|
||||
|
||||
// Live, with the store and a bus: `cancel <build>` and `clear` fail the walk that asked the build, and the verb
|
||||
// says its end, failed (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247).
|
||||
func TestAWalkEndedByCancelOrClearIsSaidEnded(t *testing.T) {
|
||||
_, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
next := planMovedHeard(t)
|
||||
twoTiers(t, open)
|
||||
for _, verb := range [][]string{{"cancel"}, {"clear"}} {
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
asked, _ := link.BuildAskedAt(id)
|
||||
walk := aStartedWalkAsking(t, open, "plan-308-"+verb[0], id, asked)
|
||||
args := verb
|
||||
if verb[0] == "cancel" {
|
||||
args = append(args, id)
|
||||
}
|
||||
if err := asTyped(t, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended := next(inventory.WalkEnded)
|
||||
if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkFailed {
|
||||
t.Fatalf("%s: the end is %v", verb[0], ended)
|
||||
}
|
||||
// Ended, and so the next walk may be cut on the same repository.
|
||||
p, _ := open.inventory.PlanByID(t.Context(), walk.ID)
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("%s left the walk %s", verb[0], p.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store and a bus: `rebuild <module>` joining a failed walk sets it building again, and the verb
|
||||
// says its start (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247).
|
||||
func TestAWalkRestartedByARebuildIsSaidStarted(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
asksRecorded(t)
|
||||
next := planMovedHeard(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
ended := before.Add(time.Minute)
|
||||
failed := inventory.Plan{ID: "plan-308-rebuild", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.CancelledByHand}},
|
||||
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(t.Context(), &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asTyped(t, "rebuild", "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if started := next(inventory.WalkStarted); started["walk"] != failed.ID {
|
||||
t.Fatalf("the rebuilt walk's start: %v", started)
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store: a walk a person stopped and then retried, by `plans retry` or by `rebuild`, walks again
|
||||
// without the stop, so when it fails again it ends failed, not stopped, and the search for the merge that
|
||||
// brought the failure is not kept from it (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247).
|
||||
func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
for _, again := range [][]string{{"plans", "retry"}, {"rebuild", "a"}} {
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before)
|
||||
if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The build asked before the stop fails after it.
|
||||
p, _ := open.inventory.PlanByID(ctx, walk.ID)
|
||||
p.Modules["a"].State, p.Modules["a"].Why = "failed", "it broke"
|
||||
if err := open.inventory.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := again
|
||||
if again[0] == "plans" {
|
||||
args = append(append([]string(nil), again...), walk.ID)
|
||||
}
|
||||
if err := asTyped(t, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, _ = open.inventory.PlanByID(ctx, walk.ID)
|
||||
if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") {
|
||||
t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery)
|
||||
}
|
||||
asking, _ := link.BuildAskedAt(p.Modules["a"].Build)
|
||||
planBuilt(ctx, open, "a", "c0ffee", "it broke again", asking, p.Modules["a"].Build)
|
||||
if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed {
|
||||
t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,9 +68,10 @@ func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) {
|
||||
holder := func(serves ...string) Manifest {
|
||||
return Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode, Serves: serves}}}
|
||||
}
|
||||
eight := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}
|
||||
eight := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal",
|
||||
"reset-failed", "wanted-by", "unlink-dangling"}
|
||||
if err := CanHold(holder(eight...), seat); err != nil {
|
||||
t.Fatalf("today's holder, without failed, is refused: %v", err)
|
||||
t.Fatalf("a holder without failed is refused: %v", err)
|
||||
}
|
||||
if err := CanHold(holder(append(eight, "failed")...), seat); err != nil {
|
||||
t.Fatalf("a holder serving failed is refused: %v", err)
|
||||
@@ -81,7 +82,7 @@ func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) {
|
||||
if err := CanHold(holder(eight[1:]...), seat); err == nil || !strings.Contains(err.Error(), "does not serve units") {
|
||||
t.Fatalf("a holder missing a required verb was accepted: %v", err)
|
||||
}
|
||||
optional := map[string]bool{"failed": true, "reset-failed": true, "wanted-by": true, "unlink-dangling": true}
|
||||
optional := map[string]bool{"failed": true}
|
||||
for _, v := range seat.Serves {
|
||||
if v.Optional != optional[v.Name] {
|
||||
t.Errorf("%s optional: %v", v.Name, v.Optional)
|
||||
@@ -89,26 +90,35 @@ func TestFailedIsAnOptionalVerbOfTheServiceManager(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **`reset-failed` and `wanted-by` join the seat optional** (novox/hq issue 332, ADR 0246 step 1): the
|
||||
// holder running today, serving the nine verbs, still holds the seat; a holder serving all eleven is not
|
||||
// refused; each replaces the shell command it names; and read back from the store's row they stay optional.
|
||||
func TestResetFailedAndWantedByAreOptionalVerbsOfTheServiceManager(t *testing.T) {
|
||||
// **`reset-failed`, `wanted-by` and `unlink-dangling` are required** (novox/hq issue 332, ADR 0246 step 3):
|
||||
// every holder serves them, so a holder serving only the nine verbs before them is refused by the verb it
|
||||
// lacks; a holder serving all twelve holds the seat; each replaces the shell command it names; and read
|
||||
// back from the store's row they stay required.
|
||||
func TestResetFailedWantedByAndUnlinkDanglingAreRequiredVerbsOfTheServiceManager(t *testing.T) {
|
||||
defer UseSeats(DefaultSeats())
|
||||
nine := []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal", "failed"}
|
||||
all := append(slices.Clone(nine), "reset-failed", "wanted-by", "unlink-dangling")
|
||||
holder := func(serves ...string) Manifest {
|
||||
return Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode, Serves: serves}}}
|
||||
}
|
||||
refusedWithout := func(seat Seat) {
|
||||
t.Helper()
|
||||
for _, verb := range []string{"reset-failed", "wanted-by", "unlink-dangling"} {
|
||||
serves := slices.DeleteFunc(slices.Clone(all), func(v string) bool { return v == verb })
|
||||
if err := CanHold(holder(serves...), seat); err == nil || !strings.Contains(err.Error(), "does not serve "+verb) {
|
||||
t.Errorf("a holder not serving %s holds the seat: %v", verb, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
seat, _ := SeatNamed(ServiceManagerSeat)
|
||||
if err := CanHold(holder(nine...), seat); err != nil {
|
||||
t.Fatalf("today's holder, without the two verbs, is refused: %v", err)
|
||||
if err := CanHold(holder(nine...), seat); err == nil {
|
||||
t.Fatal("a holder serving only the nine verbs before issue 332 holds the seat")
|
||||
}
|
||||
if err := CanHold(holder(append(nine, "reset-failed", "wanted-by")...), seat); err != nil {
|
||||
t.Fatalf("a holder serving both is refused: %v", err)
|
||||
refusedWithout(seat)
|
||||
if err := CanHold(holder(all...), seat); err != nil {
|
||||
t.Fatalf("a holder serving all twelve is refused: %v", err)
|
||||
}
|
||||
if err := CanHold(holder(append(nine, "reset-failed", "wanted-by", "unlink-dangling")...), seat); err != nil {
|
||||
t.Fatalf("a holder serving unlink-dangling too is refused: %v", err)
|
||||
}
|
||||
if err := CanHold(holder(append(nine, "why-started")...), seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||
if err := CanHold(holder(append(slices.Clone(all), "why-started")...), seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
|
||||
t.Fatalf("a verb the seat does not promise was accepted: %v", err)
|
||||
}
|
||||
replaces := map[string]string{"reset-failed": "systemctl reset-failed", "wanted-by": "systemctl list-dependencies --reverse",
|
||||
@@ -118,6 +128,9 @@ func TestResetFailedAndWantedByAreOptionalVerbsOfTheServiceManager(t *testing.T)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if v.Optional {
|
||||
t.Errorf("%s is still optional", v.Name)
|
||||
}
|
||||
if !slices.Contains(v.Replaces, want) {
|
||||
t.Errorf("%s does not say it replaces %q: %v", v.Name, want, v.Replaces)
|
||||
}
|
||||
@@ -148,8 +161,9 @@ func TestResetFailedAndWantedByAreOptionalVerbsOfTheServiceManager(t *testing.T)
|
||||
}
|
||||
UseSeats(rows)
|
||||
seat, _ = SeatNamed(ServiceManagerSeat)
|
||||
if err := CanHold(holder(nine...), seat); err != nil {
|
||||
t.Fatalf("read back from the row, the two verbs are required: %v", err)
|
||||
refusedWithout(seat)
|
||||
if err := CanHold(holder(all...), seat); err != nil {
|
||||
t.Fatalf("read back from the row, a holder serving all twelve is refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,8 +183,9 @@ func TestAnOptionalVerbStaysOptionalInASetReadFromTheStore(t *testing.T) {
|
||||
UseSeats(rows)
|
||||
seat, _ := SeatNamed(ServiceManagerSeat)
|
||||
holder := Manifest{Module: "systemd", Version: "1", Claims: []Claim{{Name: ServiceManagerSeat, Scope: ScopeNode,
|
||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}}}
|
||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal",
|
||||
"reset-failed", "wanted-by", "unlink-dangling"}}}}
|
||||
if err := CanHold(holder, seat); err != nil {
|
||||
t.Fatalf("a set read from rows refuses today's holder: %v", err)
|
||||
t.Fatalf("a set read from rows refuses a holder without the optional failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -678,13 +678,13 @@ func serviceManagerVerbs() []Verb {
|
||||
Replaces: []string{"systemctl --failed"}},
|
||||
// **Clearing a failed record, and finding what starts a unit** (novox/hq issue 332): a unit whose
|
||||
// file is gone stays failed in its manager until the record is reset, and what keeps asking for it
|
||||
// is a dependency or an enable link no verb could show. Both optional while their holders catch up
|
||||
// (ADR 0246 step 1); a later change requires them.
|
||||
{Name: "reset-failed", Optional: true, Description: "Clear one unit's failed record in its manager — " +
|
||||
// is a dependency or an enable link no verb could show. Required since every holder serves them
|
||||
// (ADR 0246 step 3).
|
||||
{Name: "reset-failed", Description: "Clear one unit's failed record in its manager — " +
|
||||
"its failed state and its start-limit count — and answer its state after. It removes no file and " +
|
||||
"starts nothing.",
|
||||
Input: scoped(unit, []string{"unit"}), Replaces: []string{"systemctl reset-failed"}},
|
||||
{Name: "wanted-by", Optional: true, Description: "What wants, requires or triggers one unit, read-only: the " +
|
||||
{Name: "wanted-by", Description: "What wants, requires or triggers one unit, read-only: the " +
|
||||
"manager's reverse dependencies (WantedBy, RequiredBy, UpheldBy, BoundBy, TriggeredBy and the rest) and " +
|
||||
"every enable link naming it in the scope's configuration directories (*.wants, *.requires, *.upholds), " +
|
||||
"each with where it points and whether it dangles — the account's own directory included in user scope.",
|
||||
@@ -692,9 +692,9 @@ func serviceManagerVerbs() []Verb {
|
||||
Replaces: []string{"systemctl list-dependencies --reverse", "systemctl show -p WantedBy",
|
||||
"ls ~/.config/systemd/user/*.wants"}},
|
||||
// **A dangling enable link removed on the person's word** (novox/hq issue 332): `systemctl disable`
|
||||
// leaves a link whose unit file is gone, so the manager keeps asking for the unit. Optional until its
|
||||
// holders serve it (ADR 0246 step 1).
|
||||
{Name: "unlink-dangling", Optional: true, Description: "Remove the dangling enable links named for one " +
|
||||
// leaves a link whose unit file is gone, so the manager keeps asking for the unit. Required since
|
||||
// every holder serves it (ADR 0246 step 3).
|
||||
{Name: "unlink-dangling", Description: "Remove the dangling enable links named for one " +
|
||||
"unit — links in a *.wants, *.requires or *.upholds directory of the account's or the machine's own " +
|
||||
"configuration whose target does not exist, which `disable` leaves once the unit's file is gone — on " +
|
||||
"the person's word: `why` is required. Each is recorded (its path, its target, the reason) before it is " +
|
||||
|
||||
@@ -61,6 +61,10 @@ type Plan struct {
|
||||
// Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out
|
||||
// from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`).
|
||||
Phases *WalkPhases `json:"phases,omitempty"`
|
||||
// WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308
|
||||
// decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every
|
||||
// other save.
|
||||
WalkSaid string `json:"-"`
|
||||
}
|
||||
|
||||
// PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6).
|
||||
@@ -73,6 +77,10 @@ type PlanTimes struct {
|
||||
// Class is the walk's module class, read at its cut: core when it moves a module on the controller's own
|
||||
// path or the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
|
||||
Class string `json:"class,omitempty"`
|
||||
// Started is when the walk was said started, and Ended when it was said ended (novox/hq ADR 0308 decision
|
||||
// 1): each said once, on the save that marked it.
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
}
|
||||
|
||||
// PlanCommit is one repository's commit a walk carries: the latest merge of its branch in the batch.
|
||||
@@ -183,8 +191,8 @@ type PlanDelivery struct {
|
||||
Go *time.Time `json:"go,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Stopped is who ended the walk through the delivery's owner, and StoppedWhy why: the walk is failed,
|
||||
// and the delivery reads it as stopped, not as a build that failed.
|
||||
// Stopped is who ended the walk, through the delivery's owner or by hand (`plans stop|close`), and StoppedWhy
|
||||
// why: the walk is failed, and the delivery reads it as stopped, not as a build that failed.
|
||||
Stopped string `json:"stopped,omitempty"`
|
||||
StoppedWhy string `json:"stopped_why,omitempty"`
|
||||
// Merges is every merge the walk or batch answers (novox/hq ADR 0276): its own commits, and the earlier
|
||||
@@ -236,10 +244,12 @@ func (p Plan) Waiting() bool {
|
||||
return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
|
||||
}
|
||||
|
||||
// PlanSaved is told every plan this process kept, after it is kept (novox/hq ADR 0239): the serving
|
||||
// controller says it on the bus as `plan-moved`. Nil in a command, which says nothing; whoever follows a
|
||||
// walk also asks for it, so a save made by a command is found by comparison.
|
||||
var PlanSaved func(Plan)
|
||||
// PlanSaved is told every plan this process kept, after it is kept, with the inventory that kept it (novox/hq
|
||||
// ADR 0239): the serving controller says it on the bus as `plan-moved`, and so does a command (the controller's
|
||||
// run installs it for every verb, novox/hq ADR 0308 decision 1), so the save that starts or ends a walk is said
|
||||
// whichever process made it. Nil says nothing; whoever follows a walk also asks for it, and finds an unsaid save
|
||||
// by comparison.
|
||||
var PlanSaved func(*Inventory, Plan)
|
||||
|
||||
// ErrPlanMoved is a save against a plan written by somebody else since it was read.
|
||||
var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")
|
||||
@@ -451,7 +461,7 @@ type MergeMove struct {
|
||||
// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by
|
||||
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
|
||||
// the transaction is committed, in order.
|
||||
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error {
|
||||
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) (err error) {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
if len(plans) > 0 {
|
||||
@@ -472,6 +482,20 @@ func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []
|
||||
entered time.Time
|
||||
}
|
||||
done := make([]written, len(plans))
|
||||
// **The save that starts or ends a walk is marked** (novox/hq ADR 0308 decision 1), in the same write, and
|
||||
// unmarked should the write not be made, so the next save marks it again.
|
||||
marked := time.Now()
|
||||
undo := make([]func(), 0, len(plans))
|
||||
defer func() {
|
||||
if err != nil {
|
||||
for _, u := range undo {
|
||||
u()
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, p := range plans {
|
||||
undo = append(undo, p.markWalk(marked))
|
||||
}
|
||||
for n, p := range plans {
|
||||
revision, entered, err := writePlan(ctx, tx, p, epoch)
|
||||
if err != nil {
|
||||
@@ -497,7 +521,7 @@ func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []
|
||||
p.Epoch = 0
|
||||
}
|
||||
if PlanSaved != nil {
|
||||
PlanSaved(*p)
|
||||
PlanSaved(i, *p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -202,14 +202,15 @@ func TestASeatsVerbGainsTheArgumentsTheBinaryNames(t *testing.T) {
|
||||
t.Fatal("failed was not added to the row")
|
||||
}
|
||||
// The optional mark is not stored; the working set read from the rows takes it from the compiled seat,
|
||||
// so today's holder, which does not serve failed, still holds the seat.
|
||||
// so a holder that does not serve the optional failed still holds the seat.
|
||||
catalogue.UseSeats(seats)
|
||||
defer catalogue.UseSeats(catalogue.DefaultSeats())
|
||||
live, _ := catalogue.SeatNamed(catalogue.ServiceManagerSeat)
|
||||
holder := catalogue.Manifest{Module: "systemd", Version: "1", Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat,
|
||||
Scope: catalogue.ScopeNode, Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}}}
|
||||
Scope: catalogue.ScopeNode, Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal",
|
||||
"reset-failed", "wanted-by", "unlink-dangling"}}}}
|
||||
if err := catalogue.CanHold(holder, live); err != nil {
|
||||
t.Fatalf("the seat read from the store refuses today's holder: %v", err)
|
||||
t.Fatalf("the seat read from the store refuses a holder without the optional failed: %v", err)
|
||||
}
|
||||
// Seeding again changes nothing more.
|
||||
before := verbs["journal"]
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package inventory
|
||||
|
||||
import "time"
|
||||
|
||||
// A walk says when it starts and when it ends (novox/hq ADR 0308 decision 1): the save that starts it and the
|
||||
// save that ends it are marked, so `plan-moved` names what the walk touches, its bound and its outcome on those
|
||||
// two saves, and on no other. The marks are kept in the walk's times, so a controller replaced mid-walk (a core
|
||||
// walk replaces it) neither says a start again nor misses the end.
|
||||
|
||||
// The moments of a walk `plan-moved` names.
|
||||
const (
|
||||
WalkStarted = "started"
|
||||
WalkEnded = "ended"
|
||||
)
|
||||
|
||||
// The outcomes of an ended walk (ADR 0308 decision 1).
|
||||
const (
|
||||
WalkDelivered = "delivered"
|
||||
WalkFailed = "failed"
|
||||
WalkStopped = "stopped"
|
||||
)
|
||||
|
||||
// Live says the walk is walking: cut, not waiting for its delivery's word, and not ended.
|
||||
func (p Plan) Live() bool { return p.Open() && !p.Batch() && !p.Waiting() }
|
||||
|
||||
// Ended says the walk is in a state it does not leave by itself: done, failed or superseded.
|
||||
func (p Plan) Ended() bool {
|
||||
return p.State == PlanDone || p.State == PlanFailed || p.State == PlanSuperseded
|
||||
}
|
||||
|
||||
// Outcome is how an ended walk ended: delivered when done; stopped when a person or the delivery's owner stopped
|
||||
// it, or a newer walk took it over; failed otherwise. Empty for a walk not ended.
|
||||
func (p Plan) Outcome() string {
|
||||
switch {
|
||||
case p.State == PlanDone:
|
||||
return WalkDelivered
|
||||
case p.State == PlanSuperseded:
|
||||
return WalkStopped
|
||||
case p.State == PlanFailed && p.Delivery != nil && p.Delivery.Stopped != "":
|
||||
return WalkStopped
|
||||
case p.State == PlanFailed:
|
||||
return WalkFailed
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// markWalk marks the save that starts or ends the walk, in its times and in WalkSaid, and returns what undoes
|
||||
// the mark should the save not be written. A walk starts on the first save that finds it live — when it is cut
|
||||
// and needs no word, or when it is let go — which is before its first tier is asked, and so before anything is
|
||||
// sent: what it restarts is held from its first send on. It starts again when a failed walk is retried. It ends
|
||||
// on the first save that finds it ended after it started; one that ends without ever starting (a batch that
|
||||
// moved nothing, a walk stopped while it waited for its word) touched nothing and says neither. A walk kept
|
||||
// before its times were (ADR 0282 decision 6) is not marked; a release plan is, its times holding only these.
|
||||
func (p *Plan) markWalk(now time.Time) (undo func()) {
|
||||
p.WalkSaid = ""
|
||||
if p.Batch() || (p.Times == nil && p.Release == nil) {
|
||||
return func() {}
|
||||
}
|
||||
var was *PlanTimes
|
||||
if p.Times != nil {
|
||||
kept := *p.Times
|
||||
was = &kept
|
||||
}
|
||||
times := PlanTimes{}
|
||||
if p.Times != nil {
|
||||
times = *p.Times
|
||||
}
|
||||
at := now.UTC()
|
||||
switch {
|
||||
case p.Live() && (times.Started == nil || times.Ended != nil):
|
||||
times.Started, times.Ended = &at, nil
|
||||
p.WalkSaid = WalkStarted
|
||||
case p.Ended() && times.Started != nil && times.Ended == nil:
|
||||
times.Ended = &at
|
||||
p.WalkSaid = WalkEnded
|
||||
default:
|
||||
return func() {}
|
||||
}
|
||||
p.Times = ×
|
||||
return func() { p.Times, p.WalkSaid = was, "" }
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A walk is said started on the save that finds it live, ended on the save that finds it ended after it started,
|
||||
// and neither on any other save (novox/hq ADR 0308 decision 1).
|
||||
func TestAWalkIsSaidStartedOnceAndEndedOnce(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC)
|
||||
cut := now.Add(-time.Second)
|
||||
p := Plan{ID: "plan-1", State: PlanBuilding, Tiers: [][]string{{"messenger"}},
|
||||
Modules: map[string]*PlanModule{"messenger": {}}, Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}}
|
||||
p.markWalk(now)
|
||||
if p.WalkSaid != WalkStarted || p.Times.Started == nil || !p.Times.Started.Equal(now) {
|
||||
t.Fatalf("a live walk's first save starts it: %q %+v", p.WalkSaid, p.Times)
|
||||
}
|
||||
p.State = PlanRolling
|
||||
p.markWalk(now.Add(time.Minute))
|
||||
if p.WalkSaid != "" || !p.Times.Started.Equal(now) {
|
||||
t.Fatalf("a later save says nothing and keeps the start: %q %+v", p.WalkSaid, p.Times)
|
||||
}
|
||||
p.State = PlanDone
|
||||
p.markWalk(now.Add(2 * time.Minute))
|
||||
if p.WalkSaid != WalkEnded || p.Times.Ended == nil || p.Outcome() != WalkDelivered {
|
||||
t.Fatalf("the save that finds it done ends it, delivered: %q %+v %q", p.WalkSaid, p.Times, p.Outcome())
|
||||
}
|
||||
p.markWalk(now.Add(3 * time.Minute))
|
||||
if p.WalkSaid != "" {
|
||||
t.Fatalf("an ended walk saved again says nothing: %q", p.WalkSaid)
|
||||
}
|
||||
// Retried, it walks again: a new start.
|
||||
p.State = PlanBuilding
|
||||
p.markWalk(now.Add(4 * time.Minute))
|
||||
if p.WalkSaid != WalkStarted || p.Times.Ended != nil || !p.Times.Started.Equal(now.Add(4*time.Minute)) {
|
||||
t.Fatalf("a retried walk starts again: %q %+v", p.WalkSaid, p.Times)
|
||||
}
|
||||
}
|
||||
|
||||
// What is never said: a batch, a walk waiting for its word, a walk that ends without having started, and a walk
|
||||
// kept before its times were. A release plan is said.
|
||||
func TestWhatIsNotAWalkStartIsNotSaid(t *testing.T) {
|
||||
now := time.Now()
|
||||
for name, p := range map[string]Plan{
|
||||
"a batch": {State: PlanAssembling, Times: &PlanTimes{}},
|
||||
"waiting for a word": {State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery"}},
|
||||
"ended, never begun": {State: PlanDone, Times: &PlanTimes{}},
|
||||
"kept before times": {State: PlanBuilding},
|
||||
"stopped while waits": {State: PlanFailed, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Stopped: "x"}},
|
||||
} {
|
||||
p.markWalk(now)
|
||||
if p.WalkSaid != "" {
|
||||
t.Errorf("%s: said %q", name, p.WalkSaid)
|
||||
}
|
||||
}
|
||||
r := Plan{State: PlanRolling, Release: &PlanRelease{Order: []string{"ace"}}}
|
||||
r.markWalk(now)
|
||||
if r.WalkSaid != WalkStarted || r.Times == nil || r.Times.Started == nil || r.Times.Class != "" {
|
||||
t.Fatalf("a release plan is said started, its times holding only that: %q %+v", r.WalkSaid, r.Times)
|
||||
}
|
||||
// Let go, the walk starts.
|
||||
go1 := now
|
||||
w := Plan{State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Go: &go1}}
|
||||
w.markWalk(now)
|
||||
if w.WalkSaid != WalkStarted {
|
||||
t.Fatalf("a walk let go starts: %q", w.WalkSaid)
|
||||
}
|
||||
}
|
||||
|
||||
// An ended walk's outcome: delivered, failed, or stopped by a person or the delivery's owner, or by a newer walk.
|
||||
func TestAWalksOutcome(t *testing.T) {
|
||||
for want, p := range map[string]Plan{
|
||||
WalkDelivered: {State: PlanDone},
|
||||
WalkFailed: {State: PlanFailed},
|
||||
WalkStopped: {State: PlanFailed, Delivery: &PlanDelivery{Stopped: "mesh-delivery"}},
|
||||
"": {State: PlanRolling},
|
||||
} {
|
||||
if got := p.Outcome(); got != want {
|
||||
t.Errorf("%s: outcome %q, want %q", p.State, got, want)
|
||||
}
|
||||
}
|
||||
if (Plan{State: PlanSuperseded}).Outcome() != WalkStopped {
|
||||
t.Error("a superseded walk is stopped")
|
||||
}
|
||||
}
|
||||
|
||||
// The marks are kept with the walk, so a controller replaced mid-walk neither says its start again nor misses
|
||||
// its end; and a save refused leaves the walk unmarked, so the save that is made says it.
|
||||
func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
cut := time.Now().UTC().Add(-time.Minute)
|
||||
p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding,
|
||||
Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}},
|
||||
Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}}
|
||||
var said []string
|
||||
was := PlanSaved
|
||||
PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) }
|
||||
t.Cleanup(func() { PlanSaved = was })
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf {
|
||||
t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times)
|
||||
}
|
||||
// Read back by another controller and saved: not said again.
|
||||
back.State = PlanRolling
|
||||
if err := inv.SavePlan(ctx, &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A stale copy ending it is refused, and unmarked.
|
||||
stale := p
|
||||
stale.Revision = 1
|
||||
stale.State = PlanDone
|
||||
if err := inv.SavePlan(ctx, &stale); err == nil {
|
||||
t.Fatal("a stale save is refused")
|
||||
}
|
||||
if stale.WalkSaid != "" || stale.Times.Ended != nil {
|
||||
t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times)
|
||||
}
|
||||
back.State = PlanDone
|
||||
if err := inv.SavePlan(ctx, &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded {
|
||||
t.Fatalf("said: %q, want started, nothing, ended", said)
|
||||
}
|
||||
}
|
||||
Vendored
+8
@@ -14,3 +14,11 @@
|
||||
./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance
|
||||
./internal/inventory TestAConditionsLooksAreKeptBesideTheBucket
|
||||
./cmd/mesh-controller TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds
|
||||
./internal/inventory TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid
|
||||
./cmd/mesh-controller TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded
|
||||
./cmd/mesh-controller TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped
|
||||
./cmd/mesh-controller TestAWalkRetriedByAPersonIsSaidStartedAgain
|
||||
./cmd/mesh-controller TestABacklogReleasedByAPersonIsSaidStarted
|
||||
./cmd/mesh-controller TestAWalkEndedByCancelOrClearIsSaidEnded
|
||||
./cmd/mesh-controller TestAWalkRestartedByARebuildIsSaidStarted
|
||||
./cmd/mesh-controller TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain
|
||||
|
||||
Reference in New Issue
Block a user