Compare commits
89
Commits
b1cfb85f86
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c2a8d6210 | ||
|
|
c07f604047 | ||
|
|
b893cc829c | ||
|
|
70cb217601 | ||
|
|
1d0df24a0d | ||
|
|
202b453de1 | ||
|
|
7a0b275020 | ||
|
|
40350d82fa | ||
|
|
1f708e2d8d | ||
|
|
7f23c537cc | ||
|
|
0aac89f4e2 | ||
|
|
d8c13d97ca | ||
|
|
f1775281e2 | ||
|
|
4083755ca1 | ||
|
|
7b83aaa949 | ||
|
|
23a209f3d0 | ||
|
|
3448f98000 | ||
|
|
12db1d3748 | ||
|
|
7770d193da | ||
|
|
248fe521a3 | ||
|
|
666dcff79b | ||
|
|
caa3fc8995 | ||
|
|
a0c971cea9 | ||
|
|
8802076499 | ||
|
|
2d2e736b45 | ||
|
|
fdfda0e839 | ||
|
|
540cda11df | ||
|
|
ad753c70f6 | ||
|
|
da20fea549 | ||
|
|
2f4f50b9ca | ||
|
|
14f405ece5 | ||
|
|
0d1d780d43 | ||
|
|
5c807808e8 | ||
|
|
da5a5c9b67 | ||
|
|
07a8f04e65 | ||
|
|
dd802c81a7 | ||
|
|
a5a43c8f31 | ||
|
|
3e7f34d041 | ||
|
|
e29e855bdf | ||
|
|
da3c01089e | ||
|
|
253283d68e | ||
|
|
3f038b989e | ||
|
|
3834c72747 | ||
|
|
68088ec782 | ||
|
|
8da2619920 | ||
|
|
59c8cf2f24 | ||
|
|
3ac8657a49 | ||
|
|
230f0a1a18 | ||
|
|
6e487fa01f | ||
|
|
7a74cee83f | ||
|
|
378e65780d | ||
|
|
296e292f55 | ||
|
|
791b92c875 | ||
|
|
1490c3b075 | ||
|
|
4104d49ec8 | ||
|
|
1991fe931b | ||
|
|
d8249dad8b | ||
|
|
df4d1204c7 | ||
|
|
e8c8f0488b | ||
|
|
cfb6a8c531 | ||
|
|
9b7f9a7fd2 | ||
|
|
65632e7999 | ||
|
|
f6a48ac5ef | ||
|
|
93a82cb032 | ||
|
|
de651e0fd1 | ||
|
|
97ee33e8f4 | ||
|
|
19e3eac746 | ||
|
|
863db82737 | ||
|
|
a069898c0c | ||
|
|
ffad53aad2 | ||
|
|
89cfa9431d | ||
|
|
5939bbdefe | ||
|
|
ec85cdf70c | ||
|
|
cd303f68ed | ||
|
|
2cbebbc011 | ||
|
|
ac79e29319 | ||
|
|
ef0ca69cb0 | ||
|
|
cec26d8596 | ||
|
|
e621e15d2f | ||
|
|
a08498b240 | ||
|
|
39ae7de6d3 | ||
|
|
4e12fc19b7 | ||
|
|
19f15d8ea0 | ||
|
|
fb66608a13 | ||
|
|
7be2df627f | ||
|
|
cb006fc8dd | ||
|
|
0273609090 | ||
|
|
3b49d7044f | ||
|
|
ba59283b4b |
@@ -432,6 +432,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
result.Against = built.Against
|
||||
result.SourceFingerprint = built.Source
|
||||
result.SourceInputs = built.SourceInputs
|
||||
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||
result.ReuseKey = built.ReuseKey
|
||||
if built.Reused != nil {
|
||||
|
||||
@@ -33,7 +33,7 @@ const packageManagerKey = "the package manager ("
|
||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
heard := map[string]bool{}
|
||||
for _, m := range f.machines {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) {
|
||||
heard[m.name] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,8 +208,11 @@ type asker struct {
|
||||
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
||||
// asked, and why. Nil raises nothing (a test that does not look).
|
||||
raise func(ctx context.Context, obs []conditions.Observation) error
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// tell tells the operator one plain notice that needs no answer (operator-channel.tell, novox/hq ADR 0380):
|
||||
// the refusal of an answer to a plan's ask (ADR 0449 §3). Nil tells nothing (a test that does not look).
|
||||
tell func(ctx context.Context, headline string, lines []string) error
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
saidNoRouter bool
|
||||
saidNoGrant bool
|
||||
@@ -406,7 +409,9 @@ func (a *asker) reconcile(ctx context.Context) error {
|
||||
for _, r := range all {
|
||||
k := partKey(r.Condition, r.Part)
|
||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||
answered[k] = r
|
||||
if prior, has := answered[k]; !has || r.Ended.After(prior.Ended) {
|
||||
answered[k] = r
|
||||
}
|
||||
}
|
||||
if r.State == string(asks.OutcomeRefused) {
|
||||
if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) {
|
||||
@@ -497,7 +502,9 @@ func (a *asker) reconcile(ctx context.Context) error {
|
||||
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
||||
}
|
||||
}
|
||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
||||
// **A condition raised after the answered ask ended is a new one** (novox/hq ADR 0449 §2): a retried
|
||||
// plan that fails again is asked again, not held back for the hour its earlier answer holds.
|
||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) && !c.Raised.After(r.Ended) {
|
||||
if _, held := byCondition[key]; !held {
|
||||
continue
|
||||
}
|
||||
@@ -636,6 +643,8 @@ func doesWords(act conditions.Action) string {
|
||||
return "the delivery starts"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||
return "the delivery is stopped"
|
||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "":
|
||||
return "the failed builds are asked again"
|
||||
case strings.HasSuffix(act.Verb, ".restart"):
|
||||
return "its service is restarted on " + act.Machine
|
||||
}
|
||||
@@ -884,6 +893,7 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
ended := a.now()
|
||||
if acted != nil {
|
||||
outcome = "failed: " + acted.Error()
|
||||
a.tellRefused(ctx, *r, act, acted)
|
||||
}
|
||||
r.Ended, r.Acted = ended, outcome
|
||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||
@@ -918,6 +928,55 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// tellRefused tells the operator what became of the answer they gave to a plan's ask, when it was not done (novox/hq
|
||||
// ADR 0449 §3): the controller's `plans` verb refuses a Retry, Start or Stop when the plan moved on between the ask and
|
||||
// the answer, and the refusal was only recorded, so the operator who pressed the answer heard nothing. **A refusal is
|
||||
// told as one, and nothing else is**: a verb that said no changed nothing, but a call that got no answer (a timeout, a
|
||||
// refused grant) may have run, and is told as not known. One plain notice with the ask's headline, beside the hand act
|
||||
// the caller still records. An answer to any other ask is not told here. A notice that cannot be told is said in the
|
||||
// log and changes nothing.
|
||||
func (a *asker) tellRefused(ctx context.Context, r asked, act conditions.Action, failure error) {
|
||||
if a.tell == nil || act.Verb != broker.ControllerSeat+".plans" {
|
||||
return
|
||||
}
|
||||
var lines []string
|
||||
var refused verbRefused
|
||||
if errors.As(failure, &refused) {
|
||||
lines = []string{
|
||||
fmt.Sprintf("Your answer, %s, could not be done: %s.", act.Label, strings.TrimRight(refusalWords(refused.words, act), ".")),
|
||||
"Nothing was changed.",
|
||||
}
|
||||
} else {
|
||||
lines = []string{
|
||||
fmt.Sprintf("Your answer, %s, was given, and it is not known whether it was done.", act.Label),
|
||||
"Look at the delivery before you answer again.",
|
||||
}
|
||||
}
|
||||
if err := a.tell(ctx, r.Ask.Headline, lines); err != nil {
|
||||
a.logf("the operator could NOT be told that %s was refused (ask %s): %v", act.Label, r.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusalWords is a verb's refusal as the operator reads it: the plans named as "the plan" (plainRefusal), and, where
|
||||
// the words still carry an identifier the operator's channel withholds, one sentence of the answer's own.
|
||||
func refusalWords(words string, act conditions.Action) string {
|
||||
return plainRefusalOr(words, refusalFallback(act))
|
||||
}
|
||||
|
||||
// refusalFallback is what an answer's refusal says when its own words cannot be said: one sentence for each answer
|
||||
// to a plan's ask, never another answer's.
|
||||
func refusalFallback(act conditions.Action) string {
|
||||
switch {
|
||||
case act.Arguments["retry"] != "":
|
||||
return "a retry would not run now; the details say why"
|
||||
case act.Arguments["go"] != "":
|
||||
return "the delivery could not be started now; the details say why"
|
||||
case act.Arguments["stop"] != "":
|
||||
return "the delivery could not be stopped now; the details say why"
|
||||
}
|
||||
return "the controller refused it; the details say why"
|
||||
}
|
||||
|
||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||
// never a repair (handActVerbs).
|
||||
const handActWarrant = "warrant"
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
|
||||
// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
|
||||
func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
ended := before.Add(time.Minute)
|
||||
failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Delivery: &inventory.PlanDelivery{},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}},
|
||||
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
// The controller's seat answers `plans` as it does: by running the verb.
|
||||
sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
|
||||
var args map[string]string
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
answer := link.Answer{Result: json.RawMessage(`"ok"`)}
|
||||
if len(args) != 1 || args["retry"] == "" {
|
||||
answer = link.Answer{Error: "the retry was asked with more than the plan"}
|
||||
} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
|
||||
answer = link.Answer{Error: err.Error()}
|
||||
}
|
||||
body, _ := json.Marshal(answer)
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
// What S23 raises for it, read from the store as the watchdog reads it.
|
||||
raised := func(context.Context) ([]conditions.Condition, error) {
|
||||
facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
|
||||
out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
now := time.Now()
|
||||
store := memAskedStore{}
|
||||
var sent []published
|
||||
var acts []link.HandAct
|
||||
a := &asker{open: raised, store: store,
|
||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
||||
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
||||
sent = append(sent, published{subject, id, body})
|
||||
return nil
|
||||
},
|
||||
call: callAction(conn),
|
||||
record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
|
||||
now: func() time.Time { return now },
|
||||
logf: t.Logf}
|
||||
cs, _ := raised(ctx)
|
||||
if len(cs) != 1 || len(cs[0].Actions) != 1 {
|
||||
t.Fatalf("the failed plan raised %+v", cs)
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w asks.Warrant
|
||||
for _, r := range store {
|
||||
for _, o := range r.Ask.Options {
|
||||
if o.Label == "Retry" {
|
||||
w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
|
||||
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
||||
AskDigest: r.Ask.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.Ask == "" {
|
||||
t.Fatalf("no ask offers Retry: %+v", store)
|
||||
}
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := p.Modules["a"]
|
||||
if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" {
|
||||
t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("the build was asked %d times: %v", len(*asked), *asked)
|
||||
}
|
||||
if got := store[w.Ask]; got.Acted != "done" {
|
||||
t.Fatalf("the ask ended %q", got.Acted)
|
||||
}
|
||||
if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
|
||||
t.Fatalf("the warrant was recorded as %+v", acts)
|
||||
}
|
||||
if cs, _ = raised(ctx); len(cs) != 0 {
|
||||
t.Fatalf("the plan-failed condition did not end: %+v", cs)
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,15 @@ type askerRig struct {
|
||||
silenced []string
|
||||
acts []link.HandAct
|
||||
now time.Time
|
||||
// refuses, when set, is what every verb the asker calls answers; told is every notice the operator was told.
|
||||
refuses error
|
||||
told []told
|
||||
}
|
||||
|
||||
// told is one plain notice the asker told the operator.
|
||||
type told struct {
|
||||
headline string
|
||||
lines []string
|
||||
}
|
||||
|
||||
func newAskerRig(t *testing.T) *askerRig {
|
||||
@@ -102,6 +111,10 @@ func newAskerRig(t *testing.T) *askerRig {
|
||||
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
||||
raw, _ := json.Marshal(args)
|
||||
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
||||
return r.refuses
|
||||
},
|
||||
tell: func(_ context.Context, headline string, lines []string) error {
|
||||
r.told = append(r.told, told{headline, lines})
|
||||
return nil
|
||||
},
|
||||
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
||||
@@ -1013,3 +1026,215 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
|
||||
func failedPlanCondition(plan string) conditions.Condition {
|
||||
p := aFailedPlan(plan, nil)
|
||||
f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
o := watchPlansFailed(f)[0]
|
||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
}
|
||||
|
||||
// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
|
||||
func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
r.open = []conditions.Condition{c}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
||||
t.Fatalf("asked %+v", sent)
|
||||
}
|
||||
o := sent[0].Options[0]
|
||||
if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
|
||||
t.Errorf("the option is %+v", o)
|
||||
}
|
||||
if err := sent[0].Check(r.now); err != nil {
|
||||
t.Errorf("the ask is refused as made: %v", err)
|
||||
}
|
||||
if strings.Contains(sent[0].Explanation, "mesh MCP server") {
|
||||
t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
|
||||
// the same warrant heard again does nothing.
|
||||
func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition(plan)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, c.Key, "Retry")
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w)
|
||||
if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
|
||||
t.Fatalf("called %v, want exactly %s", r.called, want)
|
||||
}
|
||||
if len(r.acts) != 1 {
|
||||
t.Fatalf("hand-acts %+v", r.acts)
|
||||
}
|
||||
act := r.acts[0]
|
||||
if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
|
||||
act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
|
||||
t.Errorf("the hand-act is %+v", act)
|
||||
}
|
||||
if got := r.store[w.Ask]; got.Acted != "done" {
|
||||
t.Errorf("kept %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is
|
||||
// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before.
|
||||
func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
c conditions.Condition
|
||||
label string
|
||||
}{
|
||||
{"retry", failedPlanCondition(plan), "Retry"},
|
||||
{"stop", waiting, "Stop"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"}
|
||||
r.open = []conditions.Condition{tc.c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, tc.c.Key, tc.label)
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w) // heard again: told once
|
||||
if len(r.told) != 1 {
|
||||
t.Fatalf("told %+v", r.told)
|
||||
}
|
||||
got := r.told[0]
|
||||
if got.headline != tc.c.Headline {
|
||||
t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline)
|
||||
}
|
||||
said := strings.Join(got.lines, " ")
|
||||
if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) {
|
||||
t.Errorf("the lines do not carry the answer and the refusal's words: %q", said)
|
||||
}
|
||||
if why, ok := conditions.Plain(said); !ok {
|
||||
t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said)
|
||||
}
|
||||
if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") {
|
||||
t.Errorf("the failure is not recorded as before: %+v", r.acts)
|
||||
}
|
||||
if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") {
|
||||
t.Errorf("kept %+v", kept)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here.
|
||||
func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
if len(r.told) != 0 {
|
||||
t.Fatalf("a retry that was done told %+v", r.told)
|
||||
}
|
||||
|
||||
r = newAskerRig(t)
|
||||
r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"}
|
||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
||||
r.open = []conditions.Condition{module}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, module.Key, "Restart"))
|
||||
if len(r.told) != 0 || len(r.acts) != 1 {
|
||||
t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts)
|
||||
}
|
||||
}
|
||||
|
||||
// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was
|
||||
// changed"; and the sentence a refusal falls back to fits the answer given.
|
||||
func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
r := newAskerRig(t)
|
||||
r.refuses = errors.New("mesh-controller did not answer plans within 30s")
|
||||
c := failedPlanCondition(plan)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
if len(r.told) != 1 {
|
||||
t.Fatalf("told %+v", r.told)
|
||||
}
|
||||
said := strings.Join(r.told[0].lines, " ")
|
||||
if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") {
|
||||
t.Errorf("a call that may have run is told as %q", said)
|
||||
}
|
||||
if why, ok := conditions.Plain(said); !ok {
|
||||
t.Errorf("not plain: %s: %q", why, said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
for _, tc := range []struct {
|
||||
act conditions.Action
|
||||
want string
|
||||
}{
|
||||
{retryAction(plan), "a retry would not run now"},
|
||||
{conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"},
|
||||
{conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"},
|
||||
} {
|
||||
// Words the operator's channel withholds: a hash and a dotted name.
|
||||
got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act)
|
||||
if !strings.Contains(got, tc.want) {
|
||||
t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want)
|
||||
}
|
||||
if _, ok := conditions.Plain(got); !ok {
|
||||
t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the
|
||||
// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not.
|
||||
func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
c.Raised = r.now.Add(-time.Minute)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
// Still open right after the answer: not asked again at once.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != 1 {
|
||||
t.Fatalf("asked again while the same condition stayed open: %d", n)
|
||||
}
|
||||
// The retry ran, the plan left failed and the condition ended.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
r.open = nil
|
||||
_ = r.a.reconcile(context.Background())
|
||||
// It fails again, ten minutes after the answer: a condition raised after the answered ask ended.
|
||||
r.now = r.now.Add(8 * time.Minute)
|
||||
again := failedPlanCondition("plan-1792433168676870614")
|
||||
again.Raised = r.now
|
||||
r.open = []conditions.Condition{again}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(r.asksSent(t)); n != 2 {
|
||||
t.Fatalf("the second failure was asked %d times in all, want 2", n)
|
||||
}
|
||||
if len(r.called) != 1 {
|
||||
t.Fatalf("something retried without an answer: %v", r.called)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,6 +135,12 @@ func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// verbRefused is a verb's own refusal of an act: the seat answered, and said no. Any other error of callAction (a bus
|
||||
// that did not answer, a grant that is refused) is not one: the verb may have run.
|
||||
type verbRefused struct{ verb, words string }
|
||||
|
||||
func (e verbRefused) Error() string { return e.verb + " refused: " + e.words }
|
||||
|
||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||
@@ -168,7 +174,23 @@ func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action,
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s refused: %s", a.Verb, answer.Error)
|
||||
return verbRefused{verb: a.Verb, words: answer.Error}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// tellThrough tells the operator one plain notice through the operator channel's `tell`, over the controller's own
|
||||
// connection and grant (novox/hq ADR 0380); a holder's refusal is an error naming it.
|
||||
func tellThrough(conn *nats.Conn) func(ctx context.Context, headline string, lines []string) error {
|
||||
return func(ctx context.Context, headline string, lines []string) error {
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell",
|
||||
map[string]any{"headline": headline, "lines": lines}, tellWithin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("operator-channel.tell refused: %s", answer.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -576,6 +598,7 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper
|
||||
_, err := link.RecordHandAct(ctx, conn, act)
|
||||
return err
|
||||
},
|
||||
tell: tellThrough(conn),
|
||||
routerRecord: routerRecordOf(conn, open.inventory),
|
||||
routerHere: routerHereIn(open.inventory),
|
||||
grantHeld: grantHeldIn(open, conn),
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
||||
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
||||
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
||||
|
||||
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
||||
// is away once presence says so). A machine
|
||||
// never heard from has not joined and is not away.
|
||||
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
if m.silent(now) {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
||||
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
||||
// before: a machine is not passed on a reading that was not made.
|
||||
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
machines := make([]machineFacts, len(nodes))
|
||||
for i, n := range nodes {
|
||||
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
||||
}
|
||||
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return awayOf(machines, now), nil
|
||||
}
|
||||
|
||||
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
||||
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
||||
for _, n := range running {
|
||||
if away[n] && !slices.Contains(p.Away, n) {
|
||||
p.Away = append(p.Away, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(p.Away)
|
||||
}
|
||||
|
||||
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
||||
// those it did not pass as away.
|
||||
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
||||
if !slices.Contains(p.Away, n) {
|
||||
return false
|
||||
}
|
||||
_, sent := s.Rest[n]
|
||||
return !sent
|
||||
})
|
||||
}
|
||||
|
||||
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
||||
func awaySaying(p inventory.Plan) string {
|
||||
var said []string
|
||||
for _, n := range p.Away {
|
||||
said = append(said, n+" was away: sent when it is heard again")
|
||||
}
|
||||
return strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
||||
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
||||
away, err := judgeAway(ctx, open, now)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
||||
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
||||
}
|
||||
return away
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it
|
||||
// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard
|
||||
// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's
|
||||
// builds, so the send it gets when it is heard again carries them (AC3).
|
||||
func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
// laptop is away as every tier is sent; anchor is heard.
|
||||
wasAway := judgeAway
|
||||
judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true}, nil
|
||||
}
|
||||
t.Cleanup(func() { judgeAway = wasAway })
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = nil })
|
||||
sentHeard := make(chan *nats.Msg, 32)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
wasSaved := inventory.PlanSaved
|
||||
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||
|
||||
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||
var mu sync.Mutex
|
||||
var asks []string
|
||||
var sends [][]string
|
||||
wasAsk := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
||||
asks = append(asks, source.Repository+"#"+id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = wasAsk })
|
||||
digests := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
sends = append(sends, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds := map[string]string{}
|
||||
for _, m := range []string{"a", "b"} {
|
||||
builds[m] = current[m].Commit
|
||||
}
|
||||
for _, node := range names {
|
||||
digests++
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
register := func(m, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record := func(id, m, commit string, at time.Time) {
|
||||
t.Helper()
|
||||
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
for _, m := range []string{"a", "b"} {
|
||||
record("build-"+m+"-1", m, "c1", old)
|
||||
register(m, "c1", old)
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||
register("a", "c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// A machine's report on what it was sent, heard as the controller hears it.
|
||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||
seq := int64(0)
|
||||
reports := func(node string) {
|
||||
t.Helper()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||
}
|
||||
seq++
|
||||
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||
"report_sequence": seq})
|
||||
var report link.Report
|
||||
if err := json.Unmarshal(body, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := listener.Heard(ctx, report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
plan := func() inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := inv.PlanByID(ctx, "plan-1160")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
built := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||
"b": {},
|
||||
}}
|
||||
first, stopFirst := context.WithCancel(ctx)
|
||||
defer stopFirst()
|
||||
waitFirst := startRolloutEvents(first, open, js)
|
||||
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
||||
t.Cleanup(func() { stopFirst(); waitFirst() })
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "tier 0 sent to anchor alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor"
|
||||
})
|
||||
reports("anchor")
|
||||
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
||||
var moved string
|
||||
eventually(t, "tier 1 reached on anchor's report alone", func() bool {
|
||||
if q := plan(); q.Tier == 1 {
|
||||
moved = q.Note
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
|
||||
t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
|
||||
}
|
||||
// The ask for tier 1's build is the next step, and records its build id on the module.
|
||||
eventually(t, "tier 1 asked", func() bool { m := plan().Modules["b"]; return m != nil && m.Build != "" })
|
||||
b := plan().Modules["b"]
|
||||
askedAt, _ := link.BuildAskedAt(b.Build)
|
||||
record(b.Build, "b", "c2", askedAt)
|
||||
register("b", "c2", askedAt)
|
||||
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
||||
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
||||
})
|
||||
// 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again.
|
||||
was, known, err := inv.SentBuilds(ctx, "laptop")
|
||||
if err != nil || !known {
|
||||
t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if was["a"] == current["a"].Commit || was["a"] != "c1" {
|
||||
t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit)
|
||||
}
|
||||
if got := plan().Away; strings.Join(got, ",") != "laptop" {
|
||||
t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
||||
|
||||
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
||||
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
machines := []machineFacts{
|
||||
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
||||
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
||||
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
||||
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
||||
{name: "new"},
|
||||
}
|
||||
got := awayOf(machines, now)
|
||||
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
||||
"new": false} {
|
||||
if got[node] != want {
|
||||
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
||||
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
||||
p := &inventory.Plan{}
|
||||
away := map[string]bool{"g14": true, "ace": true}
|
||||
rememberAway(p, []string{"anchor", "g14"}, away)
|
||||
rememberAway(p, []string{"g14", "ace"}, away)
|
||||
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
||||
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC2: the plan says it, where its other passed machines are said.
|
||||
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
||||
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
||||
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
||||
t.Fatalf("1161/AC2: the plan says %q", say)
|
||||
}
|
||||
if got := awaySaying(inventory.Plan{}); got != "" {
|
||||
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
||||
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
||||
p := inventory.Plan{Away: []string{"g14"}}
|
||||
none := &inventory.PlanModule{}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
||||
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
||||
}
|
||||
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
||||
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -176,6 +176,11 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
repository, m.Base, m.Commit, already)
|
||||
return nil
|
||||
}
|
||||
if already == 0 {
|
||||
if done, err := f.firstBuilds(ctx, m, entries, read, now); err != nil || done {
|
||||
return err
|
||||
}
|
||||
}
|
||||
fmt.Print(nothingReadsIt(repository, m))
|
||||
return nil
|
||||
}
|
||||
@@ -362,7 +367,7 @@ func buildsEvery(p inventory.Plan, modules []inventory.Entry) bool {
|
||||
// ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word.
|
||||
func ownPath(touches []inventory.Entry) bool {
|
||||
for _, e := range touches {
|
||||
if _, own := onTheControllersPath[e.Manifest.Module]; own {
|
||||
if onTheOwnPath(e.Manifest.Module, e.Manifest) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -742,6 +747,9 @@ func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inven
|
||||
type keptMerge struct {
|
||||
link.SourceMoved
|
||||
Moves []string `json:"moves,omitempty"`
|
||||
// FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first
|
||||
// build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146).
|
||||
FirstBuild bool `json:"first_build,omitempty"`
|
||||
}
|
||||
|
||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||
@@ -937,7 +945,11 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
||||
var read map[string][]inventory.ReadRepository
|
||||
var edges []inventory.Edge
|
||||
judged := false
|
||||
openWalks, err := openWalksOf(ctx, inv, started)
|
||||
onPath, err := ownPathNow(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
openWalks, err := openWalksOf(ctx, inv, started, onPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -970,7 +982,7 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
||||
var fold, keep []inventory.Plan
|
||||
if !b.OwnPath() {
|
||||
for _, w := range waiting {
|
||||
ww, err := openWalkOf(ctx, inv, w)
|
||||
ww, err := openWalkOf(ctx, inv, w, onPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1005,7 +1017,7 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
||||
if !cut.Open() {
|
||||
continue
|
||||
}
|
||||
w, err := openWalkOf(ctx, inv, cut)
|
||||
w, err := openWalkOf(ctx, inv, cut, onPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1383,8 +1395,9 @@ func classOfModules(modules []string, entries []inventory.Entry) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
onPath := ownPathOf(entries)
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
||||
if _, own := onPath[m]; own || resolvers[m] {
|
||||
return inventory.ClassCore
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -152,6 +153,7 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
Path: result.Path,
|
||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
||||
SourceFingerprint: result.SourceFingerprint,
|
||||
SourceInputs: result.SourceInputs,
|
||||
// And its reuse key (novox/hq ADR 0350 §1), which the chain names (ADR 0408 §1).
|
||||
ReuseKey: result.ReuseKey,
|
||||
}
|
||||
@@ -236,47 +238,56 @@ func buildsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
for _, b := range builds {
|
||||
what := b.Module
|
||||
if what == "" {
|
||||
// It failed before knowing what it was building, which is most of the interesting
|
||||
// failures. The repository is what a person has to go and look at.
|
||||
what = "?"
|
||||
}
|
||||
outcome := "built " + short(b.Commit)
|
||||
if !b.Worked() {
|
||||
outcome = "failed"
|
||||
}
|
||||
fmt.Printf("%-18s %-14s %-10s %s %s\n",
|
||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||
fmt.Printf(" %s", b.Repository)
|
||||
if b.Ref != "" {
|
||||
fmt.Printf(" at %s", b.Ref)
|
||||
}
|
||||
fmt.Println()
|
||||
for _, made := range b.Made {
|
||||
fmt.Printf(" %-10s %s\n", made.Kind, made.Reference)
|
||||
}
|
||||
// What it published and under what reuse key (novox/hq ADR 0408 §1).
|
||||
if p := b.Published; p != nil {
|
||||
if p.Artifact != "" {
|
||||
fmt.Printf(" published %s\n", p.Artifact)
|
||||
}
|
||||
if p.Package != nil {
|
||||
fmt.Printf(" published %s@%s\n", p.Package.Name, p.Package.Version)
|
||||
}
|
||||
}
|
||||
if b.ReuseKey != "" {
|
||||
fmt.Printf(" reuse key %s\n", b.ReuseKey)
|
||||
}
|
||||
if !b.Worked() {
|
||||
// The builder's own first line. The whole failure is often a build log, and printing
|
||||
// it here would bury every other row.
|
||||
fmt.Printf(" %s\n", firstLine(b.Failed))
|
||||
}
|
||||
printBuild(os.Stdout, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printBuild says one build: what, its outcome, where and when, what it made and published, its reuse key, and its
|
||||
// source fingerprint (novox/hq issue 1204: the one tool that reads what the builder recorded it was made from).
|
||||
func printBuild(w io.Writer, b inventory.Build) {
|
||||
what := b.Module
|
||||
if what == "" {
|
||||
// It failed before knowing what it was building, which is most of the interesting
|
||||
// failures. The repository is what a person has to go and look at.
|
||||
what = "?"
|
||||
}
|
||||
outcome := "built " + short(b.Commit)
|
||||
if !b.Worked() {
|
||||
outcome = "failed"
|
||||
}
|
||||
fmt.Fprintf(w, "%-18s %-14s %-10s %s %s\n",
|
||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||
fmt.Fprintf(w, " %s", b.Repository)
|
||||
if b.Ref != "" {
|
||||
fmt.Fprintf(w, " at %s", b.Ref)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
for _, made := range b.Made {
|
||||
fmt.Fprintf(w, " %-10s %s\n", made.Kind, made.Reference)
|
||||
}
|
||||
// What it published and under what reuse key (novox/hq ADR 0408 §1).
|
||||
if p := b.Published; p != nil {
|
||||
if p.Artifact != "" {
|
||||
fmt.Fprintf(w, " published %s\n", p.Artifact)
|
||||
}
|
||||
if p.Package != nil {
|
||||
fmt.Fprintf(w, " published %s@%s\n", p.Package.Name, p.Package.Version)
|
||||
}
|
||||
}
|
||||
if b.ReuseKey != "" {
|
||||
fmt.Fprintf(w, " reuse key %s\n", b.ReuseKey)
|
||||
}
|
||||
if b.SourceFingerprint != "" {
|
||||
fmt.Fprintf(w, " source %s\n", b.SourceFingerprint)
|
||||
}
|
||||
if !b.Worked() {
|
||||
// The builder's own first line. The whole failure is often a build log, and printing
|
||||
// it here would bury every other row.
|
||||
fmt.Fprintf(w, " %s\n", firstLine(b.Failed))
|
||||
}
|
||||
}
|
||||
|
||||
// builderCommand issues a build machine its own broker credential.
|
||||
//
|
||||
// **A build machine is not a node**, and giving it a node's account would let it read another
|
||||
@@ -473,7 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
return "", err
|
||||
}
|
||||
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -812,7 +823,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
|
||||
@@ -567,13 +567,49 @@ func sourceWords(repository, seat string) string {
|
||||
return buildSource{Repository: repository, Seat: seat}.String()
|
||||
}
|
||||
|
||||
// firstBuildWarrant is what the controller checked before it asks the first build of the modules a merge adds to a
|
||||
// repository nothing is built from: the merge was made on the operator's Approve of its head (approvedAnnounced). It
|
||||
// travels in the context of that ask alone, in this process, so no verb carries one.
|
||||
type firstBuildWarrant struct {
|
||||
repository string // owner/name on the git seat
|
||||
paths map[string]bool
|
||||
why string
|
||||
}
|
||||
|
||||
type firstBuildWarrantKey struct{}
|
||||
|
||||
// withFirstBuildWarrant is ctx carrying the warrant for asking the first builds of paths in repository.
|
||||
func withFirstBuildWarrant(ctx context.Context, repository string, paths []string, why string) context.Context {
|
||||
w := &firstBuildWarrant{repository: repository, paths: map[string]bool{}, why: why}
|
||||
for _, p := range paths {
|
||||
w.paths[strings.Trim(p, "/")] = true
|
||||
}
|
||||
return context.WithValue(ctx, firstBuildWarrantKey{}, w)
|
||||
}
|
||||
|
||||
func firstBuildWarrantIn(ctx context.Context) *firstBuildWarrant {
|
||||
w, _ := ctx.Value(firstBuildWarrantKey{}).(*firstBuildWarrant)
|
||||
return w
|
||||
}
|
||||
|
||||
// covers says the warrant is for exactly this ask: the repository on the git seat, and a module path the merge holds.
|
||||
func (w *firstBuildWarrant) covers(source buildSource, path string) bool {
|
||||
return w != nil && source.Seat == gitSeat && strings.EqualFold(strings.Trim(source.Repository, "/"), w.repository) &&
|
||||
w.paths[strings.Trim(path, "/")]
|
||||
}
|
||||
|
||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
||||
func verbMayAsk(ctx context.Context, source buildSource, url string) error {
|
||||
func verbMayAsk(ctx context.Context, source buildSource, url, path string) error {
|
||||
if startedAtTheTerminal() {
|
||||
return nil
|
||||
}
|
||||
// **Or the first build of a module a merge adds on the operator's Approve** (novox/hq ADR 0432, issue 1146): the
|
||||
// controller's own ask, carrying the warrant it checked, for that repository and the module paths the merge holds.
|
||||
if firstBuildWarrantIn(ctx).covers(source, path) {
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -134,18 +134,18 @@ func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T)
|
||||
ctx := t.Context()
|
||||
t.Setenv(verbVar, "build")
|
||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git", "")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||
}
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git", ""); err != nil {
|
||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "", ""); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -317,7 +317,7 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
||||
}
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, _, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
if id != a.merge.AskID {
|
||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
||||
}
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -32,3 +37,43 @@ func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 1204: builds says the source fingerprint the builder recorded, the one tool that reads it.
|
||||
func TestBuildsSaysTheSourceFingerprint(t *testing.T) {
|
||||
at := time.Date(2026, 10, 19, 10, 40, 0, 0, time.UTC)
|
||||
built := inventory.Build{ID: "build-1", Module: "mailu", Commit: "b4527d9091e972f28cb365623d9280045e7741a3", On: "ace",
|
||||
Repository: "novox/mesh-catalog", At: at, ReuseKey: "r1:abc",
|
||||
SourceFingerprint: "src1:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}
|
||||
|
||||
// 1204/AC1
|
||||
var out bytes.Buffer
|
||||
printBuild(&out, built)
|
||||
lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n")
|
||||
if last := lines[len(lines)-1]; last != " source "+built.SourceFingerprint {
|
||||
t.Errorf("last line %q; want the source line after the reuse key:\n%s", last, out.String())
|
||||
}
|
||||
if !strings.Contains(lines[len(lines)-2], "reuse key r1:abc") {
|
||||
t.Errorf("the source line does not follow the reuse key:\n%s", out.String())
|
||||
}
|
||||
|
||||
// 1204/AC2
|
||||
without := built
|
||||
without.SourceFingerprint = ""
|
||||
var plain bytes.Buffer
|
||||
printBuild(&plain, without)
|
||||
if strings.Contains(plain.String(), "source") {
|
||||
t.Errorf("a build with no fingerprint says a source line:\n%s", plain.String())
|
||||
}
|
||||
if want := strings.Join(lines[:len(lines)-1], "\n") + "\n"; plain.String() != want {
|
||||
t.Errorf("the other lines changed:\n%s\nwant\n%s", plain.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1215/AC4: what the builder said of the inputs is what the controller keeps of the build.
|
||||
func TestTheControllerKeepsTheInputsABuilderSaid(t *testing.T) {
|
||||
in := &sdkdelivery.SourceInputs{Module: "mailu", Tree: "t", Toolchains: map[string]string{"code": "ts sha256:aa 01"}}
|
||||
kept := buildFrom(link.BuildResult{ID: "build-1", Repository: "r", Commit: "c", SourceFingerprint: "src1:x", SourceInputs: in})
|
||||
if !reflect.DeepEqual(kept.SourceInputs, in) {
|
||||
t.Errorf("the controller keeps %+v, the builder said %+v", kept.SourceInputs, in)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1212/AC1 and AC2: the check request names the pulls bucket of each module claiming the delivery seat, in the order
|
||||
// of the claimants' names.
|
||||
func TestACheckNamesThePullsBucketsOfTheSeatsClaimants(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
entries []inventory.Entry
|
||||
want []string
|
||||
}{
|
||||
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")}, []string{"mesh-delivery-gate_pulls"}},
|
||||
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery"), namedOnly("gitea")}, []string{"mesh-delivery_pulls"}},
|
||||
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
||||
"none": {[]inventory.Entry{namedOnly("gitea")}, nil},
|
||||
} {
|
||||
if got := pullsBucketsOf(c.entries); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: %v, want %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1212/AC1 and AC2: the check a request carries names those buckets (the wiring, not only the helper).
|
||||
func TestTheCheckRequestCarriesThePullsBuckets(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
entries []inventory.Entry
|
||||
want []string
|
||||
}{
|
||||
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}, []string{"mesh-delivery-gate_pulls"}},
|
||||
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls"}},
|
||||
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
||||
} {
|
||||
got := checkOf(link.PullUpdated{Owner: "novox", Repo: "a", Number: 7}, checkScope{}, c.entries, nil, nil, nil, nil)
|
||||
if !reflect.DeepEqual(got.Pulls, c.want) || got.Owner != "novox" || got.Number != 7 {
|
||||
t.Errorf("%s: %+v, want pulls %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
@@ -12,6 +14,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -140,18 +143,26 @@ func sourceOwner(repository string) string {
|
||||
|
||||
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
|
||||
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
_, err := f.pullUpdated(ctx, p, os.Stdout)
|
||||
return err
|
||||
}
|
||||
|
||||
// pullUpdated is PullUpdated saying what it did to w. It returns the id of the check it asked of the build seat, or
|
||||
// "" when it said a verdict itself and asked nothing. A command that answers JSON on standard output (`delivery
|
||||
// check`) gives it standard error, so its output stays the document alone (novox/hq issue 1311).
|
||||
func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Writer) (string, error) {
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull
|
||||
@@ -160,12 +171,12 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA}
|
||||
if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone},
|
||||
map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil {
|
||||
return err
|
||||
return "", err
|
||||
} else if cyclic {
|
||||
failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA)
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary)
|
||||
sayChecked(ctx, failing)
|
||||
return nil
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary)
|
||||
sayChecked(ctx, w, failing)
|
||||
return "", nil
|
||||
}
|
||||
// **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403
|
||||
// clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is
|
||||
@@ -178,7 +189,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Print(planText(plan))
|
||||
fmt.Fprint(w, planText(plan))
|
||||
direct := directVerdict(p, plan)
|
||||
if composed {
|
||||
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
||||
@@ -186,19 +197,19 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
}
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
sayChecked(ctx, w, direct)
|
||||
return "", nil
|
||||
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
|
||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
||||
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
||||
sayChecked(ctx, direct)
|
||||
return nil
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
||||
sayChecked(ctx, w, direct)
|
||||
return "", nil
|
||||
}
|
||||
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
request.Check.Composed = composed
|
||||
@@ -207,11 +218,11 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
||||
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
||||
@@ -225,9 +236,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||
scope.Width, scope.Tiers)
|
||||
}
|
||||
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit, seat, what, request.ID)
|
||||
return nil
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1).
|
||||
@@ -305,13 +316,19 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Running: runningBy(ctx)},
|
||||
Check: checkOf(p, scope, entries, beside, builds, namedForTheCheck(ctx, p, builds), runningBy(ctx)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkOf is the check a request carries: the pull request, what the change touches, and the buckets of the pull
|
||||
// requests' keys of the modules claiming the delivery seat (novox/hq ADR 0437 §3, issue 1212).
|
||||
func checkOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry, beside map[string]link.CheckedOut,
|
||||
builds []link.CheckBuild, named []link.CheckedBuild, running map[string]string) *link.CheckRequest {
|
||||
return &link.CheckRequest{Pulls: pullsBucketsOf(entries), Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Trunk: p.BaseSHA, Ready: p.Ready, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents,
|
||||
New: scope.New, Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: named, Running: running}
|
||||
}
|
||||
|
||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||
// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module.
|
||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
@@ -521,7 +538,7 @@ func checked(ctx context.Context, result link.BuildResult) {
|
||||
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
||||
return
|
||||
}
|
||||
sayChecked(ctx, checkedOf(result))
|
||||
sayChecked(ctx, os.Stdout, checkedOf(result))
|
||||
}
|
||||
|
||||
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
|
||||
@@ -637,7 +654,7 @@ func prefixedAll(prefix string, items []string) []string {
|
||||
}
|
||||
|
||||
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
||||
func sayChecked(ctx context.Context, c link.Checked) {
|
||||
func sayChecked(ctx context.Context, w io.Writer, c link.Checked) {
|
||||
repo := "none"
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
@@ -646,7 +663,7 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
||||
if c.Trunk != "" {
|
||||
judged = " merged into trunk " + c.Trunk
|
||||
}
|
||||
fmt.Printf("%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
fmt.Fprintf(w, "%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
@@ -658,6 +675,16 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
|
||||
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
||||
fmt.Fprintf(w, "%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// pullsBucketsOf are the buckets of the pull requests' keys a check names for the build agent: one per module
|
||||
// claiming the delivery seat, in the order of their names (novox/hq ADR 0437 §3, issue 1212).
|
||||
func pullsBucketsOf(entries []inventory.Entry) []string {
|
||||
var out []string
|
||||
for _, m := range claimantsOf(entries, catalogue.DeliverySeat) {
|
||||
out = append(out, broker.BucketName(m, "pulls"))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
+145
-33
@@ -42,13 +42,53 @@ import (
|
||||
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
|
||||
// go`, `delivery stop` and `delivery walks`.
|
||||
|
||||
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
|
||||
// onTheControllersPath are the modules, by name, whose walk never waits for the delivery's owner, and what each is.
|
||||
// The delivery's owner is not among them: it is whichever module claims the seat (ownPathOf, novox/hq ADR 0437 §3).
|
||||
var onTheControllersPath = map[string]string{
|
||||
"mesh-controller": "the controller",
|
||||
"mesh-host": "the node-engine",
|
||||
"node-tools": "the node tools",
|
||||
"nats": "the bus",
|
||||
catalogue.DeliverySeat: "the delivery's owner",
|
||||
"mesh-controller": "the controller",
|
||||
"mesh-host": "the node-engine",
|
||||
"node-tools": "the node tools",
|
||||
"nats": "the bus",
|
||||
}
|
||||
|
||||
// claimantsOf are the modules among the catalogued entries that claim a seat, each once, in name order.
|
||||
func claimantsOf(entries []inventory.Entry, seat string) []string {
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.Manifest.ClaimsSeat(seat) && !slices.Contains(out, e.Manifest.Module) {
|
||||
out = append(out, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ownPathOf is the controller's own path among these entries: the modules by name, and the modules that claim
|
||||
// the delivery seat, which is how the delivery's owner is found (novox/hq ADR 0437 §3).
|
||||
func ownPathOf(entries []inventory.Entry) map[string]string {
|
||||
own := make(map[string]string, len(onTheControllersPath)+1)
|
||||
for m, what := range onTheControllersPath {
|
||||
own[m] = what
|
||||
}
|
||||
for _, m := range claimantsOf(entries, catalogue.DeliverySeat) {
|
||||
own[m] = "the delivery's owner"
|
||||
}
|
||||
return own
|
||||
}
|
||||
|
||||
// onTheOwnPath is whether one module, with its manifest at hand, is on the controller's own path.
|
||||
func onTheOwnPath(module string, m catalogue.Manifest) bool {
|
||||
_, named := onTheControllersPath[module]
|
||||
return named || m.ClaimsSeat(catalogue.DeliverySeat)
|
||||
}
|
||||
|
||||
// ownPathNow is ownPathOf the catalogue as it is.
|
||||
func ownPathNow(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ownPathOf(entries), nil
|
||||
}
|
||||
|
||||
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
|
||||
@@ -69,8 +109,9 @@ func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDeliv
|
||||
if !deliverySeatHeld(entries) {
|
||||
return nil
|
||||
}
|
||||
onPath := ownPathOf(entries)
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own {
|
||||
if _, own := onPath[m]; own {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -327,16 +368,17 @@ func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Pla
|
||||
// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds.
|
||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge,
|
||||
now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) {
|
||||
own := false
|
||||
for _, n := range movesOf(again) {
|
||||
if _, on := onTheControllersPath[n]; on {
|
||||
own = true
|
||||
}
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
own := false
|
||||
onPath := ownPathOf(entries)
|
||||
for _, n := range movesOf(again) {
|
||||
if _, on := onPath[n]; on {
|
||||
own = true
|
||||
}
|
||||
}
|
||||
joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
@@ -742,6 +784,27 @@ func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[s
|
||||
return out
|
||||
}
|
||||
|
||||
// changeMember is one change as the planner is asked about it, the same for `delivery plan` and `plans --what-if`.
|
||||
func changeMember(repository, base, head string, paths, removed []string, moduleDirs *[]string) orderMember {
|
||||
m := orderMember{ID: repository + "@" + head, Repository: repository, Base: base, Head: head,
|
||||
Paths: paths, Removed: removed}
|
||||
if moduleDirs != nil {
|
||||
m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// deliveryPlanDocument is the answer of `delivery plan` and of `plans --what-if … --data`: one change's
|
||||
// change plan with its reach, the mesh's own part in it and whether the gate would run. One function, so
|
||||
// the two verbs cannot differ (novox/hq ADR 0450 §2, issue 1324).
|
||||
func deliveryPlanDocument(m orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||
edges []inventory.Edge, policy policyOf) map[string]any {
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
plan := changePlanOf(m.Repository, m.Base, m.Head, s.Reach, entries, policy)
|
||||
return map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
|
||||
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()}
|
||||
}
|
||||
|
||||
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
|
||||
//
|
||||
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
|
||||
@@ -763,6 +826,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
membersFile := set.String("members-file", "", "a file of a group's members, as JSON: the same as --members, for a list "+
|
||||
"too long for an argument (novox/hq issue 1272)")
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
|
||||
"(novox/hq ADR 0406 §3)")
|
||||
@@ -777,6 +842,13 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
var members []orderMember
|
||||
if *membersFile != "" {
|
||||
raw, err := os.ReadFile(*membersFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--members-file: %w", err)
|
||||
}
|
||||
*membersJSON = string(raw)
|
||||
}
|
||||
if *membersJSON != "" {
|
||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||
return fmt.Errorf("the members are not readable: %w", err)
|
||||
@@ -804,22 +876,15 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if *repository == "" || *head == "" && *paths == "" {
|
||||
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
|
||||
}
|
||||
m := orderMember{ID: *repository + "@" + *head, Repository: *repository, Base: *base, Head: *head,
|
||||
Paths: splitList(*paths), Removed: splitList(*removed)}
|
||||
if d := moduleDirsOf(*moduleDirs); d != nil {
|
||||
m.ModuleDirs, m.ModuleDirsSaid = *d, true
|
||||
}
|
||||
m := changeMember(*repository, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs))
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s := pullScope(m.pull(), entries, read, edges)
|
||||
plan := changePlanOf(*repository, *base, *head, s.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
return answer(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
return answer(map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
|
||||
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()})
|
||||
}))
|
||||
case "order":
|
||||
if len(members) == 0 {
|
||||
return errors.New("delivery order --members <json>: a group has members")
|
||||
@@ -837,10 +902,14 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if *group == "" && len(members) == 1 {
|
||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||
// own path for a pull request, run because the delivery's owner asked.
|
||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open: open, oneShot: true}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"rechecked": members[0].ID})
|
||||
return recheckOneHead(members[0].ID, answer, func() (string, error) {
|
||||
var asked string
|
||||
err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||
asked, err = (following{open: open, oneShot: true}).pullUpdated(ctx, members[0].pull(), os.Stderr)
|
||||
return err
|
||||
})
|
||||
return asked, err
|
||||
})
|
||||
}
|
||||
if *group == "" || len(members) < 2 {
|
||||
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
|
||||
@@ -935,7 +1004,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
||||
}
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
"own-path": sortedKeysOf(ownPathOf(entries))})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub)
|
||||
}
|
||||
@@ -1006,9 +1075,6 @@ func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Ti
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownPathWords is the controller's own path as words, for an answer.
|
||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||
|
||||
// theGraph is what the planner reads.
|
||||
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
|
||||
[]inventory.Edge, error) {
|
||||
@@ -1094,7 +1160,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
||||
if failing, cyclic, err := cycleVerdict(ctx, group, members, reach, edges, entries); err != nil {
|
||||
return "", err
|
||||
} else if cyclic {
|
||||
sayChecked(ctx, failing)
|
||||
sayChecked(ctx, os.Stderr, failing)
|
||||
return failing.ID, nil
|
||||
}
|
||||
primary := groupPrimary(members, reach)
|
||||
@@ -1181,8 +1247,32 @@ func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
||||
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
||||
// the walk recorded.
|
||||
//
|
||||
// The plan the announcement is made from is a copy that shares nothing with the walk, taken before the goroutine
|
||||
// starts: the walk goes on writing its steps' maps and pointers while the announcement is marshalled and its
|
||||
// touches read (novox/hq issue 1260), and it says the plan as it was when this was called.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, facts, p)
|
||||
kept, err := copyOfPlan(p)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
|
||||
return
|
||||
}
|
||||
go publishPlanMoved(ctx, bus, facts, kept)
|
||||
}
|
||||
|
||||
// copyOfPlan is the plan with nothing shared with the original: its JSON round trip, which is all of it but what
|
||||
// the plan never keeps (WalkSaid), carried over.
|
||||
func copyOfPlan(p inventory.Plan) (inventory.Plan, error) {
|
||||
raw, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
var kept inventory.Plan
|
||||
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
kept.WalkSaid = p.WalkSaid
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
@@ -1290,3 +1380,25 @@ func overNATS(conn *nats.Conn) (link.Bus, error) {
|
||||
}
|
||||
return link.OverNATS{Conn: conn, JS: js}, nil
|
||||
}
|
||||
|
||||
// recheckAnswer is what `delivery check` answers for one head: the id of the check asked of the build seat (empty when
|
||||
// the controller said the verdict itself and asked nothing), which mesh-delivery reads, and the head rechecked.
|
||||
func recheckAnswer(asked, member string) map[string]any {
|
||||
return map[string]any{"asked": asked, "rechecked": member}
|
||||
}
|
||||
|
||||
// recheckOneHead runs a one-head `delivery check` and answers it as data. Standard output is the answer's JSON alone,
|
||||
// or the seat verb reads no answer from it (novox/hq issue 1311): while the check runs, whatever it prints to standard
|
||||
// output (the controller's own path prints its prose there, as do the stores it reads) goes to standard error.
|
||||
func recheckOneHead(member string, answer func(any) error, ask func() (string, error)) error {
|
||||
real := os.Stdout
|
||||
os.Stdout = os.Stderr
|
||||
asked, err := func() (string, error) {
|
||||
defer func() { os.Stdout = real }()
|
||||
return ask()
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(recheckAnswer(asked, member))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `delivery check` with one head answers the id of the check it asked, as data (novox/hq issue 1311): the prose
|
||||
// the check says goes to standard error, so standard output is the document alone and answerOf reads it.
|
||||
func TestDeliveryCheckOfOneHeadAnswersTheRequestID(t *testing.T) {
|
||||
argv := []string{"delivery", "check", "--members", "[]"}
|
||||
doc, err := json.Marshal(recheckAnswer("check-42", "novox/x@abc"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Seen failing first: the prose the command used to print on standard output before the document.
|
||||
polluted := append([]byte("novox/x#7 (abc): asked the build seat as check-42\n"), doc...)
|
||||
if got := answerOf(argv, polluted, "", true); got.Answer != nil {
|
||||
t.Fatalf("prose before the document should leave no answer; got %v", got.Answer)
|
||||
}
|
||||
got := answerOf(argv, doc, "novox/x#7 (abc): asked the build seat as check-42\n", true)
|
||||
m, ok := got.Answer.(map[string]any)
|
||||
if !ok || m["asked"] != "check-42" || m["rechecked"] != "novox/x@abc" {
|
||||
t.Fatalf("the answer is %v; want asked check-42 and rechecked novox/x@abc", got.Answer)
|
||||
}
|
||||
if !strings.Contains(got.Output, "asked the build seat") {
|
||||
t.Fatalf("the prose is still said in the text: %q", got.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// The text of the other callers does not change: sayChecked writes the same line to the writer it is given, and
|
||||
// the controller's own path still gives PullUpdated standard output.
|
||||
func TestSayCheckedWritesTheSameLineToItsWriter(t *testing.T) {
|
||||
var w bytes.Buffer
|
||||
sayChecked(context.Background(), &w, link.Checked{ID: "c1", Owner: "novox", Repo: "x", Number: 7, Commit: "0123456789abcdef",
|
||||
Verdict: "pass", Summary: "fine"})
|
||||
want := "c1: novox/x#7 at 01234567 checked on a machine that did not say which: gate PASS — fine; repository none\n"
|
||||
if w.String() != want {
|
||||
t.Fatalf("sayChecked said %q; want %q", w.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// Whatever the check prints to standard output while it runs (checks.go namedForTheCheck, the inventory's pointer
|
||||
// repair) must not reach the answer's standard output: it is exactly one JSON document (novox/hq issue 1311).
|
||||
func TestOneHeadRecheckLeavesStandardOutputTheAnswerAlone(t *testing.T) {
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e, ew, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
realOut, realErr := os.Stdout, os.Stderr
|
||||
os.Stdout, os.Stderr = w, ew
|
||||
runErr := recheckOneHead("novox/x@abc", func(v any) error { return json.NewEncoder(os.Stdout).Encode(v) },
|
||||
func() (string, error) {
|
||||
fmt.Println(" the earlier checks' builds could not be read, so the check builds")
|
||||
fmt.Fprintln(os.Stdout, "repaired a pointer")
|
||||
return "check-42", nil
|
||||
})
|
||||
os.Stdout, os.Stderr = realOut, realErr
|
||||
w.Close()
|
||||
ew.Close()
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
var out, errOut bytes.Buffer
|
||||
_, _ = out.ReadFrom(r)
|
||||
_, _ = errOut.ReadFrom(e)
|
||||
var doc map[string]any
|
||||
if err := json.Unmarshal(out.Bytes(), &doc); err != nil || doc["asked"] != "check-42" || doc["rechecked"] != "novox/x@abc" {
|
||||
t.Fatalf("standard output %q is not the answer alone (%v)", out.String(), err)
|
||||
}
|
||||
if !strings.Contains(errOut.String(), "repaired a pointer") {
|
||||
t.Fatalf("the prose should be on standard error: %q", errOut.String())
|
||||
}
|
||||
}
|
||||
@@ -55,6 +55,17 @@ type stalledLine struct {
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
// Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty
|
||||
// or anything else is a warning, as before.
|
||||
Severity string `json:"severity,omitempty"`
|
||||
}
|
||||
|
||||
// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise.
|
||||
func (l stalledLine) severity() conditions.Severity {
|
||||
if l.Severity == string(conditions.Urgent) {
|
||||
return conditions.Urgent
|
||||
}
|
||||
return conditions.Warning
|
||||
}
|
||||
|
||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||
@@ -165,7 +176,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
||||
Severity: conditions.Warning,
|
||||
Severity: l.severity(),
|
||||
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
||||
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
||||
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
func severityRaisedFor(t *testing.T, line string) conditions.Severity {
|
||||
t.Helper()
|
||||
var l stalledLine
|
||||
if err := json.Unmarshal([]byte(line), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := stalledObservations([]stalledLine{l})
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%d conditions", len(got))
|
||||
}
|
||||
return got[0].Severity
|
||||
}
|
||||
|
||||
func TestAnUrgentStalledLineRaisesAnUrgentCondition(t *testing.T) { // 1278/AC1
|
||||
line := `{"id":"novox/mesh-delivery#5","state":"held","for":"3m0s","bound":"0s","h2":"close","says":"held","severity":"urgent"}`
|
||||
if got := severityRaisedFor(t, line); got != conditions.Urgent {
|
||||
t.Errorf("an urgent line raised %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStalledLineWithoutOrWithAnUnknownSeverityRaisesAWarning(t *testing.T) { // 1278/AC2
|
||||
for _, extra := range []string{``, `,"severity":""`, `,"severity":"critical"`, `,"severity":"warning"`} {
|
||||
line := `{"id":"novox/a#1","state":"unanswered","for":"3m0s","bound":"2m0s","h2":"close","says":"x"` + extra + `}`
|
||||
if got := severityRaisedFor(t, line); got != conditions.Warning {
|
||||
t.Errorf("the line %s raised %q", extra, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -120,6 +120,16 @@ var probeRegistry = []probe{
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a
|
||||
// node's presence, which the silent check takes.
|
||||
{ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " +
|
||||
"not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156",
|
||||
Kind: kindStuck, Phase: 3, run: probeStuck},
|
||||
// A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the
|
||||
// operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8).
|
||||
{ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " +
|
||||
"minutes for mesh-delivery's announcement of the operator's Approve of its head", From: "ADR 0432, issue 1146",
|
||||
Kind: kindFirstBuildUnasked, Phase: 3, run: probeFirstBuilds},
|
||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
||||
@@ -178,6 +188,10 @@ var probeRegistry = []probe{
|
||||
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
|
||||
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
|
||||
run: probeBackupHeld},
|
||||
// A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it.
|
||||
{ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " +
|
||||
"bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1,
|
||||
run: probeSendKeyRefused},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
|
||||
@@ -72,8 +72,6 @@ func summarise(ds []inventory.Duration) []durationSummary {
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
@@ -138,7 +136,7 @@ func durationsCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// aForgeAndABuildSeat gives a test what a real ask of a build needs: the forge holding the git seat, assigned to anchor, so
|
||||
// the clone URL composes; the identity store; the build seat's queue on a real bus; and the environment of the serving controller, which is
|
||||
// never the controller's terminal (a verb's, in effect: startedAtTheTerminal is false).
|
||||
func aForgeAndABuildSeat(t *testing.T, open *stores) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, _ := aBuildQueue(t)
|
||||
// The ask signs what it sends with the controller's identity, as the serving controller does.
|
||||
identity.ForTest(t)
|
||||
forge, err := catalogue.ParseManifest([]byte(`{"module":"gitea","version":"1",
|
||||
"claims":[{"name":"git","scope":"mesh"}],"provides":[{"name":"git","scope":"mesh"}],
|
||||
"serves":{"git":{"scheme":"http","port":3000}}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, forge)
|
||||
if _, err := assign(t.Context(), open, "anchor", "gitea"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(servedVar, "1")
|
||||
return js
|
||||
}
|
||||
|
||||
// 1146/AC9, through the real ask (askABuild, buildOneAsked, verbMayAsk) and a real bus: the first build of a module a
|
||||
// merge adds on the operator's Approve is queued on the build seat; the same ask without that warrant, and the warrant's
|
||||
// ask for a path the merge does not hold, are refused by the guard of a build through a verb.
|
||||
func TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js := aForgeAndABuildSeat(t, open)
|
||||
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
|
||||
// Without the warrant the guard refuses, whatever the repository holds.
|
||||
_, err := askABuild(ctx, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", "main")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("1146/AC9: a first build asked without the warrant was not refused: %v", err)
|
||||
}
|
||||
// And a path the merge does not hold is refused on the warrant.
|
||||
warranted := withFirstBuildWarrant(ctx, "novox/mesh-delivery", []string{""}, "the Approve")
|
||||
if _, err := askABuild(warranted, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", "main"); !errors.As(err, &policy) {
|
||||
t.Fatalf("1146/AC9: the warrant covered a path the merge does not hold: %v", err)
|
||||
}
|
||||
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(queue.Asks) != 1 {
|
||||
t.Fatalf("1146/AC9: the build queue holds %+v, want the one first build", queue)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1146/AC9: the warrant the controller checked covers exactly the repository on the git seat and the module paths the
|
||||
// merge holds; nothing else passes verbMayAsk on it.
|
||||
func TestTheFirstBuildWarrantCoversOnlyItsRepositoryAndItsModulePaths(t *testing.T) {
|
||||
ctx := withFirstBuildWarrant(t.Context(), "novox/mesh-delivery", []string{"", "view"}, "the Approve")
|
||||
w := firstBuildWarrantIn(ctx)
|
||||
for _, c := range []struct {
|
||||
source buildSource
|
||||
path string
|
||||
covers bool
|
||||
}{
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", true},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "/view/", true},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", false},
|
||||
{buildSource{Repository: "agent/mesh-delivery", Seat: "git"}, "", false},
|
||||
{buildSource{Repository: "novox/mesh-delivery"}, "", false},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "elsewhere"}, "", false},
|
||||
} {
|
||||
if got := w.covers(c.source, c.path); got != c.covers {
|
||||
t.Errorf("1146/AC9: %v at %q covered = %v, want %v", c.source, c.path, got, c.covers)
|
||||
}
|
||||
}
|
||||
if firstBuildWarrantIn(t.Context()).covers(buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "") {
|
||||
t.Error("1146/AC9: a context with no warrant covers an ask")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// firstBuilds asks the first build of every module a merge adds to a repository no registered module is built from, when
|
||||
// the merge was announced as made on the operator's Approve of that head (novox/hq ADR 0432, issue 1146): the modules'
|
||||
// directories are the ones the announcement says hold a module.json at the merge commit, as touchedBy reads them. The
|
||||
// merge is kept, so the build's outcome is judged by the rule that registers it (mayRegisterFrom, approvedMove).
|
||||
//
|
||||
// **Complete in either order, with no timer** (ADR 0426): when mesh-delivery's word is kept first, it is found here; when
|
||||
// the forge's merge is heard first, the merge is kept as a first merge waiting for the word, and keepingDeliveryMerges
|
||||
// runs this again when the word is kept (resumeFirstBuilds). A word that says no (a Decline, another head) asks nothing and
|
||||
// says why. Answers whether the merge was dealt with here.
|
||||
func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, now time.Time) (bool, error) {
|
||||
inv := f.open.inventory
|
||||
repository := m.Owner + "/" + m.Repo
|
||||
added, why, err := firstBuildsOf(ctx, inv, m, entries, read)
|
||||
switch {
|
||||
case errors.Is(err, errFirstNotApproved):
|
||||
fmt.Printf("%s merged into %s (%.8s) adds %s, and no build is asked: %v\n", repository, m.Base, m.Commit,
|
||||
strings.Join(addedDirs(m, entries, read), ", "), err)
|
||||
return true, nil
|
||||
case errors.Is(err, errNotAnnounced):
|
||||
// Kept, waiting for mesh-delivery's word: a restart keeps it, and the word's arrival asks the builds.
|
||||
if kerr := keepFirstMerge(ctx, inv, m, now, true); kerr != nil {
|
||||
return true, notNow(kerr)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) adds %s; it is kept, and the first build is asked when mesh-delivery "+
|
||||
"announces the merge on the operator's Approve\n", repository, m.Base, m.Commit,
|
||||
strings.Join(addedDirs(m, entries, read), ", "))
|
||||
return true, nil
|
||||
case err != nil:
|
||||
return true, notNow(err)
|
||||
case len(added) == 0:
|
||||
return false, nil
|
||||
}
|
||||
if err := askFirstBuilds(ctx, inv, m, now, added, why); err != nil {
|
||||
return true, notNow(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keepFirstMerge keeps a first merge as the controller heard it, waiting (no plan answers it).
|
||||
func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, waitingForWord bool) error {
|
||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||
if err != nil {
|
||||
merged = now
|
||||
}
|
||||
event, err := json.Marshal(keptMerge{SourceMoved: m, FirstBuild: waitingForWord})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = inv.AddMerge(ctx, inventory.BatchedMerge{Repository: m.Owner + "/" + m.Repo, Branch: m.Base, Commit: m.Commit,
|
||||
Merged: merged.UTC(), Heard: now, Event: event})
|
||||
return err
|
||||
}
|
||||
|
||||
// askFirstBuilds keeps the merge, asks the builds, and marks the merge as no longer waiting for them.
|
||||
func askFirstBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, added []string,
|
||||
why string) error {
|
||||
repository := m.Owner + "/" + m.Repo
|
||||
if err := keepFirstMerge(ctx, inv, m, now, false); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so the "+
|
||||
"first build of %s is asked\n", repository, m.Base, m.Commit, why, strings.Join(added, ", "))
|
||||
// The ask carries the warrant just checked, for this repository and these module paths alone: the guard of a build
|
||||
// through a verb (verbMayAsk) lets exactly that through.
|
||||
paths := make([]string, 0, len(added))
|
||||
for _, dir := range added {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
paths = append(paths, dir)
|
||||
}
|
||||
askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, m, buildSource{Repository: repository, Seat: gitSeat}, added)
|
||||
// Answered: it no longer waits, so the word's arrival again asks nothing twice.
|
||||
return inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false)
|
||||
}
|
||||
|
||||
// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull
|
||||
// request's head: the other order of the same two events.
|
||||
func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error {
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
inv := f.open.inventory
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||
k.HeadSHA != d.HeadSHA {
|
||||
continue
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read)
|
||||
switch {
|
||||
case errors.Is(err, errFirstNotApproved):
|
||||
fmt.Printf("%s/%s merged into %s (%.8s) was waiting for mesh-delivery's word, and no build is asked: %v\n",
|
||||
k.Owner, k.Repo, k.Base, k.Commit, err)
|
||||
if aerr := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); aerr != nil {
|
||||
return notNow(aerr)
|
||||
}
|
||||
case err != nil:
|
||||
return notNow(err)
|
||||
case len(added) > 0:
|
||||
if err := askFirstBuilds(ctx, inv, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFirstNotApproved says a merge into a repository the mesh builds nothing from is not one the operator approved.
|
||||
var errFirstNotApproved = errors.New("it is no merge made on the operator's Approve of its head, so a hand `build` " +
|
||||
"registers a module from a repository the mesh does not build yet")
|
||||
|
||||
// addedDirs are the module directories the merge adds to a repository no registered module is built from, none where
|
||||
// the merge is not into the trunk or some module is built from the repository already.
|
||||
func addedDirs(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []string {
|
||||
if m.Base != "main" {
|
||||
return nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && sameRepository(e.Source.Repository, m) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
_, added, _ := touchedBy(nil, entries, m, read)
|
||||
return added
|
||||
}
|
||||
|
||||
// firstBuildsOf is the directories whose first build a merge asks, and the warrant it was made on; none when the merge
|
||||
// is no first merge into a repository the mesh builds nothing from. errFirstNotApproved when it is and the word on record
|
||||
// is no Approve of its head; errNotAnnounced when mesh-delivery's word is not kept yet.
|
||||
func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) ([]string, string, error) {
|
||||
added := addedDirs(m, entries, read)
|
||||
if len(added) == 0 {
|
||||
return nil, "", nil
|
||||
}
|
||||
why, err := approvedAnnounced(ctx, inv, m.Owner, m.Repo, keptMerge{SourceMoved: m})
|
||||
switch {
|
||||
case err == nil:
|
||||
return added, why, nil
|
||||
case errors.Is(err, errWordNotAnApprove):
|
||||
return nil, "", fmt.Errorf("%w (%v)", errFirstNotApproved, err)
|
||||
}
|
||||
// Not announced yet, or the bus or the store could not be asked: the caller waits or tries again, and asks nothing.
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
|
||||
const probeFirstBuildsID = "D18"
|
||||
|
||||
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
|
||||
const kindFirstBuildUnasked = "first-build-unasked"
|
||||
|
||||
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
|
||||
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
|
||||
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
|
||||
const firstBuildWaitBound = 10 * time.Minute
|
||||
|
||||
// firstBuildObservations are the conditions of the first merges waiting past the bound, one for each; a merge answered is
|
||||
// no longer listed as waiting, so its condition clears with the next look.
|
||||
func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
|
||||
continue
|
||||
}
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
for_ := now.Sub(w.Heard).Round(time.Minute)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge,
|
||||
ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), Kind: kindFirstBuildUnasked, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the merge %.8s of %s adds modules the mesh builds nothing from yet, and has waited %s for "+
|
||||
"mesh-delivery's announcement of the operator's Approve of its head: no build is asked", k.Commit, repository, for_),
|
||||
Said: fmt.Sprintf("%s at %s waits for mesh-delivery's word on pull request %d (head %.8s)", repository, k.Commit,
|
||||
k.Number, k.HeadSHA),
|
||||
Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository),
|
||||
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+
|
||||
"only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+
|
||||
"announcement has come in %s. The merge is kept, and the builds are asked the moment the announcement is kept.",
|
||||
k.Commit, repository, for_),
|
||||
Resolved: "mesh-delivery announces the merge, or a hand `build` registers the modules",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
|
||||
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
waiting, err := d.open.inventory.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return firstBuildObservations(waiting, time.Now().UTC()), nil
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1146/AC7, whole against a real bus and the store, in both orders of the two events (novox/hq ADR 0432, issue 1146):
|
||||
// mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's merge,
|
||||
// heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the controller
|
||||
// consumes with the router's record of the ask in its bucket, come in either order; each way the first build of the
|
||||
// module the merge commit holds is asked exactly once, its outcome registers the module, and it is in the catalogue.
|
||||
// Without the Approve in the record nothing is asked. Store-backed: skips without one.
|
||||
func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
wordFirst bool
|
||||
approved bool // the router's record holds the operator's Approve of the merged head
|
||||
wantRegiter bool
|
||||
}{
|
||||
{what: "the forge's merge before mesh-delivery's word", approved: true, wantRegiter: true},
|
||||
{what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegiter: true},
|
||||
{what: "a word whose record holds no Approve", wordFirst: true},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
// The real ask, guard and queue included: only its answers are kept, so the build's outcome can be matched.
|
||||
aForgeAndABuildSeat(t, open)
|
||||
// Written by the controller's merge path on its own goroutine, read by the test: under one lock.
|
||||
var mu sync.Mutex
|
||||
var asks [][3]string
|
||||
var ids []string
|
||||
asked := func() [][3]string { mu.Lock(); defer mu.Unlock(); return slices.Clone(asks) }
|
||||
realAsk := askABuild
|
||||
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id, err := realAsk(c, source, path, ref)
|
||||
if err == nil {
|
||||
mu.Lock()
|
||||
asks = append(asks, [3]string{source.Repository, path, ref})
|
||||
ids = append(ids, id)
|
||||
mu.Unlock()
|
||||
}
|
||||
return id, err
|
||||
}
|
||||
t.Cleanup(func() { askABuild = realAsk })
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
if c.approved {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
following := following{open: open}
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims, kept: following.resumeFirstBuilds}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.AssertServingConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
|
||||
word := func() {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the word being kept", func() bool {
|
||||
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, head)
|
||||
return err == nil && kept
|
||||
})
|
||||
time.Sleep(500 * time.Millisecond) // the keeper's hook runs as the word is kept
|
||||
}
|
||||
forge := func() {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := following.SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if c.wordFirst {
|
||||
word()
|
||||
forge()
|
||||
} else {
|
||||
forge()
|
||||
if len(asked()) != 0 {
|
||||
t.Fatalf("1146/AC7: the merge asked %v before mesh-delivery's word was kept", asked())
|
||||
}
|
||||
word()
|
||||
}
|
||||
if !c.wantRegiter {
|
||||
if len(asked()) != 0 {
|
||||
t.Fatalf("1146/AC7: a merge whose record holds no Approve asked %v", asked())
|
||||
}
|
||||
return
|
||||
}
|
||||
want := [3]string{"novox/mesh-delivery", "", "main"}
|
||||
becomes(t, 10*time.Second, "the first build asked", func() bool { return len(asked()) >= 1 })
|
||||
if got := asked(); len(got) != 1 || got[0] != want {
|
||||
t.Fatalf("1146/AC7: the merge asked %v, want %v once", got, want)
|
||||
}
|
||||
mu.Lock()
|
||||
first := ids[0]
|
||||
mu.Unlock()
|
||||
build := onTrunk(first, "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-view", "version": "1"})
|
||||
build.Commit = commit
|
||||
if _, _, err := takeIn(ctx, inv, build); err != nil {
|
||||
t.Fatalf("1146/AC7: the outcome of the first build was refused: %v", err)
|
||||
}
|
||||
src, err := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||
if err != nil || src.Repository != "novox/mesh-delivery" {
|
||||
t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err)
|
||||
}
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); !known {
|
||||
t.Fatal("1146/AC7: the module is not in the catalogue")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) {
|
||||
t.Helper()
|
||||
entries, err := inv.Catalogued(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == module {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Entry{}, false
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1146/AC6: the first merge into a repository no registered module is built from asks the first build of every module
|
||||
// the merge commit holds when it was made on the operator's Approve of that head, and asks nothing without it.
|
||||
func TestAFirstMergeIntoANewRepositoryAsksItsModulesBuildsOnlyOnTheApprove(t *testing.T) {
|
||||
const head = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
now := time.Date(2026, 10, 19, 0, 6, 0, 0, time.UTC)
|
||||
mergedAt := now.Add(-time.Hour).Format(time.RFC3339Nano)
|
||||
heard := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: "0177ea7a",
|
||||
Number: 1, MergedAt: mergedAt, ModuleDirsSaid: true, ModuleDirs: []string{"watchman", "view"},
|
||||
Paths: []string{"view/module.json", "watchman/module.json", "README.md"}}
|
||||
built := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git"}}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge, *link.SourceMoved, *[]inventory.Entry)
|
||||
unkept bool
|
||||
dirs []string
|
||||
refused bool
|
||||
waits bool
|
||||
}{
|
||||
{what: "the Approve of the merged head", dirs: []string{"view", "watchman"}},
|
||||
{what: "a Decline", refused: true, change: func(a *anApprovedMerge, _ *link.SourceMoved, _ *[]inventory.Entry) {
|
||||
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
||||
}},
|
||||
{what: "no word of mesh-delivery yet, however long ago the merge was made", unkept: true, waits: true},
|
||||
{what: "a module already built from the repository", change: func(_ *anApprovedMerge, _ *link.SourceMoved, e *[]inventory.Entry) {
|
||||
*e = append(*e, inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}})
|
||||
}},
|
||||
{what: "a merge into another branch", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { m.Base = "feat" }},
|
||||
{what: "a merge that holds no module.json", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) {
|
||||
m.ModuleDirs, m.Paths = nil, []string{"README.md"}
|
||||
}},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
a.merge.Number = 1
|
||||
m, entries := heard, []inventory.Entry{built}
|
||||
if c.change != nil {
|
||||
c.change(&a, &m, &entries)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
dirs, _, err := firstBuildsOf(t.Context(), nil, m, entries, nil)
|
||||
switch {
|
||||
case c.waits && !errors.Is(err, errNotAnnounced):
|
||||
t.Fatalf("1146/AC6: %s: want a wait for the word, got %v", c.what, err)
|
||||
case c.refused && !errors.Is(err, errFirstNotApproved):
|
||||
t.Fatalf("1146/AC6: %s: want the Approve missing, got %v (dirs %v)", c.what, err, dirs)
|
||||
case !c.refused && !c.waits && err != nil:
|
||||
t.Fatalf("1146/AC6: %s: %v", c.what, err)
|
||||
case !slices.Equal(dirs, c.dirs):
|
||||
t.Fatalf("1146/AC6: %s: first builds asked of %v, want %v", c.what, dirs, c.dirs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1146/AC8: a first merge waiting past the bound for mesh-delivery's word is said, naming the repository, the commit and
|
||||
// the missing word; one inside the bound, one that is no first merge, and one answered (no longer waiting) are not.
|
||||
func TestAFirstMergeWaitingPastItsBoundIsSaidAndClearsWhenAnswered(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC)
|
||||
waiting := func(commit string, first bool, heard time.Time) inventory.BatchedMerge {
|
||||
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
|
||||
Commit: commit, Number: 1, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: first})
|
||||
return inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, Heard: heard, Event: event}
|
||||
}
|
||||
got := firstBuildObservations([]inventory.BatchedMerge{
|
||||
waiting("0177ea7a00", true, now.Add(-firstBuildWaitBound-time.Minute)),
|
||||
waiting("1111111100", true, now.Add(-time.Minute)),
|
||||
waiting("2222222200", false, now.Add(-time.Hour)),
|
||||
}, now)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("1146/AC8: %d conditions raised, want the one merge past the bound: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != kindFirstBuildUnasked || o.Scope != conditions.ScopeMerge || o.Severity != conditions.Warning ||
|
||||
!strings.Contains(o.Summary, "novox/mesh-delivery") || !strings.Contains(o.Summary, "0177ea7a") ||
|
||||
!strings.Contains(o.Summary, "announcement of the operator's Approve") {
|
||||
t.Fatalf("1146/AC8: the condition is %+v", o)
|
||||
}
|
||||
// Answered, the merge is no longer listed as waiting: nothing is observed, and the condition clears.
|
||||
if cleared := firstBuildObservations(nil, now); len(cleared) != 0 {
|
||||
t.Fatalf("1146/AC8: an answered merge still raises %+v", cleared)
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
resolver := entry("dnsmasq", "modules/dnsmasq")
|
||||
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
||||
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
||||
delivery.Manifest.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||
webApp := entry("photos", "modules/photos")
|
||||
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
||||
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
||||
@@ -44,7 +45,7 @@ func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
||||
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-delivery/module.json"}, inventory.ClassCore},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-host/module.json"}, inventory.ClassCore},
|
||||
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
} {
|
||||
if got := plan(c.paths...).Class; got != c.want {
|
||||
|
||||
@@ -69,7 +69,7 @@ var (
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
|
||||
// delivery budget; how many readings its gate takes is gateRuleOf's.
|
||||
func coreModule(module string, m catalogue.Manifest) bool {
|
||||
if _, own := onTheControllersPath[module]; own {
|
||||
if onTheOwnPath(module, m) {
|
||||
return true
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
@@ -106,7 +106,7 @@ func gateRuleOf(module string, m catalogue.Manifest) gateRule {
|
||||
case coreComponent(module) == lease.ComponentNodeTools:
|
||||
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
|
||||
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||
case module == catalogue.DeliverySeat:
|
||||
case m.ClaimsSeat(catalogue.DeliverySeat):
|
||||
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
|
||||
case coreModule(module, m):
|
||||
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
|
||||
}{
|
||||
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
||||
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
||||
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
|
||||
{catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"},
|
||||
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
@@ -56,7 +56,12 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
|
||||
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
||||
func TestTheCoreStaysTheCore(t *testing.T) {
|
||||
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
||||
if !coreModule(m, catalogue.Manifest{Module: m}) {
|
||||
// The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 §3, issue 1210).
|
||||
claims := []catalogue.Claim(nil)
|
||||
if m == catalogue.DeliverySeat {
|
||||
claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||
}
|
||||
if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) {
|
||||
t.Errorf("%s is no longer judged as the core for its class", m)
|
||||
}
|
||||
}
|
||||
@@ -78,7 +83,11 @@ func aGateOn(module, node string, since time.Time) *gateOn {
|
||||
}
|
||||
|
||||
func (o *gateOn) read(at time.Time, h health, why string) {
|
||||
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
|
||||
m := catalogue.Manifest{Module: o.module}
|
||||
if o.module == catalogue.DeliverySeat { // today's module, which claims the seat
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||
}
|
||||
r := gateRuleOf(o.module, m)
|
||||
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
||||
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
||||
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
||||
|
||||
@@ -72,6 +72,10 @@ var handActVerbs = []handActVerb{
|
||||
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
|
||||
// channel that proved who answered, performed by the controller as itself.
|
||||
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
|
||||
// A seat passed by the controller to its only claimant when its holder on record left (novox/hq ADR 0449
|
||||
// §4): the mesh working as decided. A person's own `seat --to` is not recorded under this cause.
|
||||
{Verb: "seat", Decision: "the controller passes a seat held once to its only claimant when its recorded " +
|
||||
"holder is assigned nowhere (ADR 0449 §4)", DecidedFor: []string{"seat-passed"}},
|
||||
{Verb: "broker consumer-reset"},
|
||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||
|
||||
@@ -77,15 +77,11 @@ func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
|
||||
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Last heard (novox/hq ADR 0433 §3, issue 1183): one answer to when the controller last heard a machine, read from the
|
||||
// watchman's presence. The alive message that used to keep last_seen fresh is gone; a report, which is rare, is the
|
||||
// answer only where presence has no key for the machine.
|
||||
|
||||
// presenceReader reads presence.<node>'s entry for each node; a node without an entry is left out.
|
||||
type presenceReader func(ctx context.Context, nodes []string) (map[string]presenceEntry, error)
|
||||
|
||||
// lastHeard is when a machine was last heard: now when presence says it is online, the entry's time when it says offline
|
||||
// (the time it went), and the machine's last report only when presence has no entry for it. Zero is never heard.
|
||||
func lastHeard(report time.Time, entry *presenceEntry, now time.Time) time.Time {
|
||||
switch {
|
||||
case entry == nil:
|
||||
return report
|
||||
case entry.Online:
|
||||
return now
|
||||
default:
|
||||
return entry.At
|
||||
}
|
||||
}
|
||||
|
||||
// nodesHeard returns the nodes with their last-heard time (LastSeen) as lastHeard says it. Unreadable presence is
|
||||
// returned as an error with the nodes unchanged, so a caller says it and shows the reports.
|
||||
func nodesHeard(ctx context.Context, nodes []inventory.Node, read presenceReader, now time.Time) ([]inventory.Node, error) {
|
||||
names := make([]string, len(nodes))
|
||||
for i, n := range nodes {
|
||||
names[i] = n.Name
|
||||
}
|
||||
entries, err := read(ctx, names)
|
||||
if err != nil {
|
||||
return nodes, fmt.Errorf("what the watchman says of the machines' presence cannot be read: %w", err)
|
||||
}
|
||||
out := make([]inventory.Node, len(nodes))
|
||||
for i, n := range nodes {
|
||||
var entry *presenceEntry
|
||||
if e, ok := entries[n.Name]; ok {
|
||||
entry = &e
|
||||
}
|
||||
n.LastSeen = lastHeard(n.LastSeen, entry, now)
|
||||
out[i] = n
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heardNodes is every node, read with its last-heard time from the watchman's presence over a connection of its own. A bus
|
||||
// that cannot be reached is said on standard error and the reports stand, which reads a machine as quiet rather than well.
|
||||
func (h *stores) heardNodes(ctx context.Context) ([]inventory.Node, error) {
|
||||
nodes, err := h.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.withPresence(ctx, nodes), nil
|
||||
}
|
||||
|
||||
// heardNode is one node, read the same way.
|
||||
func (h *stores) heardNode(ctx context.Context, name string) (inventory.Node, error) {
|
||||
n, err := h.inventory.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
return h.withPresence(ctx, []inventory.Node{n})[0], nil
|
||||
}
|
||||
|
||||
func (h *stores) withPresence(ctx context.Context, nodes []inventory.Node) []inventory.Node {
|
||||
address, err := broker.BusAddress()
|
||||
var js *broker.JetStream
|
||||
if err == nil {
|
||||
js, err = broker.Dial(address)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "what the watchman says of the machines' presence cannot be read: %v\n", err)
|
||||
return nodes
|
||||
}
|
||||
defer js.Close()
|
||||
read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
|
||||
return readPresence(ctx, js.Conn(), names)
|
||||
}
|
||||
heard, err := nodesHeard(ctx, nodes, read, time.Now())
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
return heard
|
||||
}
|
||||
|
||||
// heardFromAt says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never spoken did not finish
|
||||
// joining; a node last heard from an hour ago is running an hour-old picture of the mesh.
|
||||
func heardFromAt(n inventory.Node, now time.Time) string {
|
||||
if n.LastSeen.IsZero() {
|
||||
return "never spoken"
|
||||
}
|
||||
if silent := now.Sub(n.LastSeen); silent > SilentFor {
|
||||
return "out of touch " + roughly(silent)
|
||||
}
|
||||
return "here"
|
||||
}
|
||||
|
||||
// notHeardLately is the nodes never heard from or not within the hour: different from failing, for a machine that says
|
||||
// nothing may be new, switched off or unreachable, and none of those is a machine that tried and could not.
|
||||
func notHeardLately(nodes []inventory.Node, now time.Time) []inventory.Node {
|
||||
var out []inventory.Node
|
||||
for _, n := range nodes {
|
||||
if n.LastSeen.IsZero() || now.Sub(n.LastSeen) > time.Hour {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1183/AC5: with the watchman's presence bucket holding a node online and no alive message ever sent, the node reads here
|
||||
// in `nodes` four minutes later by the controller's clock, however old its last report; and an alive message still sent
|
||||
// by a node-engine is published to the same bus without error and changes nothing.
|
||||
func TestAnOnlineNodeReadsHereWithNoAliveMessage(t *testing.T) { // 1183/AC5
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if _, err := broker.RaiseBuckets(js, []broker.Bucket{
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(node string, online bool) {
|
||||
t.Helper()
|
||||
key, err := sdkdelivery.PresenceNodeKey(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := sdkdelivery.EncodePresence(sdkdelivery.Presence{Online: online})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put("ace", true)
|
||||
put("laptop", false)
|
||||
// A node-engine still sending its alive message: harmless.
|
||||
if err := js.Conn().Publish("mesh.control.ace.alive", []byte(`{"node":"ace"}`)); err != nil {
|
||||
t.Fatalf("an alive message could not be published: %v", err)
|
||||
}
|
||||
if err := js.Conn().Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
later := time.Now().Add(4 * time.Minute)
|
||||
nodes := []inventory.Node{{Name: "ace", LastSeen: time.Now().Add(-time.Hour)},
|
||||
{Name: "laptop", LastSeen: time.Now().Add(-time.Hour)}}
|
||||
read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
|
||||
return readPresence(ctx, js.Conn(), names)
|
||||
}
|
||||
heard, err := nodesHeard(ctx, nodes, read, later)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := heardFromAt(heard[0], later); got != "here" {
|
||||
t.Fatalf("a node online by presence reads %q four minutes later", got)
|
||||
}
|
||||
if got := heardFromAt(heard[1], later); got == "here" {
|
||||
t.Fatalf("a node offline by presence reads %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// aPresenceFake is the watchman's presence bucket as a reader sees it.
|
||||
func aPresenceFake(entries map[string]presenceEntry, err error) presenceReader {
|
||||
return func(_ context.Context, _ []string) (map[string]presenceEntry, error) { return entries, err }
|
||||
}
|
||||
|
||||
// 1183/AC1: one function says when a machine was last heard: online is now, offline is the entry's time, and only where
|
||||
// presence has no key, the last report.
|
||||
func TestLastHeardIsPresenceThenTheReport(t *testing.T) { // 1183/AC1
|
||||
now := time.Now()
|
||||
report := now.Add(-time.Hour)
|
||||
if got := lastHeard(report, nil, now); !got.Equal(report) {
|
||||
t.Fatalf("no presence key: last heard is %v, want the report %v", got, report)
|
||||
}
|
||||
if got := lastHeard(report, &presenceEntry{Online: true, At: now.Add(-9 * time.Hour)}, now); !got.Equal(now) {
|
||||
t.Fatalf("online: last heard is %v, want now", got)
|
||||
}
|
||||
offlineAt := now.Add(-4 * time.Minute)
|
||||
if got := lastHeard(now.Add(-time.Minute), &presenceEntry{At: offlineAt}, now); !got.Equal(offlineAt) {
|
||||
t.Fatalf("offline: last heard is %v, want the entry's time %v, not a report's", got, offlineAt)
|
||||
}
|
||||
if got := lastHeard(time.Time{}, nil, now); !got.IsZero() {
|
||||
t.Fatalf("never heard and no key: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The machines facts, the nodes list, node and status all take that one function.
|
||||
func TestEveryReaderOfLastHeardTakesPresence(t *testing.T) { // 1183/AC1
|
||||
now := time.Now()
|
||||
m := machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
|
||||
applyPresence(&m, nil, &presenceEntry{At: now.Add(-4 * time.Minute)}, now)
|
||||
if !m.lastHeard.Equal(now.Add(-4 * time.Minute)) {
|
||||
t.Fatalf("machine facts take a report over the offline entry: %v", m.lastHeard)
|
||||
}
|
||||
}
|
||||
|
||||
func twoNodes(now time.Time) []inventory.Node {
|
||||
return []inventory.Node{
|
||||
{Name: "anchor", LastSeen: now.Add(-time.Hour)},
|
||||
{Name: "laptop", LastSeen: now.Add(-30 * time.Second)},
|
||||
{Name: "fresh"},
|
||||
}
|
||||
}
|
||||
|
||||
// 1183/AC2: a machine online by presence whose last report is an hour old reads here, is not quiet and is not silent; one
|
||||
// offline for 4 minutes reads out of touch.
|
||||
func TestOnlineByPresenceReadsHereWhateverTheReport(t *testing.T) { // 1183/AC2
|
||||
now := time.Now()
|
||||
read := aPresenceFake(map[string]presenceEntry{
|
||||
"anchor": {Online: true, At: now.Add(-6 * time.Hour)},
|
||||
"laptop": {Online: false, At: now.Add(-4 * time.Minute)},
|
||||
}, nil)
|
||||
nodes, err := nodesHeard(context.Background(), twoNodes(now), read, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := heardFromAt(nodes[0], now); got != "here" {
|
||||
t.Errorf("anchor online with an hour-old report reads %q", got)
|
||||
}
|
||||
if got := heardFromAt(nodes[1], now); got != "out of touch 4m" {
|
||||
t.Errorf("laptop offline 4 minutes reads %q", got)
|
||||
}
|
||||
if got := heardFromAt(nodes[2], now); got != "never spoken" {
|
||||
t.Errorf("a machine with no key and no report reads %q", got)
|
||||
}
|
||||
quiet := notHeardLately(nodes, now)
|
||||
if len(quiet) != 1 || quiet[0].Name != "fresh" {
|
||||
t.Errorf("quiet is %+v: an online machine is not quiet, and a machine offline 4 minutes is not quiet yet", quiet)
|
||||
}
|
||||
f := &machineFacts{name: "anchor", lastHeard: nodes[0].LastSeen}
|
||||
if f.silent(now) {
|
||||
t.Errorf("a machine online by presence is silent")
|
||||
}
|
||||
}
|
||||
|
||||
// Unreadable presence is said, and the report stands as it was.
|
||||
func TestUnreadablePresenceLeavesTheReport(t *testing.T) { // 1183/AC1
|
||||
now := time.Now()
|
||||
nodes, err := nodesHeard(context.Background(), twoNodes(now), aPresenceFake(nil, errors.New("no bus")), now)
|
||||
if err == nil || !strings.Contains(err.Error(), "no bus") {
|
||||
t.Fatalf("an unreadable presence is not said: %v", err)
|
||||
}
|
||||
if !nodes[0].LastSeen.Equal(now.Add(-time.Hour)) {
|
||||
t.Fatalf("the report was replaced: %v", nodes[0].LastSeen)
|
||||
}
|
||||
}
|
||||
|
||||
// 1183/AC4: where the watchman gives no verdict a machine is silent after a fixed three minutes, whatever interval a
|
||||
// node-engine once said.
|
||||
func TestTheSilentBoundIsFixed(t *testing.T) { // 1183/AC4
|
||||
now := time.Now()
|
||||
for _, c := range []struct {
|
||||
ago time.Duration
|
||||
want bool
|
||||
}{{179 * time.Second, false}, {181 * time.Second, true}} {
|
||||
m := machineFacts{name: "laptop", lastHeard: now.Add(-c.ago)}
|
||||
if got := m.silent(now); got != c.want {
|
||||
t.Errorf("heard %v ago: silent %v, want %v", c.ago, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// aLongGroup is members as JSON over 200 KB: 1666 paths of the shape of mesh-delivery #5's change (novox/hq issue 1272).
|
||||
func aLongGroup(t *testing.T) string {
|
||||
t.Helper()
|
||||
var paths []string
|
||||
for i := 0; i < 1666; i++ {
|
||||
paths = append(paths, fmt.Sprintf("modules/mesh-delivery-forge-bridge/vendor/github.com/nats-io/nats-server/v2/server/certstore/certstore_windows_file_%04d.go", i))
|
||||
}
|
||||
body, err := json.Marshal([]map[string]any{{"id": "novox/mesh-delivery#5", "paths": paths}, {"id": "novox/mesh-catalog#374"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(body) < 200<<10 {
|
||||
t.Fatalf("the members are only %d bytes", len(body))
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
func readBackFromTheCommand(t *testing.T, argv []string, members string) {
|
||||
t.Helper()
|
||||
for _, a := range argv {
|
||||
if len(a) > 64<<10 {
|
||||
t.Errorf("an argument of %d bytes is handed to the command", len(a))
|
||||
}
|
||||
}
|
||||
i := slices.Index(argv, "--members-file")
|
||||
if i < 0 {
|
||||
t.Fatalf("the command does not name a members file: %.200v", argv)
|
||||
}
|
||||
raw, err := os.ReadFile(argv[i+1])
|
||||
if err != nil || string(raw) != members {
|
||||
t.Errorf("the file does not hold the whole members list (%d bytes, %v)", len(raw), err)
|
||||
}
|
||||
removeMembersFile(argv)
|
||||
if _, err := os.Stat(argv[i+1]); !os.IsNotExist(err) {
|
||||
t.Errorf("the file is still there after the command: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryCheckHandsALongMembersListInAFile(t *testing.T) { // 1272/AC1
|
||||
members := aLongGroup(t)
|
||||
argv, err := argvFor("delivery-check", map[string]any{"group": "feat/x", "members": members})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if argv[0] != "delivery" || argv[1] != "check" || !slices.Contains(argv, "--group") {
|
||||
t.Errorf("the command is %.100v", strings.Join(argv, " "))
|
||||
}
|
||||
readBackFromTheCommand(t, argv, members)
|
||||
}
|
||||
|
||||
func TestDeliveryOrderHandsALongMembersListInAFile(t *testing.T) { // 1272/AC2
|
||||
members := aLongGroup(t)
|
||||
argv, err := argvFor("delivery-order", map[string]any{"members": members})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if argv[0] != "delivery" || argv[1] != "order" {
|
||||
t.Errorf("the command is %.100v", strings.Join(argv, " "))
|
||||
}
|
||||
readBackFromTheCommand(t, argv, members)
|
||||
}
|
||||
|
||||
func TestAShortMembersListIsStillTheArgumentItWas(t *testing.T) { // 1272/AC3
|
||||
members := `[{"id":"novox/a#1"},{"id":"novox/b#2"}]`
|
||||
argv, err := argvFor("delivery-check", map[string]any{"group": "feat/x", "members": members, "ready": "true"})
|
||||
want := []string{"delivery", "check", "--group", "feat/x", "--members", members, "--ready"}
|
||||
if err != nil || !slices.Equal(argv, want) {
|
||||
t.Errorf("a short list gives %v, %v, want %v", argv, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCommandReadsTheMembersFromTheFileAsFromTheArgument(t *testing.T) { // 1272/AC1
|
||||
members := `[{"id":"novox/a#1"}]`
|
||||
argv, _ := membersFlags(strings.Repeat(" ", membersInArgument) + members)
|
||||
defer removeMembersFile(argv)
|
||||
if len(argv) != 2 || argv[0] != "--members-file" {
|
||||
t.Fatalf("a long list gives %v", argv)
|
||||
}
|
||||
if st, err := os.Stat(argv[1]); err != nil || st.Mode().Perm() != 0o600 {
|
||||
t.Errorf("the file is %v, %v", st, err)
|
||||
}
|
||||
}
|
||||
@@ -154,6 +154,24 @@ type mergeWidthOf struct {
|
||||
// wideRebuild is how many modules a rebuild may take before the gate says so as a warning.
|
||||
var wideRebuild = 12
|
||||
|
||||
// changedPaths are the paths --changed (comma-separated) and --changed-file (one per line) name together.
|
||||
func changedPaths(list, file string) ([]string, error) {
|
||||
out := splitList(list)
|
||||
if file == "" {
|
||||
return out, nil
|
||||
}
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("--changed-file: %w", err)
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if line = strings.TrimSpace(line); line != "" {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mergeGateCommand is `gate`:
|
||||
//
|
||||
// merge-gate --facts <file> --store <postgres> [--repository owner/repo --tree <dir>] [--changed a,b] [--json]
|
||||
@@ -167,6 +185,8 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
repository := set.String("repository", "", "owner/repository of the change")
|
||||
tree := set.String("tree", "", "the change's checkout: every module.json in it replaces the mesh's")
|
||||
changed := set.String("changed", "", "the paths the change touches, comma-separated, for its rebuild's width")
|
||||
changedFile := set.String("changed-file", "", "a file of the paths the change touches, one per line; the same as "+
|
||||
"--changed, for a list too long for an argument (novox/hq issue 1247)")
|
||||
group := set.String("group", "", "a delivery group's other heads, as JSON [{repository, tree, changed}], composed "+
|
||||
"with this one as one future state (novox/hq ADR 0239)")
|
||||
asJSON := set.Bool("json", false, "print the verdict as JSON")
|
||||
@@ -198,7 +218,11 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
// verdict at all. What is said goes beside the report.
|
||||
os.Stdout = os.Stderr
|
||||
}
|
||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: splitList(*changed),
|
||||
changedList, err := changedPaths(*changed, *changedFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: changedList,
|
||||
group: others, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||
os.Stdout = stdout
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTheGateJudgesAFileOfPathsAsItDoesTheSameListOfThem(t *testing.T) { // 1247/AC3
|
||||
paths := []string{"modules/a/x.go", "modules/a/y.go", "modules/b/z.go", "README.md"}
|
||||
file := filepath.Join(t.TempDir(), "changed.txt")
|
||||
if err := os.WriteFile(file, []byte(strings.Join(paths, "\n")+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fromFile, err := changedPaths("", file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fromList, err := changedPaths(strings.Join(paths, ","), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(fromFile, fromList) || len(fromFile) != len(paths) {
|
||||
t.Errorf("the file gives %v, the list %v", fromFile, fromList)
|
||||
}
|
||||
if _, err := changedPaths("", filepath.Join(t.TempDir(), "none")); err == nil {
|
||||
t.Error("a file that is not there is read as no paths")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1211/AC1 to AC3: the controller keeps a merge announced by a module that claims the delivery seat, whatever its
|
||||
// name, and takes no word from one that does not (novox/hq ADR 0437 §3). With no bus held, a word that passes the
|
||||
// claim is held for the bus (ErrTryAgain); a word that does not is refused before the bus is asked.
|
||||
func TestOnlyAModuleClaimingTheSeatIsTheDeliverysWordOnAMerge(t *testing.T) {
|
||||
entries := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("gitea"), namedOnly("mesh-delivery")}
|
||||
keeper := keepingDeliveryMerges{
|
||||
js: func() *broker.JetStream { return nil },
|
||||
catalog: func(context.Context) ([]inventory.Entry, error) { return entries, nil },
|
||||
}
|
||||
body := []byte(`{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve"}`)
|
||||
for emitter, passes := range map[string]bool{
|
||||
"mesh-delivery-gate": true, // the claimant, under another name
|
||||
"gitea": false, // claims no seat
|
||||
"mesh-delivery": false, // named so, claiming no seat here
|
||||
} {
|
||||
err := keeper.KeepDeliveryMerge(context.Background(), emitter, body)
|
||||
if passes != errors.Is(err, link.ErrTryAgain) || (!passes && !errors.Is(err, link.ErrNotTheDeliverysWord)) {
|
||||
t.Errorf("%s: %v (passes the claim: %v)", emitter, err, passes)
|
||||
}
|
||||
}
|
||||
entries = []inventory.Entry{claimingTheSeat("mesh-delivery")}
|
||||
if err := keeper.KeepDeliveryMerge(context.Background(), "mesh-delivery", body); !errors.Is(err, link.ErrTryAgain) {
|
||||
t.Errorf("today's module, claiming the seat: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -926,13 +926,45 @@ func brokerReachableAt(ctx context.Context, inv *inventory.Inventory, known brok
|
||||
// A module that runs its machine's provisioner client and declares no broker is issued the client's account
|
||||
// alone (issue 917): the client speaks on the bus only as itself.
|
||||
func mayIssue(m catalogue.Manifest) error {
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads && !runsTheProvisionerClient(m) {
|
||||
processes := m.ProcessBuses()
|
||||
// A module whose own tools run in the machine's tool runner — mesh-mail's real shape — declares no
|
||||
// `broker` of its own at all (novox/hq ADR 0436 §2): its process's account is a different account,
|
||||
// delivered as its own secret below, so a process with a bus is reason enough to pass this check on
|
||||
// its own, before the module's own `broker` is asked for.
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads && !runsTheProvisionerClient(m) && len(processes) == 0 {
|
||||
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
||||
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
||||
}
|
||||
// And every process with a bus declaration (novox/hq ADR 0436 §2): its own account is delivered as the
|
||||
// module's own secret broker-<process id>, never the module's `broker`, so a process with a bus and no
|
||||
// such secret has nowhere to read an account either.
|
||||
for id := range processes {
|
||||
name := processSecretName(id)
|
||||
if _, reads := m.OwnSecrets[name]; !reads {
|
||||
return fmt.Errorf("%s's process %s declares bus, and declares no own secret named %s, so there "+
|
||||
"is nowhere to deliver its account; a process with a bus declares \"own-secrets\": "+
|
||||
"{%q: <path>} (novox/hq ADR 0436 §2)", m.Module, id, name, name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// processSecretName is the own secret a process's bus account is delivered in (novox/hq ADR 0436 §2):
|
||||
// broker-<process id>, never the module's own `broker`.
|
||||
func processSecretName(processID string) string { return "broker-" + processID }
|
||||
|
||||
// processPrincipalFor is the one process of m's bus declarations named processID, as `rollout mint` finds it
|
||||
// again from a KindProcess principal's bare id (novox/hq ADR 0436 §1, §2): false when the manifest no longer
|
||||
// declares it.
|
||||
func processPrincipalFor(m catalogue.Manifest, processID string) (broker.ProcessPrincipal, bool) {
|
||||
for _, pr := range inventory.ProcessesOf(m) {
|
||||
if pr.Process == processID {
|
||||
return pr, true
|
||||
}
|
||||
}
|
||||
return broker.ProcessPrincipal{}, false
|
||||
}
|
||||
|
||||
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||
//
|
||||
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||
@@ -968,12 +1000,82 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Every process with a bus declaration gets an account of its own (novox/hq ADR 0436 §1, §2), minted and
|
||||
// delivered whether or not the module itself speaks on the bus — its own secret is broker-<process id>,
|
||||
// never the module's `broker`.
|
||||
for _, pr := range inventory.ProcessesOf(m) {
|
||||
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !runsTheProvisionerClient(m) {
|
||||
return nil
|
||||
}
|
||||
return issueProvisionerClient(ctx, inv, m, node, known, reachable)
|
||||
}
|
||||
|
||||
// issueProcess gives one process its account: minted under its own name <node>.<module>.<process id>,
|
||||
// delivered as the module's own secret broker-<process id> in the provisioner client's form (novox/hq ADR
|
||||
// 0436 §2), handed to the vault as any covered account is, and its work-queue consumer or consumers
|
||||
// created where the bus can be reached — never the module's claims, since a process holds no seat.
|
||||
func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
|
||||
node, busAddress string, known broker.Broker, reachable string) error {
|
||||
user := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := processSecretName(pr.Process)
|
||||
held, err := heldProcessAccount(known, reachable, node, m.Module, user, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, name, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := handIssuedToVault(ctx, inv, m, node, name, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(pr.Queues) > 0 {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s's process %s consumes its work queue; its consumer is created when the bus is "+
|
||||
"reachable (`push --behind`, then `rollout mint` again is harmless)\n", m.Module, pr.Process)
|
||||
} else {
|
||||
js, err := broker.Dial(busAddress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
||||
"%s's process %s's work-queue consumer: %w", m.Module, pr.Process, err)
|
||||
}
|
||||
defer js.Close()
|
||||
who := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}
|
||||
for _, q := range pr.Queues {
|
||||
if err := js.EnsureConsumer(broker.QueueConsumer(who, q)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("bus user %s minted for %s's process %s, scoped to its bus declaration\n", user, m.Module, pr.Process)
|
||||
fmt.Printf(" sealed to %s as %s. It arrives with the next push — `push %s` to send it\n", node, name, node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// heldProcessAccount is a process's account, in the provisioner client's form (novox/hq ADR 0436 §2):
|
||||
// {url, fingerprint, node, module, user, password} — never claims, since a process holds no seat (§3).
|
||||
func heldProcessAccount(known broker.Broker, reachable, node, module, user, password string) ([]byte, error) {
|
||||
return json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, Module: module, User: user,
|
||||
Password: password})
|
||||
}
|
||||
|
||||
// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an
|
||||
// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret
|
||||
// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor).
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -12,8 +13,10 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -44,6 +47,28 @@ type deliveryMerge struct {
|
||||
AskID string `json:"ask_id"`
|
||||
Option string `json:"option"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
// Asker is the module that asked the ask, whose warrant is read from the router under that name (novox/hq ADR
|
||||
// 0437 §6): empty is today's module, mesh-delivery.
|
||||
Asker string `json:"asker,omitempty"`
|
||||
}
|
||||
|
||||
// askerOf is the asker whose warrant the merge is read under: the one the announcement names, mesh-delivery when it
|
||||
// names none.
|
||||
func (d deliveryMerge) askerOf() string {
|
||||
if d.Asker == "" {
|
||||
return theDelivery
|
||||
}
|
||||
return d.Asker
|
||||
}
|
||||
|
||||
// askerAccepted is whether the controller takes a warrant from this asker for a merge: the teller, and mesh-delivery
|
||||
// until the gate takes the seat over (novox/hq ADR 0437 §6). Any other registers no move.
|
||||
func askerAccepted(asker string) error {
|
||||
if asker == theDelivery || asker == sdkdelivery.TellerModule {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("the merge names the asker %q, and the controller takes a warrant only from %s and %s",
|
||||
asker, sdkdelivery.TellerModule, theDelivery)
|
||||
}
|
||||
|
||||
// parseDeliveryMerge reads an announcement, refusing one that names no pull request, head or ask.
|
||||
@@ -79,6 +104,10 @@ func mergeAct(ask, option string, heads map[string]string) asks.Act {
|
||||
// proved by the channel it came through, of this very ask as it was published (digest), bound to the heads the
|
||||
// announcement names, and those heads hold the head merged. Answers why it holds, or why not.
|
||||
func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
||||
asker := d.askerOf()
|
||||
if err := askerAccepted(asker); err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch {
|
||||
case ask == nil || w == nil:
|
||||
return "", errors.New("the router holds no closed record of the ask the merge was made on")
|
||||
@@ -87,7 +116,7 @@ func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, er
|
||||
case d.Option != optionApprove:
|
||||
return "", fmt.Errorf("the merge was made on the option %q, not an Approve", d.Option)
|
||||
}
|
||||
option, err := w.For(theDelivery, *ask)
|
||||
option, err := w.For(asker, *ask)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -115,9 +144,26 @@ func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, er
|
||||
// keepingDeliveryMerges keeps each merge mesh-delivery announces in the controller's bucket for them.
|
||||
type keepingDeliveryMerges struct {
|
||||
js func() *broker.JetStream
|
||||
// catalog is the catalogue, whose modules claiming the delivery seat are the only ones whose word on a merge is
|
||||
// kept (novox/hq ADR 0437 §3).
|
||||
catalog func(ctx context.Context) ([]inventory.Entry, error)
|
||||
// kept, when set, is told each merge once it is kept: a first merge waiting for the word asks its builds then
|
||||
// (novox/hq ADR 0432, issue 1146).
|
||||
kept func(ctx context.Context, d deliveryMerge) error
|
||||
}
|
||||
|
||||
func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, body []byte) error {
|
||||
func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, emitter string, body []byte) error {
|
||||
if k.catalog == nil {
|
||||
return fmt.Errorf("%w: the catalogue is not read", link.ErrTryAgain)
|
||||
}
|
||||
entries, err := k.catalog(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
if !slices.Contains(claimantsOf(entries, catalogue.DeliverySeat), emitter) {
|
||||
return fmt.Errorf("%w: %q is not among %v", link.ErrNotTheDeliverysWord, emitter,
|
||||
claimantsOf(entries, catalogue.DeliverySeat))
|
||||
}
|
||||
d, err := parseDeliveryMerge(body)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -137,6 +183,9 @@ func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, body []byt
|
||||
if _, err := kv.Put(ctx, deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA), body); err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
if k.kept != nil {
|
||||
return k.kept(ctx, d)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -166,9 +215,9 @@ var keptDeliveryMerge = func(ctx context.Context, owner, repo string, number int
|
||||
return d, err == nil, err
|
||||
}
|
||||
|
||||
// routerAskOf is the router's record of one of mesh-delivery's asks: the ask as published and its warrant. A
|
||||
// routerAskOf is the router's record of one of the delivery's asks, under its asker: the ask as published and its warrant. A
|
||||
// variable so a test can hand it one.
|
||||
var routerAskOf = func(ctx context.Context, inv catalogueReader, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
var routerAskOf = func(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, nil, errors.New("only the serving controller reads the router's record")
|
||||
@@ -180,7 +229,7 @@ var routerAskOf = func(ctx context.Context, inv catalogueReader, id string) (*as
|
||||
if bucket == "" {
|
||||
return nil, nil, errors.New("no module declares the operator channel's records")
|
||||
}
|
||||
_, ask, w, err := readRouterEntry(ctx, serving.Conn(), bucket, theDelivery, id)
|
||||
_, ask, w, err := readRouterEntry(ctx, serving.Conn(), bucket, asker, id)
|
||||
return ask, w, err
|
||||
}
|
||||
|
||||
@@ -236,24 +285,42 @@ func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, bu
|
||||
if !known {
|
||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, r.Commit)
|
||||
}
|
||||
k := announcedOf(merged)
|
||||
return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged))
|
||||
}
|
||||
|
||||
// errWordNotAnApprove says the merge cannot be shown to be the operator's Approve of its head: its announcement names no
|
||||
// pull request, or the word on record is not that Approve.
|
||||
var errWordNotAnApprove = errors.New("the word on record is no Approve of the head merged")
|
||||
|
||||
// errNotAnnounced says mesh-delivery's word on a merge is not kept (yet): the merge may have been made by someone else,
|
||||
// or its word may still be on its way.
|
||||
var errNotAnnounced = errors.New("mesh-delivery announced no such merge")
|
||||
|
||||
// approvedAnnounced is the warrant of the merge the forge announced as k: mesh-delivery announced it, and the ask it made
|
||||
// it on holds the operator's Approve of that head. Why it holds, or why not. Shared by the approved move of a known
|
||||
// module and the first build of a new repository's modules (ADR 0420 §7, ADR 0432).
|
||||
func approvedAnnounced(ctx context.Context, inv catalogueReader, owner, name string, k keptMerge) (string, error) {
|
||||
if k.Number <= 0 || k.HeadSHA == "" {
|
||||
return "", errors.New("the forge's announcement of the merge names no pull request or head")
|
||||
return "", fmt.Errorf("%w: the forge's announcement of the merge names no pull request or head", errWordNotAnApprove)
|
||||
}
|
||||
d, kept, err := keptDeliveryMerge(ctx, owner, name, k.Number, k.HeadSHA)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !kept {
|
||||
return "", fmt.Errorf("mesh-delivery announced no merge of %s/%s#%d at %.8s", owner, name, k.Number, k.HeadSHA)
|
||||
return "", fmt.Errorf("%w: %s/%s#%d at %.8s", errNotAnnounced, owner, name, k.Number, k.HeadSHA)
|
||||
}
|
||||
ask, w, err := routerAskOf(ctx, inv, d.AskID)
|
||||
// The warrant is read under the asker the announcement names, and under none the controller does not take.
|
||||
if err := askerAccepted(d.askerOf()); err != nil {
|
||||
return "", fmt.Errorf("%w: %w", errWordNotAnApprove, err)
|
||||
}
|
||||
ask, w, err := routerAskOf(ctx, inv, d.askerOf(), d.AskID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
why, err := movedOnApprove(d, ask, w)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("%w: %w", errWordNotAnApprove, err)
|
||||
}
|
||||
return fmt.Sprintf("%s/%s#%d merged at %.8s on %s", owner, name, k.Number, k.HeadSHA, why), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The controller reads a merge's warrant under the asker the announcement names, and accepts only the teller and
|
||||
// today's mesh-delivery (novox/hq ADR 0437 §6, issue 1213). The teller's name is the SDK's.
|
||||
|
||||
// askedBy is an approved merge whose announcement names an asker ("" names none) and whose warrant is that asker's.
|
||||
func askedBy(t *testing.T, named, warrantOf string) anApprovedMerge {
|
||||
t.Helper()
|
||||
a := approvedMerge(t)
|
||||
a.merge.Asker = named
|
||||
a.warrant.Asker = warrantOf
|
||||
return a
|
||||
}
|
||||
|
||||
// 1213/AC1: an announcement naming the teller, and the teller's warrant of the operator's verified Approve of the
|
||||
// merge's heads: the move is registered (the warrant is read from the router under the teller).
|
||||
func TestAMergeNamingTheTellerIsMovedOnTheTellersWarrant(t *testing.T) {
|
||||
a := askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule)
|
||||
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil {
|
||||
t.Fatalf("the teller's warrant of the operator's Approve was refused: %v", err)
|
||||
}
|
||||
was := routerAskOf
|
||||
t.Cleanup(func() { routerAskOf = was })
|
||||
wasKept := keptDeliveryMerge
|
||||
t.Cleanup(func() { keptDeliveryMerge = wasKept })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
var readUnder string
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, asker, _ string) (*asks.Ask, *asks.Warrant, error) {
|
||||
readUnder = asker
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil {
|
||||
t.Fatalf("the merge was not moved on the teller's warrant: %v", err)
|
||||
}
|
||||
if readUnder != sdkdelivery.TellerModule {
|
||||
t.Errorf("the warrant was read under %q", readUnder)
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC2: an announcement naming mesh-delivery, or none (today's module): read under mesh-delivery, registered.
|
||||
func TestAMergeNamingNoAskerIsMovedOnMeshDeliverysWarrant(t *testing.T) {
|
||||
for _, named := range []string{"", theDelivery} {
|
||||
a := askedBy(t, named, theDelivery)
|
||||
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil {
|
||||
t.Errorf("asker %q: refused: %v", named, err)
|
||||
}
|
||||
}
|
||||
was := routerAskOf
|
||||
wasKept := keptDeliveryMerge
|
||||
t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept })
|
||||
a := askedBy(t, "", theDelivery)
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
var readUnder string
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, asker, _ string) (*asks.Ask, *asks.Warrant, error) {
|
||||
readUnder = asker
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if readUnder != theDelivery {
|
||||
t.Errorf("with no asker named the warrant was read under %q, not %q", readUnder, theDelivery)
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC3: any other asker registers no move, and the reason names the asker refused; the router is not asked.
|
||||
func TestAMergeNamingAnyOtherAskerMovesNothingAndNamesIt(t *testing.T) {
|
||||
was := routerAskOf
|
||||
wasKept := keptDeliveryMerge
|
||||
t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept })
|
||||
for _, other := range []string{sdkdelivery.GateModule, "mesh-controller", "unknown-module"} {
|
||||
a := askedBy(t, other, other) // even a warrant of that very asker
|
||||
_, err := movedOnApprove(a.merge, &a.ask, &a.warrant)
|
||||
if err == nil || !strings.Contains(err.Error(), other) {
|
||||
t.Errorf("asker %q: %v", other, err)
|
||||
}
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
t.Errorf("the router was read under the refused asker %q", other)
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err == nil ||
|
||||
!strings.Contains(err.Error(), other) {
|
||||
t.Errorf("asker %q, approved as announced: %v", other, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC4: a warrant whose own asker differs from the asker read, or whose heads differ from the head merged, moves nothing.
|
||||
func TestAWarrantOfAnotherAskerOrOtherHeadsMovesNothing(t *testing.T) {
|
||||
a := askedBy(t, sdkdelivery.TellerModule, theDelivery) // read under the teller, but the warrant says mesh-delivery
|
||||
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||
t.Errorf("a warrant of another asker was accepted: %s", why)
|
||||
}
|
||||
a = askedBy(t, "", sdkdelivery.TellerModule) // today's module named, the teller's warrant
|
||||
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||
t.Errorf("the teller's warrant under mesh-delivery was accepted: %s", why)
|
||||
}
|
||||
a = askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule)
|
||||
a.merge.HeadSHA = "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4"
|
||||
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||
t.Errorf("a head the Approve did not cover was accepted: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC5: the announcement's asker is read and kept with the rest of it, under the JSON name asker.
|
||||
func TestTheAnnouncementsAskerIsReadAndKept(t *testing.T) {
|
||||
body := `{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve","asker":"mesh-delivery-teller"}`
|
||||
d, err := parseDeliveryMerge([]byte(body))
|
||||
if err != nil || d.Asker != sdkdelivery.TellerModule {
|
||||
t.Fatalf("read %+v, %v", d, err)
|
||||
}
|
||||
none, err := parseDeliveryMerge([]byte(strings.Replace(body, `,"asker":"mesh-delivery-teller"`, "", 1)))
|
||||
if err != nil || none.Asker != "" {
|
||||
t.Fatalf("an announcement naming none read as %+v, %v", none, err)
|
||||
}
|
||||
}
|
||||
|
||||
// announcedPull is the forge's announcement of a merge of a pull request at a head.
|
||||
func announcedPull(number int, head string) keptMerge {
|
||||
var k keptMerge
|
||||
k.Number, k.HeadSHA = number, head
|
||||
return k
|
||||
}
|
||||
@@ -74,7 +74,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||
@@ -90,7 +90,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
||||
|
||||
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubject, body); err != nil {
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||
|
||||
@@ -93,7 +93,7 @@ func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwis
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if c.forge != nil {
|
||||
@@ -149,7 +149,7 @@ func TestASecondNewModuleFromARepositoryIsJudgedAlongsideAndAsksNoWarrant(t *tes
|
||||
t.Error("1146/AC5: the announced merges were asked for a module judged alongside")
|
||||
return deliveryMerge{}, false, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
t.Error("1146/AC5: the router was asked for a module judged alongside")
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||
@@ -91,7 +91,7 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB
|
||||
|
||||
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubject, body); err != nil {
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||
|
||||
@@ -50,7 +50,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
return addNode(ctx, inv, args[1:])
|
||||
|
||||
case "list":
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
nodes, err := open.heardNodes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -755,22 +755,8 @@ func consumerReset(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
|
||||
// picture of the mesh.
|
||||
func heardFrom(n inventory.Node) string {
|
||||
silent, ever := n.Silent()
|
||||
switch {
|
||||
case !ever:
|
||||
return "never spoken"
|
||||
case silent > SilentFor:
|
||||
return "out of touch " + roughly(silent)
|
||||
default:
|
||||
return "here"
|
||||
}
|
||||
}
|
||||
// heardFrom is heardFromAt for now.
|
||||
func heardFrom(n inventory.Node) string { return heardFromAt(n, time.Now()) }
|
||||
|
||||
// roughly is a duration a person reads rather than parses.
|
||||
func roughly(d time.Duration) string {
|
||||
@@ -795,7 +781,7 @@ func roughly(d time.Duration) string {
|
||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
inv := stored.inventory
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
node, err := stored.heardNode(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// 1146: a merge's ask that failed is kept under an id the bus takes as a key, for a module at the repository's root
|
||||
// (".") as for one in a folder.
|
||||
func TestTheIdOfABuildNotAskedIsAKeyTheBusTakes(t *testing.T) {
|
||||
for dir, want := range map[string]string{
|
||||
".": "not-asked-0177ea7a",
|
||||
"modules/mesh-delivery-view": "not-asked-0177ea7a-modules-mesh-delivery-view",
|
||||
} {
|
||||
id := notAskedID("0177ea7a00000000", dir)
|
||||
if id != want {
|
||||
t.Errorf("notAskedID(%q) = %q, want %q", dir, id, want)
|
||||
}
|
||||
if !chain.ValidKey("req." + id) {
|
||||
t.Errorf("notAskedID(%q) = %q: the bus takes no key req.%s", dir, id, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The controller finds the delivery's owner by the claim of the seat mesh-delivery, never by the module's name
|
||||
// (novox/hq ADR 0437 §3, issue 1210). The gate is a module named mesh-delivery-gate; this is only a fixture name.
|
||||
|
||||
func claimingTheSeat(module string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module,
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
||||
}
|
||||
|
||||
func namedOnly(module string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}, On: []string{"anchor"}}
|
||||
}
|
||||
|
||||
// 1210/AC1: a module claiming the seat is on the controller's own path whatever its name, awaiting the delivery's
|
||||
// word as today's module does; a module named mesh-delivery that claims no seat is not on it.
|
||||
func TestTheModuleClaimingTheSeatIsOnTheControllersOwnPath(t *testing.T) {
|
||||
gate := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")}
|
||||
own := ownPathOf(gate)
|
||||
if _, on := own["mesh-delivery-gate"]; !on {
|
||||
t.Error("the module claiming the seat is not on the controller's own path")
|
||||
}
|
||||
if _, on := own["mesh-delivery"]; on {
|
||||
t.Error("a module named mesh-delivery that claims no seat is on the controller's own path")
|
||||
}
|
||||
if d := awaitsFor(gate, []string{"gitea", "plex"}); d == nil || d.Awaits != catalogue.DeliverySeat {
|
||||
t.Errorf("an ordinary walk awaits %+v with the gate holding the seat", d)
|
||||
}
|
||||
if d := awaitsFor(gate, []string{"mesh-delivery-gate", "gitea"}); d != nil {
|
||||
t.Errorf("a walk moving the gate awaits %+v; the gate cannot gate itself", d)
|
||||
}
|
||||
if d := awaitsFor(gate, []string{"mesh-delivery", "gitea"}); d == nil {
|
||||
t.Error("a walk moving a module named mesh-delivery that claims no seat starts at once")
|
||||
}
|
||||
if !ownWalk(walkOf("novox/one", "notes", "mesh-delivery-gate"), own) || ownWalk(walkOf("novox/one", "notes", "mesh-delivery"), own) {
|
||||
t.Error("a walk is on the own path by the claim of the module it plans")
|
||||
}
|
||||
if ownPath([]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}) != true || ownPath([]inventory.Entry{namedOnly("mesh-delivery")}) {
|
||||
t.Error("a merge touches the own path by the claim of the module it touches")
|
||||
}
|
||||
if classOfModules([]string{"mesh-delivery-gate"}, gate) != inventory.ClassCore {
|
||||
t.Error("the module claiming the seat is not core")
|
||||
}
|
||||
}
|
||||
|
||||
// 1210/AC2: the merge gate's rule of the module claiming the seat is one reading; a module claiming no delivery
|
||||
// seat gets its ordinary rule.
|
||||
func TestTheModuleClaimingTheSeatGetsTheDeliverysGateRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
claimant := claimingTheSeat("mesh-delivery-gate").Manifest
|
||||
r := gateRuleOf("mesh-delivery-gate", claimant)
|
||||
if r.passes != 1 || r.within != 0 || !strings.HasPrefix(r.said, "mesh-delivery's rule: one reading (novox/hq ADR 0351)") {
|
||||
t.Errorf("the gate's rule is %+v", r)
|
||||
}
|
||||
r = gateRuleOf("mesh-delivery", catalogue.Manifest{Module: "mesh-delivery"})
|
||||
if !strings.HasPrefix(r.said, "a leaf module's rule") {
|
||||
t.Errorf("a module named mesh-delivery claiming no seat gets %q", r.said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1210/AC3: with both today's module and the gate claiming the seat (R3d's group), both are on the own path and
|
||||
// both get the delivery's rule.
|
||||
func TestBothClaimantsAreOnTheOwnPathAndGetTheRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
both := []inventory.Entry{claimingTheSeat("mesh-delivery"), claimingTheSeat("mesh-delivery-gate")}
|
||||
own := ownPathOf(both)
|
||||
for _, m := range []string{"mesh-delivery", "mesh-delivery-gate"} {
|
||||
if _, on := own[m]; !on {
|
||||
t.Errorf("%s is not on the own path", m)
|
||||
}
|
||||
if r := gateRuleOf(m, claimingTheSeat(m).Manifest); r.passes != 1 || !strings.HasPrefix(r.said, "mesh-delivery's rule") {
|
||||
t.Errorf("%s gets %+v", m, r)
|
||||
}
|
||||
if d := awaitsFor(both, []string{m, "gitea"}); d != nil {
|
||||
t.Errorf("a walk moving %s awaits %+v", m, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that
|
||||
// announce a merge from it.
|
||||
func theDeliveryClaims(context.Context) ([]inventory.Entry, error) {
|
||||
return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil
|
||||
}
|
||||
@@ -429,6 +429,11 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"out) has not let it start yet.",
|
||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||
}),
|
||||
kindPlanFailed: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery failed",
|
||||
Explanation: "A delivery did not finish. Nothing is tried again until you say so.",
|
||||
Resolved: "Resolved: the delivery is no longer failed"}
|
||||
}),
|
||||
"ask-lost": worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A build has no result",
|
||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||
@@ -610,6 +615,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"did not answer in ten minutes, so nothing of the delivery is built or sent.",
|
||||
Resolved: "Resolved: the backup is done and the delivery goes on"}
|
||||
}),
|
||||
kindStuck: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Something in a delivery has stopped moving",
|
||||
Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " +
|
||||
"asked and not answered, a check that stopped, or a part that went offline. Nothing is lost, and it is " +
|
||||
"said again until it moves.",
|
||||
Resolved: "Resolved: it moves again"}
|
||||
}),
|
||||
kindSendKeyRefused: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery cannot watch a machine for its report",
|
||||
Explanation: "The bus refused the note that makes the mesh notice a machine that never reports on a send. " +
|
||||
"The delivery goes on, and the note is written again at its next look.",
|
||||
Resolved: "Resolved: the delivery watches the machine for its report again"}
|
||||
}),
|
||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||
|
||||
@@ -285,12 +285,28 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
assigned []string
|
||||
}
|
||||
var others []candidate
|
||||
// **Every machine's assignments, before the holdings are read by any resolution** (novox/hq ADR 0449 §4): a
|
||||
// seat held once whose holder on record is assigned on no machine needs the whole mesh to be known, and it
|
||||
// passes to its only claimant here, in the composition that finds it.
|
||||
everyAssigned := map[string][]string{}
|
||||
var assignedErr error
|
||||
for _, n := range nodes {
|
||||
theirs, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
assignedErr = errors.Join(assignedErr, fmt.Errorf("%s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
if len(theirs) > 0 {
|
||||
everyAssigned[n.Name] = theirs
|
||||
}
|
||||
}
|
||||
holdings = theMeshPassesSeatsOn(ctx, inv, shelf, everyAssigned, assignedErr, holdings)
|
||||
for _, n := range nodes {
|
||||
if n.Name == exclude {
|
||||
continue
|
||||
}
|
||||
theirs, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil || len(theirs) == 0 {
|
||||
theirs := everyAssigned[n.Name]
|
||||
if len(theirs) == 0 {
|
||||
continue
|
||||
}
|
||||
caps, _ := inv.ProfileOf(ctx, n.Name)
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// heldPublish is a bus that holds a plan-moved publish until it is let go, and keeps the body.
|
||||
type heldPublish struct {
|
||||
link.Bus
|
||||
let chan struct{}
|
||||
body chan []byte
|
||||
}
|
||||
|
||||
func (h *heldPublish) PublishSeatEvent(_ context.Context, _, _ string, body []byte) error {
|
||||
<-h.let
|
||||
h.body <- body
|
||||
return nil
|
||||
}
|
||||
|
||||
func aWalkWithSteps() inventory.Plan {
|
||||
built := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC)
|
||||
return inventory.Plan{ID: "plan-1", Repository: "novox/one", Commit: "abc", State: "walking", Tier: 1,
|
||||
Created: built, Updated: built, Revision: 4,
|
||||
Tiers: [][]string{{"alpha"}, {"beta", "gamma"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"alpha": {State: "built", BuiltAt: &built, Commit: "abc", First: []string{"ace"}},
|
||||
"beta": {State: "asked", Why: "waiting"},
|
||||
"gamma": {},
|
||||
},
|
||||
Times: &inventory.PlanTimes{Started: &built}, WalkSaid: inventory.WalkStarted}
|
||||
}
|
||||
|
||||
func TestAnAnnouncementIsThePlanAsItWasWhenSaidWhatTheWalkDoesNext(t *testing.T) { // 1260/AC1
|
||||
p := aWalkWithSteps()
|
||||
bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)}
|
||||
sayPlanMoved(t.Context(), bus, nil, p)
|
||||
|
||||
// The walk goes on, writing its steps while the announcement is being made.
|
||||
for i := 0; i < 2000; i++ {
|
||||
step := p.Modules["beta"]
|
||||
step.State, step.Why = "built", "changed after the call"
|
||||
step.First = append(step.First[:0], "novox")
|
||||
p.Modules["delta"] = &inventory.PlanModule{State: "asked"}
|
||||
p.Tiers[1][0] = "changed"
|
||||
p.Times.Started = nil
|
||||
delete(p.Modules, "delta")
|
||||
}
|
||||
close(bus.let)
|
||||
|
||||
var said inventory.Plan
|
||||
select {
|
||||
case body := <-bus.body:
|
||||
if err := json.Unmarshal(body, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("nothing was announced")
|
||||
}
|
||||
if said.Modules["beta"].State != "asked" || said.Modules["beta"].Why != "waiting" || said.Tiers[1][0] != "beta" ||
|
||||
len(said.Modules) != 3 || said.Times == nil || said.Times.Started == nil {
|
||||
t.Errorf("the announcement carries the walk as it went on, not as it was: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnnouncementOfAnUnchangedPlanIsTheBodyItWasBeforeTheCopy(t *testing.T) { // 1260/AC2
|
||||
p := aWalkWithSteps()
|
||||
bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)}
|
||||
close(bus.let)
|
||||
sayPlanMoved(t.Context(), bus, nil, p)
|
||||
var got []byte
|
||||
select {
|
||||
case got = <-bus.body:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("nothing was announced")
|
||||
}
|
||||
// What it was: the plan whole, its phases worked out, and what the walk says of itself.
|
||||
q := p
|
||||
q.Phases = q.WalkPhases(time.Now(), nil)
|
||||
want, err := json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), nil, q)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var a, b map[string]any
|
||||
if json.Unmarshal(got, &a) != nil || json.Unmarshal(want, &b) != nil {
|
||||
t.Fatal("a body is not JSON")
|
||||
}
|
||||
delete(a, "phases") // worked out from the time of the call
|
||||
delete(b, "phases")
|
||||
if !reflect.DeepEqual(a, b) {
|
||||
t.Errorf("the body changed:\n got %s\nwant %s", got, want)
|
||||
}
|
||||
if a["walk"] != "plan-1" {
|
||||
t.Errorf("the walk's own words are not in the body: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// failedModule is one module of a failed plan that failed to build, where it runs and what it said.
|
||||
type failedModule struct {
|
||||
module string
|
||||
machines []string
|
||||
why string
|
||||
}
|
||||
|
||||
// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1).
|
||||
type failedPlanFacts struct {
|
||||
id, repository, commit string
|
||||
modules []string
|
||||
failed []failedModule
|
||||
// note is where the plan stopped when no module failed to build (a rollout stopped at its first machine).
|
||||
note string
|
||||
// since is the plan's last change: when it failed.
|
||||
since time.Time
|
||||
// refusal is why `plans retry` would not run it, empty when it would.
|
||||
refusal string
|
||||
// passed is set when a newer plan of its repository and branch is done and the retry is refused for that
|
||||
// (novox/hq ADR 0451): the condition asks nothing.
|
||||
passed bool
|
||||
}
|
||||
|
||||
// machines are the machines the failed modules run on, sorted, each once.
|
||||
func (p failedPlanFacts) machines() []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, m := range p.failed {
|
||||
for _, n := range m.machines {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// failureSaid is what failed, in the mesh's own words, for the condition's evidence.
|
||||
func (p failedPlanFacts) failureSaid() string {
|
||||
var parts []string
|
||||
for _, m := range p.failed {
|
||||
line := m.module
|
||||
if len(m.machines) > 0 {
|
||||
line += " on " + strings.Join(m.machines, ", ")
|
||||
}
|
||||
if m.why != "" {
|
||||
line += ": " + oneLine(m.why)
|
||||
}
|
||||
parts = append(parts, line)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "stopped: " + oneLine(p.note)
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// refusalSaid is why a retry would not run, for the evidence.
|
||||
func (p failedPlanFacts) refusalSaid() string {
|
||||
if p.refusal == "" {
|
||||
return ""
|
||||
}
|
||||
return "; a retry would not run: " + oneLine(p.refusal)
|
||||
}
|
||||
|
||||
// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan,
|
||||
// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs).
|
||||
func retryAction(plan string) conditions.Action {
|
||||
return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"retry": plan}}
|
||||
}
|
||||
|
||||
var (
|
||||
planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`)
|
||||
walkWd = regexp.MustCompile(`\bwalks\b`)
|
||||
walkOne = regexp.MustCompile(`\bwalk\b`)
|
||||
)
|
||||
|
||||
// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a
|
||||
// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the
|
||||
// words say only that a retry would not run.
|
||||
func plainRefusal(refusal string) string {
|
||||
return plainRefusalOr(refusal, "a retry would not run now; the details say why")
|
||||
}
|
||||
|
||||
// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly.
|
||||
func plainRefusalOr(refusal, fallback string) string {
|
||||
r := planIDWord.ReplaceAllString(refusal, "the plan")
|
||||
r = walkWd.ReplaceAllString(r, "deliveries")
|
||||
r = walkOne.ReplaceAllString(r, "delivery")
|
||||
r = strings.TrimSpace(r)
|
||||
if _, ok := conditions.Plain(r); !ok {
|
||||
return fallback
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that
|
||||
// is plain, and Retry when a retry would run, else the refusal's words.
|
||||
func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) {
|
||||
name := deliveryName(p.modules, p.repository)
|
||||
headline = conditions.Capital(name) + " failed"
|
||||
what := deliveryWhat(p.modules, p.repository)
|
||||
var parts []string
|
||||
for i, m := range p.failed {
|
||||
if i == 2 {
|
||||
parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2))
|
||||
break
|
||||
}
|
||||
line := m.module
|
||||
if len(m.machines) > 0 {
|
||||
line += " for " + readableList(m.machines)
|
||||
}
|
||||
if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" {
|
||||
if _, ok := conditions.Plain(w); ok {
|
||||
line += " (" + w + ")"
|
||||
}
|
||||
}
|
||||
parts = append(parts, line)
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts))
|
||||
} else {
|
||||
explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what)
|
||||
}
|
||||
switch {
|
||||
case p.passed:
|
||||
explanation += " A newer delivery has passed it, so a retry would send older code. Nothing is needed from you."
|
||||
case p.refusal == "":
|
||||
explanation += " Nothing is tried again until you say so."
|
||||
needs = "try the failed builds again " + FromMeshMCPServer
|
||||
actions = []conditions.Action{retryAction(p.id)}
|
||||
default:
|
||||
explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "."
|
||||
needs = "look at why it failed; a retry would not run now."
|
||||
}
|
||||
resolved = conditions.Capital(name) + " is no longer failed"
|
||||
return
|
||||
}
|
||||
|
||||
// cutWords is s cut to at most n bytes at a word.
|
||||
func cutWords(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
t := s[:n]
|
||||
if i := strings.LastIndexByte(t, ' '); i > n/2 {
|
||||
t = t[:i]
|
||||
}
|
||||
return strings.TrimRight(t, " ,;:") + "…"
|
||||
}
|
||||
|
||||
// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator
|
||||
// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not
|
||||
// stopped by a person. Each carries why `plans retry` would not run it, when it would not.
|
||||
func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) {
|
||||
plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans cannot be read: %w", err)
|
||||
}
|
||||
var failed []inventory.Plan
|
||||
for _, p := range plans {
|
||||
if failedAndAsked(p) {
|
||||
failed = append(failed, p)
|
||||
}
|
||||
}
|
||||
if len(failed) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||
}
|
||||
onPath := ownPathOf(entries)
|
||||
runsOn := map[string][]string{}
|
||||
for _, e := range entries {
|
||||
runsOn[e.Manifest.Module] = e.On
|
||||
}
|
||||
var out []failedPlanFacts
|
||||
for _, p := range failed {
|
||||
out = append(out, failedFactsOf(p, plans, onPath, runsOn))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not
|
||||
// stopped by a person.
|
||||
func failedAndAsked(p inventory.Plan) bool {
|
||||
return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == ""
|
||||
}
|
||||
|
||||
// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each
|
||||
// module runs.
|
||||
func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string,
|
||||
runsOn map[string][]string) failedPlanFacts {
|
||||
f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p),
|
||||
note: p.Note, since: p.Updated}
|
||||
for _, m := range f.modules {
|
||||
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||
machines := append([]string(nil), runsOn[m]...)
|
||||
sort.Strings(machines)
|
||||
f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why})
|
||||
}
|
||||
}
|
||||
if err := retryRefusal(p, plans, onPath); err != nil {
|
||||
f.refusal = err.Error()
|
||||
var passed passedBy
|
||||
f.passed = errors.As(err, &passed)
|
||||
}
|
||||
return f
|
||||
}
|
||||
@@ -0,0 +1,338 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314).
|
||||
|
||||
func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan {
|
||||
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed,
|
||||
Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC),
|
||||
Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
||||
if mutate != nil {
|
||||
mutate(&p)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts {
|
||||
return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{},
|
||||
map[string][]string{"openrazer": {"g14", "shanks"}})
|
||||
}
|
||||
|
||||
// 1314/AC1
|
||||
func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator ||
|
||||
o.Scope != conditions.ScopePlan || o.ID != p.ID {
|
||||
t.Fatalf("kind, severity, resolver or subject: %+v", o)
|
||||
}
|
||||
if len(o.Actions) != 1 {
|
||||
t.Fatalf("actions: %+v", o.Actions)
|
||||
}
|
||||
a := o.Actions[0]
|
||||
if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove ||
|
||||
len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID {
|
||||
t.Fatalf("the action is not Retry with only the plan: %+v", a)
|
||||
}
|
||||
if o.Needs == "" {
|
||||
t.Fatal("no Needs: the asker wants an action with a need")
|
||||
}
|
||||
for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} {
|
||||
if !strings.Contains(o.Headline+o.Explanation+o.Said, want) {
|
||||
t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") {
|
||||
t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("the words are not plain: %s\n%+v", why, words)
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC2
|
||||
func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) {
|
||||
for name, p := range map[string]inventory.Plan{
|
||||
"held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }),
|
||||
"stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }),
|
||||
"no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }),
|
||||
"building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }),
|
||||
} {
|
||||
if failedAndAsked(p) {
|
||||
t.Errorf("%s: a plan not asked of the operator is asked", name)
|
||||
}
|
||||
}
|
||||
if !failedAndAsked(aFailedPlan("plan-5", nil)) {
|
||||
t.Error("a failed delivery plan is not asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it.
|
||||
func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) })
|
||||
newer := aFailedPlan("plan-2", func(p *inventory.Plan) {
|
||||
p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute)
|
||||
})
|
||||
superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" })
|
||||
done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone })
|
||||
first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC)
|
||||
gate := aFailedPlan("plan-5", func(p *inventory.Plan) {
|
||||
p.Note = "openrazer stopped at its first machine"
|
||||
p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"},
|
||||
Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"},
|
||||
Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}}
|
||||
})
|
||||
for name, c := range map[string]struct {
|
||||
p inventory.Plan
|
||||
others []inventory.Plan
|
||||
says string
|
||||
}{
|
||||
"a newer plan": {older, []inventory.Plan{newer}, "supersedes it"},
|
||||
"superseded": {superseded, nil, "superseded"},
|
||||
"done": {done, nil, "nothing to retry"},
|
||||
"gate failed": {gate, nil, "failed its gate"},
|
||||
} {
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%s: %+v", name, got)
|
||||
}
|
||||
o := got[0]
|
||||
if len(o.Actions) != 0 {
|
||||
t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions)
|
||||
}
|
||||
if !strings.Contains(o.Said, c.says) {
|
||||
t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said)
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) {
|
||||
t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23.
|
||||
func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(t.Context())
|
||||
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ := k.Open(t.Context())
|
||||
if len(open) != 1 || open[0].Kind != "plan-failed" {
|
||||
t.Fatalf("not raised: %+v", open)
|
||||
}
|
||||
// Retried: the plan is building, so it is no longer among the failed.
|
||||
f = calm(now.Add(time.Minute))
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ = k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition outlived the failure: %+v", open)
|
||||
}
|
||||
// Failed again: asked again, with Retry, and nothing retried it.
|
||||
f = calm(now.Add(2 * time.Minute))
|
||||
again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) })
|
||||
f.failed = []failedPlanFacts{failedFactsFor(again)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(t.Context())
|
||||
if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" {
|
||||
t.Fatalf("a second failure was not asked again with Retry: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing
|
||||
// S23 produces is an observation that offers an answer.
|
||||
func TestNoClockRetriesAFailedPlan(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))}
|
||||
for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} {
|
||||
f.now = now.Add(later)
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" {
|
||||
t.Fatalf("after %s: %+v", later, got)
|
||||
}
|
||||
}
|
||||
// A plan failed longer ago than the week it is kept is left to `plans`.
|
||||
f.now = now.Add(planFailedWithin + time.Second)
|
||||
if got := watchPlansFailed(f); len(got) != 0 {
|
||||
t.Fatalf("a plan failed beyond the window is still raised: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A later plan of the same repository and branch, in a state, made after the failed one.
|
||||
func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan {
|
||||
q := aFailedPlan(id, func(p *inventory.Plan) {
|
||||
p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour)
|
||||
})
|
||||
return q
|
||||
}
|
||||
|
||||
// 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan.
|
||||
func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) {
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
||||
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
||||
err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{})
|
||||
if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") {
|
||||
t.Fatalf("the retry was not refused, naming the later plan: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing.
|
||||
func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) {
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
||||
open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed)
|
||||
err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{})
|
||||
if err == nil || !strings.Contains(err.Error(), "supersedes it") {
|
||||
t.Fatalf("with the later plan open the refusal is not today's: %v", err)
|
||||
}
|
||||
other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed)
|
||||
if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil {
|
||||
t.Fatalf("a later done plan of another branch refused the retry: %v", err)
|
||||
}
|
||||
earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed)
|
||||
earlier.Created = failed.Created.Add(-time.Hour)
|
||||
if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil {
|
||||
t.Fatalf("an earlier done plan refused the retry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery.
|
||||
func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) })
|
||||
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(failed, later)}
|
||||
got := watchPlansFailed(f)
|
||||
if len(got) != 1 || len(got[0].Actions) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") {
|
||||
t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation)
|
||||
}
|
||||
if strings.Contains(o.Explanation, "details say why") {
|
||||
t.Errorf("the generic fallback was said: %q", o.Explanation)
|
||||
}
|
||||
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||
t.Errorf("not plain: %s\n%+v", why, words)
|
||||
}
|
||||
}
|
||||
|
||||
// plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since.
|
||||
type plansBehind []inventory.Plan
|
||||
|
||||
func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) {
|
||||
var out []inventory.Plan
|
||||
for _, p := range s {
|
||||
if p.Open() {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) {
|
||||
all := append([]inventory.Plan(nil), s...)
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) })
|
||||
if len(all) > limit {
|
||||
all = all[:limit]
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) {
|
||||
var out []inventory.Plan
|
||||
for _, p := range s {
|
||||
if p.Created.After(since) || p.Open() {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it.
|
||||
func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) {
|
||||
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) })
|
||||
all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)}
|
||||
for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50
|
||||
other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed)
|
||||
other.Repository = "novox/other"
|
||||
other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour)
|
||||
all = append(all, other)
|
||||
}
|
||||
plans, err := plansForRetry(t.Context(), all, failed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") {
|
||||
t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its
|
||||
// plan's last change passes 7 days.
|
||||
func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||
old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) })
|
||||
f := calm(now)
|
||||
f.failed = []failedPlanFacts{failedFactsFor(old)}
|
||||
if got := watchPlansFailed(f); len(got) != 0 {
|
||||
t.Fatalf("a plan last changed 8 days ago raised %+v", got)
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||
defer k.Close(t.Context())
|
||||
fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) })
|
||||
f.failed = []failedPlanFacts{failedFactsFor(fresh)}
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
||||
t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open)
|
||||
}
|
||||
f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old
|
||||
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the condition outlived the 7 days: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -138,7 +138,9 @@ func askToLook() {
|
||||
|
||||
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
|
||||
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
|
||||
func startPlanLooks(ctx context.Context, open *stores) {
|
||||
// It answers a channel that is closed once the look has returned after the context ended, for a test that restores what
|
||||
// the look reads (issue 1338).
|
||||
func startPlanLooks(ctx context.Context, open *stores) <-chan struct{} {
|
||||
l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) })
|
||||
l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) }
|
||||
lookerMu.Lock()
|
||||
@@ -152,8 +154,13 @@ func startPlanLooks(ctx context.Context, open *stores) {
|
||||
}
|
||||
lookerMu.Unlock()
|
||||
}()
|
||||
go l.run(ctx)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
l.run(ctx)
|
||||
}()
|
||||
l.ask()
|
||||
return done
|
||||
}
|
||||
|
||||
// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's
|
||||
@@ -163,8 +170,10 @@ var applySilentAfter = inventory.ApplySilentAfter
|
||||
// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974):
|
||||
// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own
|
||||
// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left
|
||||
// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one.
|
||||
func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) {
|
||||
// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one. It answers
|
||||
// a function that waits until the look at the plans has returned, which only a test that ends the context needs (issue
|
||||
// 1338): the serving controller ignores it.
|
||||
func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) (wait func()) {
|
||||
if err := startRolloutSilent(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err)
|
||||
}
|
||||
@@ -174,7 +183,8 @@ func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream)
|
||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
|
||||
}
|
||||
startPlanLooks(ctx, open)
|
||||
done := startPlanLooks(ctx, open)
|
||||
return func() { <-done }
|
||||
}
|
||||
|
||||
// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a
|
||||
@@ -400,34 +410,39 @@ func syncSilentKeys(ctx context.Context, open *stores) error {
|
||||
}
|
||||
|
||||
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
|
||||
// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
|
||||
// the look is retried.
|
||||
// cannot be read, one cannot be put — the refusal is raised as a condition naming the plan and the machine, never
|
||||
// stood in for by a timer, and the error is given back so the look is retried and writes the key again (novox/hq ADR
|
||||
// 0433 §4). The set of refusals is what this look found: a key written at the next look is no longer in it.
|
||||
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
|
||||
byTimer := func() {
|
||||
for w, sent := range want {
|
||||
silentByTimer(w, sent, silentTTL(sent, now))
|
||||
refused := func(err error) map[silentWait]error {
|
||||
out := map[silentWait]error{}
|
||||
for w := range want {
|
||||
out[w] = err
|
||||
}
|
||||
return out
|
||||
}
|
||||
if c == nil {
|
||||
byTimer()
|
||||
noteRefusedSendKeys(refused(errors.New("the bus's clock for the waits is not there")))
|
||||
return nil
|
||||
}
|
||||
c = boundedClock{c}
|
||||
held, err := c.Held(ctx)
|
||||
if err != nil {
|
||||
byTimer()
|
||||
noteRefusedSendKeys(refused(err))
|
||||
return err
|
||||
}
|
||||
arm, drop := silentDiff(want, held)
|
||||
var first error
|
||||
notTaken := map[silentWait]error{}
|
||||
for _, w := range arm {
|
||||
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
|
||||
silentByTimer(w, want[w], silentTTL(want[w], now))
|
||||
notTaken[w] = err
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
noteRefusedSendKeys(notTaken)
|
||||
for _, w := range drop {
|
||||
if err := c.Reported(ctx, w); err != nil && first == nil {
|
||||
first = err
|
||||
@@ -481,31 +496,6 @@ func (b boundedClock) Reported(ctx context.Context, w silentWait) error {
|
||||
return err
|
||||
}
|
||||
|
||||
var silentTimers = struct {
|
||||
mu sync.Mutex
|
||||
timers map[silentWait]time.Time
|
||||
}{timers: map[silentWait]time.Time{}}
|
||||
|
||||
// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
|
||||
// (DegradableGrants says why that can be): never silently never. Once for each send.
|
||||
func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
|
||||
silentTimers.mu.Lock()
|
||||
defer silentTimers.mu.Unlock()
|
||||
if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
|
||||
return
|
||||
}
|
||||
silentTimers.timers[w] = sent
|
||||
// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
|
||||
time.AfterFunc(ttl, func() {
|
||||
silentTimers.mu.Lock()
|
||||
if silentTimers.timers[w].Equal(sent) {
|
||||
delete(silentTimers.timers, w)
|
||||
}
|
||||
silentTimers.mu.Unlock()
|
||||
askToLook()
|
||||
})
|
||||
}
|
||||
|
||||
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
|
||||
// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires.
|
||||
func startRolloutSilent(ctx context.Context, js *broker.JetStream) error {
|
||||
|
||||
@@ -3,10 +3,12 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
@@ -159,22 +161,76 @@ func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||
return nil, errors.New("the bus is down")
|
||||
}
|
||||
|
||||
// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
|
||||
// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
|
||||
func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
|
||||
// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by
|
||||
// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error
|
||||
// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4).
|
||||
func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1
|
||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
|
||||
err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
|
||||
if err == nil {
|
||||
t.Fatal("no error given back")
|
||||
for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{},
|
||||
"no clock": nil} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
noteRefusedSendKeys(nil)
|
||||
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||
err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent)
|
||||
if clock != nil && err == nil {
|
||||
t.Fatal("no error given back")
|
||||
}
|
||||
got := observeRefusedSendKeys()
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("observations %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused ||
|
||||
!strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") {
|
||||
t.Fatalf("the condition does not name the plan and the machine: %+v", o)
|
||||
}
|
||||
})
|
||||
}
|
||||
silentTimers.mu.Lock()
|
||||
armed := silentTimers.timers[w].Equal(sent)
|
||||
silentTimers.mu.Unlock()
|
||||
if !armed {
|
||||
t.Error("no timer of its own watches the wait")
|
||||
}
|
||||
|
||||
// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is.
|
||||
func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2
|
||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
noteRefusedSendKeys(nil)
|
||||
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||
want := map[silentWait]time.Time{w: sent}
|
||||
if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 {
|
||||
t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys())
|
||||
}
|
||||
clock := &keptClock{}
|
||||
if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !clock.armed[w].Equal(sent) {
|
||||
t.Fatalf("the next look did not write the key again: %v", clock.armed)
|
||||
}
|
||||
if got := observeRefusedSendKeys(); len(got) != 0 {
|
||||
t.Fatalf("the condition stays after the key was written: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type armRefusingClock struct{ silentClock }
|
||||
|
||||
func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||
func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error {
|
||||
return errors.New("the bus refuses the key")
|
||||
}
|
||||
|
||||
// keptClock is a clock that takes every key.
|
||||
type keptClock struct {
|
||||
silentClock
|
||||
armed map[silentWait]time.Time
|
||||
}
|
||||
|
||||
func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||
func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error {
|
||||
if c.armed == nil {
|
||||
c.armed = map[silentWait]time.Time{}
|
||||
}
|
||||
c.armed[w] = sent
|
||||
return nil
|
||||
}
|
||||
|
||||
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
|
||||
@@ -189,14 +245,13 @@ func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||
}
|
||||
|
||||
// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
|
||||
// and every node for good. Every call to the clock has a bound, and the wait is looked at by a timer of its own.
|
||||
// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition.
|
||||
func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
|
||||
was := silentCallBound
|
||||
silentCallBound = 50 * time.Millisecond
|
||||
t.Cleanup(func() { silentCallBound = was })
|
||||
w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
|
||||
select {
|
||||
|
||||
@@ -154,8 +154,33 @@ func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bo
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// newerDonePlan is a done plan of the same repository and branch made after this one: the newer commit it delivered
|
||||
// runs now (novox/hq ADR 0451).
|
||||
func newerDonePlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
var found inventory.Plan
|
||||
var ok bool
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || q.State != inventory.PlanDone || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
if !ok || q.Created.After(found.Created) {
|
||||
found, ok = q, true
|
||||
}
|
||||
}
|
||||
return found, ok
|
||||
}
|
||||
|
||||
// passedBy is the refusal of a retry of a plan that a newer done plan of its repository and branch has passed.
|
||||
type passedBy struct{ plan, by, repository, commit string }
|
||||
|
||||
func (e passedBy) Error() string {
|
||||
return fmt.Sprintf("%s was passed by %s, a newer plan of %s (at %s) that is done: retrying it would build and "+
|
||||
"send the older commit over the newer one that runs", e.plan, e.by, e.repository, short(e.commit))
|
||||
}
|
||||
|
||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error {
|
||||
switch {
|
||||
case p.State == inventory.PlanDone:
|
||||
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||
@@ -165,12 +190,17 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||
}
|
||||
|
||||
// **A newer delivery of the branch has passed it** (novox/hq ADR 0451): retrying would build and send the older
|
||||
// commit over the newer one that runs.
|
||||
if q, found := newerDonePlan(p, plans); found {
|
||||
return passedBy{plan: p.ID, by: q.ID, repository: q.Repository, commit: q.Commit}
|
||||
}
|
||||
if len(failedIn(p)) > 0 {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return oneWalkAtATime(p, plans)
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
if len(stopped) == 0 {
|
||||
@@ -194,20 +224,20 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return oneWalkAtATime(p, plans)
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
|
||||
// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq
|
||||
// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same
|
||||
// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it
|
||||
// plans, which already hold their dependents, and the repositories it carries.
|
||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error {
|
||||
mine := footprintOfWalk(p, nil)
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || !q.Open() || q.Release != nil {
|
||||
continue
|
||||
}
|
||||
if ownWalk(p) || ownWalk(q) {
|
||||
if ownWalk(p, onPath) || ownWalk(q, onPath) {
|
||||
return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+
|
||||
"retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||
}
|
||||
@@ -275,6 +305,32 @@ func goesOnAgain(p *inventory.Plan) {
|
||||
}
|
||||
}
|
||||
|
||||
// planReader is what a retry reads the mesh's plans through.
|
||||
type planReader interface {
|
||||
OpenPlans(ctx context.Context) ([]inventory.Plan, error)
|
||||
RecentPlans(ctx context.Context, limit int) ([]inventory.Plan, error)
|
||||
PlansSince(ctx context.Context, since time.Time) ([]inventory.Plan, error)
|
||||
}
|
||||
|
||||
// plansForRetry are the plans a retry of p judges itself against.
|
||||
func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inventory.Plan, error) {
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **Every plan made after this one** (novox/hq ADR 0451 §1): a plan that passed it is seen however many plans
|
||||
// were made since, not only when it is among the newest.
|
||||
later, err := inv.PlansSince(ctx, p.Created)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(append(plans, recent...), later...), nil
|
||||
}
|
||||
|
||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||
// that tier: what follows is the plan going on as if they had built the first time.
|
||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
@@ -288,15 +344,10 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
plans, err := plansForRetry(ctx, inv, p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans = append(plans, recent...)
|
||||
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
||||
@@ -327,7 +378,11 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
}
|
||||
failed = toRetry(&p, recorded, byID)
|
||||
}
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
onPath, err := ownPathNow(ctx, inv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := retryRefusal(p, plans, onPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// flagsOf reads a command line's flags (and its first positional after --what-if) into a map.
|
||||
func flagsOf(argv []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for i := 0; i < len(argv); i++ {
|
||||
if !strings.HasPrefix(argv[i], "--") {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(argv[i], "--")
|
||||
if i+1 < len(argv) && !strings.HasPrefix(argv[i+1], "--") {
|
||||
out[name] = argv[i+1]
|
||||
i++
|
||||
} else {
|
||||
out[name] = "true"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **The plan of a change as data from `plans`** (novox/hq ADR 0450 §2, issue 1324 AC1): asked for one change
|
||||
// both ways, `plans` with data and `delivery-plan` answer the same plan in every field.
|
||||
func TestPlansWhatIfAsDataAnswersWhatDeliveryPlanAnswers(t *testing.T) {
|
||||
change := map[string]any{"repository": "novox/mesh-catalog", "paths": "modules/gitea/index.ts,modules/gitea/old.txt",
|
||||
"head": "abc123", "base": "main", "module-dirs": "modules/gitea", "removed": "modules/gitea/old.txt"}
|
||||
with := func(extra map[string]any) map[string]any {
|
||||
out := map[string]any{}
|
||||
for k, v := range change {
|
||||
out[k] = v
|
||||
}
|
||||
for k, v := range extra {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
plansArgv, err := argvFor("plans", with(map[string]any{"data": "true"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deliveryArgv, err := argvFor("delivery-plan", change)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plansArgv[0] != "plans" || !slicesHas(plansArgv, "--data") || !slicesHas(plansArgv, "--what-if") {
|
||||
t.Fatalf("plans with data became %q", strings.Join(plansArgv, " "))
|
||||
}
|
||||
p, d := flagsOf(plansArgv), flagsOf(deliveryArgv)
|
||||
for _, name := range []string{"paths", "head", "base", "module-dirs", "removed"} {
|
||||
if p[name] == "" || p[name] != d[name] {
|
||||
t.Errorf("--%s: plans says %q, delivery-plan %q", name, p[name], d[name])
|
||||
}
|
||||
}
|
||||
if p["what-if"] != d["repository"] {
|
||||
t.Errorf("repository: plans says %q, delivery-plan %q", p["what-if"], d["repository"])
|
||||
}
|
||||
|
||||
// Both commands build their document from the one function, from the same member.
|
||||
const src = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
gitea := fromRepo("gitea", src, "modules/gitea")
|
||||
gitea.Source.BuiltFrom = "old"
|
||||
gitea.On = []string{"anchor", "laptop"}
|
||||
entries := []inventory.Entry{gitea}
|
||||
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
member := func(f map[string]string, repository string) orderMember {
|
||||
return changeMember(repository, f["base"], f["head"], splitList(f["paths"]), splitList(f["removed"]),
|
||||
moduleDirsOf(f["module-dirs"]))
|
||||
}
|
||||
a := deliveryPlanDocument(member(p, p["what-if"]), entries, nil, nil, policy)
|
||||
b := deliveryPlanDocument(member(d, d["repository"]), entries, nil, nil, policy)
|
||||
ja, _ := json.Marshal(a)
|
||||
jb, _ := json.Marshal(b)
|
||||
if !reflect.DeepEqual(ja, jb) {
|
||||
t.Errorf("the two plans differ:\n%s\n%s", ja, jb)
|
||||
}
|
||||
plan := a["plan"].(interface{})
|
||||
raw, _ := json.Marshal(plan)
|
||||
var fields map[string]any
|
||||
_ = json.Unmarshal(raw, &fields)
|
||||
for _, want := range []string{"moved", "tiers", "machines", "summary", "class", "head", "base", "repository"} {
|
||||
if _, ok := fields[want]; !ok {
|
||||
t.Errorf("the plan has no %q: %s", want, raw)
|
||||
}
|
||||
}
|
||||
if fields["head"] != "abc123" || fields["base"] != "main" {
|
||||
t.Errorf("head and base are %v and %v", fields["head"], fields["base"])
|
||||
}
|
||||
}
|
||||
|
||||
// Without the data form the text answer is unchanged (AC2): no data flag, and the data-only arguments are not passed.
|
||||
func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) {
|
||||
argv, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a,b"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a,b" {
|
||||
t.Errorf("the text what-if reads %q", got)
|
||||
}
|
||||
if _, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"}); err == nil {
|
||||
t.Error("a head without data was passed over, not refused")
|
||||
}
|
||||
argv, _ = argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "data": "false"})
|
||||
if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a" {
|
||||
t.Errorf("data false reads %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// delivery-plan stays and answers as before (AC3, ADR 0450 §8).
|
||||
func TestDeliveryPlanStaysAsItWas(t *testing.T) {
|
||||
argv, err := argvFor("delivery-plan", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(argv, " "); got != "delivery plan --repository novox/x --paths a --head h" {
|
||||
t.Errorf("delivery-plan reads %q", got)
|
||||
}
|
||||
if _, ok := catalogueVerb("delivery-plan"); !ok {
|
||||
t.Error("delivery-plan is gone")
|
||||
}
|
||||
}
|
||||
|
||||
func catalogueVerb(name string) (catalogue.Verb, bool) { return controllerVerb(name) }
|
||||
|
||||
func slicesHas(s []string, v string) bool {
|
||||
for _, x := range s {
|
||||
if x == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -629,7 +629,7 @@ func heardMachines(d *doctor) map[string]bool {
|
||||
return out
|
||||
}
|
||||
for _, m := range f.machines {
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||
if !m.lastHeard.IsZero() && !m.silent(f.now) && !m.asleep() {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// review round 3, fault B (1181): the test asserted a literal url, while issueOnTheNewBus works out the
|
||||
// machine's reachable address from the catalogue's network records (brokerReachableAt), not from this
|
||||
// process's environment — the real try found "nats://anchor.internal:5671", not the literal guessed here.
|
||||
// Fixed by calling issueProcess directly, with the known broker and reachable address given to it, exactly
|
||||
// as issued_to_vault_test.go's issue/issueWith and issueProvisionerClient are called directly; the url
|
||||
// asserted below is one of those two inputs, never a separate guess.
|
||||
//
|
||||
// review round 1, findings 3 and 4 (1181/AC1, AC2): module issue mints and delivers a process's own
|
||||
// account against the store — no bus to dial, busAddress "" — for mesh-mail's real manifest shape (review
|
||||
// round 1 finding 1: no own secret `broker` at all, its tools served by the tool runner), and issuing it
|
||||
// again is the path every bus account takes (`rollout mint`'s KindProcess case, and a re-run of `module
|
||||
// issue`): it rotates the password, and the old hash no longer matches.
|
||||
func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
theVaultIsInstalled(t, open)
|
||||
|
||||
m := mailSenderWithProcess()
|
||||
m.Moved = true // so handIssuedToVault's accountIsCovered is true, and the hand-off is exercised whole
|
||||
register(t, open, m)
|
||||
|
||||
vault := newStandInVault(t)
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() { vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } })
|
||||
|
||||
pr, found := processPrincipalFor(m, "send")
|
||||
if !found {
|
||||
t.Fatal("mesh-mail declares no process send")
|
||||
}
|
||||
known := broker.Broker{Fingerprint: "ab"}
|
||||
const reachable = "10.0.0.1:4222"
|
||||
issue := func() {
|
||||
t.Helper()
|
||||
if err := issueProcess(ctx, inv, m, pr, "laptop", "", known, reachable); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
issue()
|
||||
user := broker.Principal{Kind: broker.KindProcess, Node: "laptop", Module: "mesh-mail", Process: "send"}.Username()
|
||||
if user != "laptop.mesh-mail.send" {
|
||||
t.Fatalf("the process's username: %q", user)
|
||||
}
|
||||
hash1, held, err := inv.BusUserHash(ctx, user)
|
||||
if err != nil || !held || hash1 == "" {
|
||||
t.Fatalf("the process's account was not minted: held %v, %v", held, err)
|
||||
}
|
||||
|
||||
// 1181/AC1: accepted with the provisioner client's JSON, and handed to the vault whole — the url and
|
||||
// the fingerprint being exactly what issueProcess was given above, not a guess at what it computes.
|
||||
taken := vault.taken["accept"]
|
||||
for _, want := range []string{`"url":"nats://` + reachable + `"`, `"fingerprint":"ab"`, `"node":"laptop"`,
|
||||
`"module":"mesh-mail"`, `"user":"laptop.mesh-mail.send"`, `"password"`} {
|
||||
if !strings.Contains(taken, want) {
|
||||
t.Errorf("the vault was handed %q, without %s", taken, want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(vault.said(), `"name":"broker-send"`) {
|
||||
t.Errorf("the vault was not told the secret's name broker-send: %s", vault.said())
|
||||
}
|
||||
// The file's own owner (mesh-mail-send, review round 1 finding 7's area) is 1170's mechanism, proven
|
||||
// generically there and for this fixture in TestAProcessesOwnSecretIsOwnedByItsOwnAccount.
|
||||
|
||||
// 1181/AC2: issued again — the path `rollout mint`'s KindProcess case and a re-run of `module issue`
|
||||
// both take — rotates the password; the old hash no longer matches what is recorded.
|
||||
issue()
|
||||
hash2, held, err := inv.BusUserHash(ctx, user)
|
||||
if err != nil || !held {
|
||||
t.Fatalf("the process's account is gone after a re-issue: held %v, %v", held, err)
|
||||
}
|
||||
if hash1 == hash2 {
|
||||
t.Fatal("issuing the process's account again did not rotate its password")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// mailSenderWithProcess is mesh-mail's real manifest shape (novox/hq ADR 0436 §7, §8, the lead's fixture):
|
||||
// no own secret named `broker` at all — its tools run in the machine's tool runner — and the process "send"
|
||||
// declares bus, whose own secret broker-send is where module issue delivers its account (ADR 0436 §2),
|
||||
// owned by mesh-mail-send, the user resource the process declares, at mode 0600 as every own secret is.
|
||||
func mailSenderWithProcess() catalogue.Manifest {
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1","emits":["mail.send.sent"],
|
||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
||||
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
||||
"bus":{"emits":["mail.send.sent"]}}]}`))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// review round 1, finding 1 (1181): a module declaring no `broker` of its own at all — mesh-mail's real
|
||||
// shape, tools served by the tool runner — still passes mayIssue on its process's bus declaration alone.
|
||||
func TestAModuleWithNoBrokerAndOnlyAProcessMayBeIssued(t *testing.T) {
|
||||
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
||||
t.Fatalf("a module with no broker of its own, whose process declares bus, was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1181/AC1: a process with a bus declaration and no own secret broker-<process id> cannot be issued — the
|
||||
// same refusal mayIssue already gives a module with no `broker` (novox/hq ADR 0436 §2).
|
||||
func TestAProcessWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
delete(m.OwnSecrets, "broker-send")
|
||||
err := mayIssue(m)
|
||||
if err == nil {
|
||||
t.Fatal("a process with no broker-<id> own secret was issued an account")
|
||||
}
|
||||
for _, want := range []string{"mesh-mail", "send", "broker-send", "own-secrets"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
||||
t.Errorf("a process declaring its own broker-<id> secret was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1181/AC1: the account held for a process is the provisioner client's form — {url, fingerprint, node,
|
||||
// module, user, password} — named for the process, never the module's own account.
|
||||
func TestAProcesssHeldAccountIsTheProvisionersForm(t *testing.T) {
|
||||
user := broker.Principal{Kind: broker.KindProcess, Node: "novox", Module: "mesh-mail", Process: "send"}.Username()
|
||||
if user != "novox.mesh-mail.send" {
|
||||
t.Fatalf("the process's username: %q", user)
|
||||
}
|
||||
held, err := heldProcessAccount(broker.Broker{Fingerprint: "abc"}, "novox.example:4222", "novox", "mesh-mail",
|
||||
user, "a-password")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(held, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]any{"url": "nats://novox.example:4222", "fingerprint": "abc", "node": "novox",
|
||||
"module": "mesh-mail", "user": "novox.mesh-mail.send", "password": "a-password"}
|
||||
for k, v := range want {
|
||||
if got[k] != v {
|
||||
t.Errorf("%s: %v, want %v", k, got[k], v)
|
||||
}
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Errorf("the held account carries %v; want exactly %v (never claims — a process holds no seat)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1181/AC1: composed, the process's own secret belongs to its own account at 0600 (novox/hq ADR 0436 §1, §2;
|
||||
// the owner mechanism is issue 1170's, proven generically there — this is mesh-mail's own shape).
|
||||
func TestAProcessesOwnSecretIsOwnedByItsOwnAccount(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
r := catalogue.Resolution{Node: "novox", Modules: []catalogue.Manifest{m}}
|
||||
out, err := r.Declaration(catalogue.Rendering{
|
||||
Needed: map[string]map[string]string{"mesh-mail": {"broker-send": "sealed-account"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("composing the declaration: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, res := range out {
|
||||
if res["type"] == "file" && res["path"] == "/var/lib/mesh-mail/broker-send" {
|
||||
found = true
|
||||
if got := res["owner"]; got != "mesh-mail-send" {
|
||||
t.Fatalf("broker-send's owner: %v; want mesh-mail-send", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("broker-send was not composed at all")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// mailSenderLiveManifest is mesh-mail's future manifest shape whole (novox/hq ADR 0436 §7, §8): the
|
||||
// process "send" declares every clause of bus (emits, its one work-queue stream, its state, asks, and its
|
||||
// invoked seat verb), so the account module issue delivers for it can be put through every grant at once.
|
||||
func mailSenderLiveManifest(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
|
||||
"emits":["mail.send.sent","mail.send.failed"],"state":["sends"],
|
||||
"streams":[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":8,
|
||||
"retention":"work-queue","max-deliveries":5}],
|
||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
||||
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
||||
"bus":{"emits":["mail.send.sent","mail.send.failed"],"queues":["mail-sends"],"state":["sends"],
|
||||
"asks":true,"invokes":["seat:operator-channel.tell"]}}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// operatorChannelWithTell is operator-channel as messenger declares it (broker.operatorChannel, novox/hq
|
||||
// ADR 0259 §3), with the one verb this release adds: `tell` (novox/hq ADR 0436 §1's own fixture), so the
|
||||
// process's invoked seat verb has a holder to answer it.
|
||||
func operatorChannelWithTell() broker.Seat {
|
||||
return broker.Seat{Name: "operator-channel", Scope: "mesh", Accepts: []string{"ask", "cancel"},
|
||||
Emits: []string{"decided"}, Serves: []string{"open", "history", "notify", "tell"},
|
||||
ByCaller: []string{"ask", "cancel", "decided"}, Records: []string{"messenger_asks"}}
|
||||
}
|
||||
|
||||
// review round 1, finding 5 (1181/AC3), the release's acceptance test, on a real bus and the store: the
|
||||
// account module issue delivers for mesh-mail's process "send" — composed from the manifest the real way
|
||||
// (inventory.ProcessesOf -> broker.Users -> broker.ComposeAccounts), minted with the password module issue
|
||||
// mints (aMesh) — does exactly what its bus declares and nothing else.
|
||||
func TestAProcessAccountOnARealBusAndStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
m := mailSenderLiveManifest(t)
|
||||
operatorChannel := operatorChannelWithTell()
|
||||
mailSends := broker.ModuleStream{Module: "mesh-mail", Name: "MAIL_SENDS", Events: []string{"mail.send.approved"},
|
||||
MaxAgeDays: 1, MaxMiB: 8, WorkQueue: true, Duplicates: 1}
|
||||
records := broker.Records{Nodes: []string{"laptop"}, Assigned: map[string][]broker.Declared{"laptop": {
|
||||
{Module: broker.RuntimeModule},
|
||||
{Module: "messenger", Holds: []broker.Seat{operatorChannel}, RunsAs: "messenger"},
|
||||
{Module: "mesh-mail", NoAccount: true, Uses: []broker.Seat{operatorChannel}, Streams: []broker.ModuleStream{mailSends},
|
||||
Processes: inventory.ProcessesOf(m)},
|
||||
}}}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The four principals this test drives, each minted the password module issue mints — the same
|
||||
// inv.MintBusPassword call issueProcess makes for the process account. mesh-mail publishes its own
|
||||
// stream's events as the tool runner carrying it (review round 2's finding): the controller's own
|
||||
// grants do not cover a module's stream, only the runtime that carries the module does.
|
||||
var admin, runtime, messenger, sender broker.Principal
|
||||
for _, p := range users {
|
||||
switch {
|
||||
case p.Kind == broker.KindController:
|
||||
admin = p
|
||||
case p.Kind == broker.KindNodeTools:
|
||||
runtime = p
|
||||
case p.Module == "messenger":
|
||||
messenger = p
|
||||
case p.Kind == broker.KindProcess:
|
||||
sender = p
|
||||
}
|
||||
}
|
||||
if admin.Kind == "" || runtime.Kind == "" || messenger.Kind == "" || sender.Kind == "" {
|
||||
t.Fatalf("not every principal was composed: admin=%+v runtime=%+v messenger=%+v sender=%+v",
|
||||
admin, runtime, messenger, sender)
|
||||
}
|
||||
if sender.Username() != "laptop.mesh-mail.send" {
|
||||
t.Fatalf("the process's username: %q", sender.Username())
|
||||
}
|
||||
|
||||
passwords := map[string]string{}
|
||||
hashes := map[string]string{}
|
||||
for _, p := range []struct {
|
||||
principal broker.Principal
|
||||
kind string
|
||||
}{{admin, inventory.BusController}, {runtime, inventory.BusNodeTools}, {messenger, inventory.BusModule},
|
||||
{sender, inventory.BusProcess}} {
|
||||
pw, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.principal.Username(), Kind: p.kind,
|
||||
Node: p.principal.Node, Module: p.principal.Module})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
passwords[p.principal.Username()] = pw
|
||||
hash, held, err := inv.BusUserHash(ctx, p.principal.Username())
|
||||
if err != nil || !held {
|
||||
t.Fatalf("the minted password of %s was not recorded: %v, %v", p.principal.Username(), held, err)
|
||||
}
|
||||
hashes[p.principal.Username()] = hash
|
||||
}
|
||||
filled, missing := broker.WithPasswords([]broker.Principal{admin, runtime, messenger, sender}, hashes)
|
||||
if len(missing) > 0 {
|
||||
t.Fatalf("no password for %v", missing)
|
||||
}
|
||||
|
||||
accounts, err := broker.ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
defer s.Shutdown()
|
||||
|
||||
// Every connection names its own inbox prefix: the grant is `_INBOX.<username>.>`, and the client
|
||||
// library's own default random inbox matches nothing granted, so replies are refused without it.
|
||||
adminConn, err := broker.Dial(s.ClientURL(), nats.UserInfo(admin.Username(), passwords[admin.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+admin.Username()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer adminConn.Close()
|
||||
if err := adminConn.EnsureStream(mailSends.Stream()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := adminConn.EnsureConsumer(broker.QueueConsumer(sender, sender.Queues[0])); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := adminConn.EnsureBucket(broker.Bucket{Module: "mesh-mail", Name: "sends"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := jetstream.New(adminConn.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "messenger_asks"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The machine's tool runner, which carries mesh-mail (it has no account of its own) and so is the
|
||||
// principal that publishes its own stream's events on its behalf.
|
||||
runtimeConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(runtime.Username(), passwords[runtime.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+runtime.Username()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer runtimeConn.Close()
|
||||
|
||||
// messenger, the router, answers exactly its granted verbs — never a wildcard the grant does not give it.
|
||||
holder, err := nats.Connect(s.ClientURL(), nats.UserInfo(messenger.Username(), passwords[messenger.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+messenger.Username()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Close()
|
||||
heard := make(chan string, 1)
|
||||
for _, verb := range []string{"open", "history", "notify", "tell"} {
|
||||
if _, err := holder.Subscribe("mesh.seat.operator-channel.tool."+verb, func(m *nats.Msg) {
|
||||
heard <- m.Subject
|
||||
_ = m.Respond([]byte(`{"result":{}}`))
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = holder.Flush()
|
||||
|
||||
refusals := make(chan error, 8)
|
||||
sender2, err := nats.Connect(s.ClientURL(), nats.UserInfo(sender.Username(), passwords[sender.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+sender.Username()),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sender2.Close()
|
||||
|
||||
// 1. Its event.
|
||||
if err := sender2.Publish("mesh.mod.mesh-mail.event.mail.send.sent", []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 2. Its state: writes and reads its own bucket.
|
||||
senderKV, err := jetstream.New(sender2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket, err := senderKV.KeyValue(ctx, "mesh-mail_sends")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := bucket.Put(ctx, "key", []byte("value")); err != nil {
|
||||
t.Fatalf("the process may not write its own state: %v", err)
|
||||
}
|
||||
if got, err := bucket.Get(ctx, "key"); err != nil || string(got.Value()) != "value" {
|
||||
t.Fatalf("the process may not read its own state back: %v, %v", got, err)
|
||||
}
|
||||
|
||||
// 3. Its queue: pulls and acknowledges MAIL_SENDS through its own consumer. Published as the tool
|
||||
// runner carrying mesh-mail — the live path a module's own stream's event takes — never as the
|
||||
// controller, whose own grants do not cover a module's stream at all.
|
||||
runtimeJS, err := jetstream.New(runtimeConn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtimeJS.Publish(ctx, broker.ModuleStreamSubject("mesh-mail", "mail.send.approved"), []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reading, err := jetstream.New(sender2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer, err := reading.Consumer(ctx, "MAIL_SENDS", broker.QueueConsumer(sender, sender.Queues[0]).Name)
|
||||
if err != nil {
|
||||
t.Fatalf("the process may not bind its own queue consumer: %v", err)
|
||||
}
|
||||
batch, err := consumer.Fetch(1, jetstream.FetchMaxWait(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := false
|
||||
for msg := range batch.Messages() {
|
||||
got = true
|
||||
if err := msg.Ack(); err != nil {
|
||||
t.Fatalf("the process may not acknowledge its own queue: %v", err)
|
||||
}
|
||||
}
|
||||
if !got {
|
||||
t.Fatal("the process did not pull the message its own queue holds")
|
||||
}
|
||||
|
||||
// 4. Its invoked seat verb, in its own name.
|
||||
if _, err := sender2.Request("mesh.seat.operator-channel.tool.tell", []byte(`{}`), 3*time.Second); err != nil {
|
||||
t.Fatalf("the process may not call operator-channel.tell: %v", err)
|
||||
}
|
||||
select {
|
||||
case s := <-heard:
|
||||
if s != "mesh.seat.operator-channel.tool.tell" {
|
||||
t.Fatalf("messenger heard %s", s)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("messenger never heard the call")
|
||||
}
|
||||
|
||||
// 5. Refused, one at a time, so each refusal is counted against the attempt that caused it rather than
|
||||
// any other: answering a tool call it holds no seat and serves no tool for.
|
||||
if _, err := sender2.Subscribe("mesh.mod.mesh-mail.tool.>", func(*nats.Msg) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = sender2.Flush()
|
||||
select {
|
||||
case err := <-refusals:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("subscribing a tool call was not refused as a permissions violation: %v", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("subscribing a tool call the process does not serve was not refused")
|
||||
}
|
||||
// 6. Refused: publishing a warrant — only the holder, messenger, may say operator-channel decided.
|
||||
if err := sender2.Publish("mesh.seat.operator-channel.event.decided.mesh-mail", []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = sender2.Flush()
|
||||
select {
|
||||
case err := <-refusals:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("publishing a warrant was not refused as a permissions violation: %v", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("publishing a warrant the process does not hold the seat for was not refused")
|
||||
}
|
||||
}
|
||||
@@ -209,7 +209,8 @@ func serve(ctx context.Context) (err error) {
|
||||
}
|
||||
// And the merges mesh-delivery made, kept a week with the ask each was made on: what registers a module that
|
||||
// moved between repositories on the operator's Approve (novox/hq ADR 0420 §7, issue 1025).
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: open.inventory.Catalogued,
|
||||
kept: following{open: open}.resumeFirstBuilds}); err != nil {
|
||||
return err
|
||||
}
|
||||
// No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable
|
||||
|
||||
@@ -68,7 +68,7 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||
} {
|
||||
err := retryRefusal(c.p, c.others)
|
||||
err := retryRefusal(c.p, c.others, ownPathOf(nil))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||
}
|
||||
@@ -76,20 +76,23 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
stopped := failed
|
||||
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
stopped.Note = "a stopped at its first machine"
|
||||
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
if err := retryRefusal(stopped, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||
}
|
||||
// A failed or done plan does not supersede it.
|
||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour)),
|
||||
plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}); err != nil {
|
||||
// A later failed plan does not supersede it; a later done one passes it (novox/hq ADR 0451).
|
||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}, ownPathOf(nil)); err != nil {
|
||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||
}
|
||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}, ownPathOf(nil)); err == nil ||
|
||||
!strings.Contains(err.Error(), "plan-5") {
|
||||
t.Errorf("a plan a later done plan passed was retried: %v", err)
|
||||
}
|
||||
// Another branch's open walk, or an older one, does not supersede it, and is one walk open: one at a time
|
||||
// (novox/hq ADR 0276).
|
||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||
other.Branch = "release"
|
||||
for _, open := range []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour))} {
|
||||
if err := retryRefusal(failed, []inventory.Plan{open}); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
||||
if err := retryRefusal(failed, []inventory.Plan{open}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
||||
t.Errorf("retried beside the open walk %s: %v", open.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,12 +144,12 @@ func relatedOpen(cand footprint, candOwn bool, open []openWalk, edges []inventor
|
||||
|
||||
// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path
|
||||
// (issue 973).
|
||||
func ownWalk(p inventory.Plan) bool {
|
||||
func ownWalk(p inventory.Plan, onPath map[string]string) bool {
|
||||
if p.OwnPath() {
|
||||
return true
|
||||
}
|
||||
for name := range p.Modules {
|
||||
if _, own := onTheControllersPath[name]; own {
|
||||
if _, own := onPath[name]; own {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -173,19 +173,19 @@ func startedWalks(plans []inventory.Plan, id string, created time.Time) []invent
|
||||
|
||||
// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the
|
||||
// merges the record holds for it.
|
||||
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (openWalk, error) {
|
||||
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan, onPath map[string]string) (openWalk, error) {
|
||||
kept, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
return openWalk{}, err
|
||||
}
|
||||
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p)}, nil
|
||||
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p, onPath)}, nil
|
||||
}
|
||||
|
||||
// openWalksOf are openWalkOf for each of plans.
|
||||
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) ([]openWalk, error) {
|
||||
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan, onPath map[string]string) ([]openWalk, error) {
|
||||
var out []openWalk
|
||||
for _, p := range plans {
|
||||
w, err := openWalkOf(ctx, inv, p)
|
||||
w, err := openWalkOf(ctx, inv, p, onPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -119,19 +119,19 @@ func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) {
|
||||
|
||||
// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973).
|
||||
func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) {
|
||||
if ownWalk(walkOf("novox/one", "notes")) {
|
||||
if ownWalk(walkOf("novox/one", "notes"), ownPathOf(nil)) {
|
||||
t.Error("an ordinary walk read as on the controller's own path")
|
||||
}
|
||||
flagged := walkOf("novox/one", "notes")
|
||||
flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||
if !ownWalk(flagged) {
|
||||
if !ownWalk(flagged, ownPathOf(nil)) {
|
||||
t.Error("a walk whose batch is own-path read as ordinary")
|
||||
}
|
||||
var module string
|
||||
for m := range onTheControllersPath {
|
||||
module = m
|
||||
}
|
||||
if !ownWalk(walkOf("novox/one", "notes", module)) {
|
||||
if !ownWalk(walkOf("novox/one", "notes", module), ownPathOf(nil)) {
|
||||
t.Errorf("a walk planning %s read as ordinary", module)
|
||||
}
|
||||
}
|
||||
@@ -168,15 +168,15 @@ func TestARetryWaitsOnlyForAnOpenWalkItIsRelatedTo(t *testing.T) {
|
||||
p.ID, p.State = id, inventory.PlanBuilding
|
||||
return p
|
||||
}
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err != nil {
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err != nil {
|
||||
t.Errorf("refused beside an unrelated walk: %v", err)
|
||||
}
|
||||
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")})
|
||||
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")}, ownPathOf(nil))
|
||||
if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") {
|
||||
t.Errorf("a retry beside a walk moving the same module: %v", err)
|
||||
}
|
||||
failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err == nil {
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err == nil {
|
||||
t.Error("an own-path retry went beside an open walk")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -701,11 +702,15 @@ func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Pl
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
open, err := openWalksOf(ctx, inv, started)
|
||||
onPath, err := ownPathNow(ctx, inv)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
mine, err := openWalkOf(ctx, inv, p)
|
||||
open, err := openWalksOf(ctx, inv, started, onPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
mine, err := openWalkOf(ctx, inv, p, onPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
@@ -938,6 +943,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return false, err
|
||||
}
|
||||
silent := silentNodes(*p, allReports, time.Now().UTC(), provLookup(ctx))
|
||||
// **A machine away when a tier is sent is not sent to and not waited for** (ADR 0435 §1, issue 1161): read once,
|
||||
// for a tier that has something left to send.
|
||||
var away map[string]bool
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s != nil && s.SentAt == nil && s.FirstAt == nil && s.State != planDeleted {
|
||||
away = judgedAway(ctx, open, time.Now().UTC())
|
||||
break
|
||||
}
|
||||
}
|
||||
var pending []string
|
||||
firstOn := map[string][]string{}
|
||||
runningOf := map[string][]string{}
|
||||
@@ -974,6 +988,16 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
if state.FirstAt == nil {
|
||||
running = withoutSilent(running, silent)
|
||||
passed := withoutSilent(running, away)
|
||||
if len(passed) < len(running) {
|
||||
for _, n := range running {
|
||||
if away[n] && !slices.Contains(p.Away, n) {
|
||||
fmt.Printf("%s: %s is away; %s is not sent to it and not waited for\n", p.ID, n, m)
|
||||
}
|
||||
}
|
||||
rememberAway(p, running, away)
|
||||
running = passed
|
||||
}
|
||||
}
|
||||
runningOf[m] = running
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
|
||||
@@ -1189,6 +1213,8 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if state.State == planDeleted {
|
||||
continue
|
||||
}
|
||||
// A machine passed as away was sent nothing to wait on (ADR 0435 §1, issue 1161).
|
||||
running = runningHeard(*p, running, state)
|
||||
// The plan sends what it waits for. A rebuild from the same source commit is not a
|
||||
// move the catalogue announces — the build machine rebuilt for a controller change
|
||||
// is one — so the roll-out that opens this gate is the plan's to make, once, and
|
||||
@@ -1292,6 +1318,10 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if end == "" {
|
||||
end = leftAsWas
|
||||
}
|
||||
// A machine passed as away is said where the others passed are said (ADR 0435 §1, issue 1161).
|
||||
if said := awaySaying(*p); said != "" {
|
||||
end = strings.TrimPrefix(end+"; "+said, "; ")
|
||||
}
|
||||
p.Tier++
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = end
|
||||
@@ -2008,6 +2038,12 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||
moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+
|
||||
"comma-separated, as the forge's announcer says them (novox/hq issue 278)")
|
||||
// The what-if as data, the document `delivery plan` answers (novox/hq ADR 0450 §2, issue 1324).
|
||||
data := set.Bool("data", false, "with --what-if: answer the change plan as data, the document `delivery plan` answers; "+
|
||||
"without it the answer is the text a person reads")
|
||||
head := set.String("head", "", "with --data: the commit at hand")
|
||||
base := set.String("base", "main", "with --data: the branch it merges into")
|
||||
removed := set.String("removed", "", "with --data: the files among --paths it deletes, comma-separated")
|
||||
// Following a plan (novox/hq issue 229): `plans wait <id>`.
|
||||
until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)")
|
||||
bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+
|
||||
@@ -2123,6 +2159,25 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if *data && *whatIf == "" {
|
||||
return errors.New("--data answers a what-if: give --what-if owner/repository")
|
||||
}
|
||||
if *whatIf != "" && *data {
|
||||
if *paths == "" {
|
||||
return errors.New("plans --what-if --data: the change plan is of the files a change touches; give --paths")
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m := changeMember(*whatIf, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs))
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
}))
|
||||
}
|
||||
if *whatIf != "" {
|
||||
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules), moduleDirsOf(*moduleDirs))
|
||||
}
|
||||
|
||||
@@ -392,6 +392,32 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
}
|
||||
modules++
|
||||
|
||||
case broker.KindProcess:
|
||||
// One process's own account (novox/hq ADR 0436 §1, §2), delivered as the module's own secret
|
||||
// broker-<process id> — minted here exactly as `module issue` mints it, so a process issued
|
||||
// once still gets a new password every time the bus's accounts are issued again.
|
||||
m, inShelf := shelf[p.Module]
|
||||
if !inShelf {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
pr, found := processPrincipalFor(m, p.Process)
|
||||
if !found {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
name := processSecretName(p.Process)
|
||||
if _, reads := m.OwnSecrets[name]; !reads {
|
||||
fmt.Printf(" %s's process %s declares bus but declares no `%s` secret to receive a "+
|
||||
"credential in; skipped\n", p.Module, p.Process, name)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if err := issueProcess(ctx, inv, m, pr, p.Node, "", known, busAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
modules++
|
||||
|
||||
default:
|
||||
skipped++
|
||||
}
|
||||
|
||||
@@ -179,7 +179,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
|
||||
// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
|
||||
first, stopFirst := context.WithCancel(ctx)
|
||||
defer stopFirst()
|
||||
startRolloutEvents(first, open, js)
|
||||
waitFirst := startRolloutEvents(first, open, js)
|
||||
built := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
@@ -199,12 +199,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
|
||||
|
||||
// The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody.
|
||||
stopFirst()
|
||||
waitFirst()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
reports("anchor")
|
||||
|
||||
second, stopSecond := context.WithCancel(ctx)
|
||||
defer stopSecond()
|
||||
startRolloutEvents(second, open, js)
|
||||
waitSecond := startRolloutEvents(second, open, js)
|
||||
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
||||
t.Cleanup(func() { stopSecond(); waitSecond() })
|
||||
// Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire.
|
||||
eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool {
|
||||
h := heldKeys()
|
||||
@@ -321,7 +324,7 @@ func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) {
|
||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||
started, stop := context.WithCancel(ctx)
|
||||
defer stop()
|
||||
startRolloutEvents(started, open, js)
|
||||
wait := startRolloutEvents(started, open, js)
|
||||
time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing
|
||||
|
||||
// The verb as its child process runs it: the delivery command, whose saves are not the serving controller's.
|
||||
@@ -333,6 +336,9 @@ func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) {
|
||||
return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"})
|
||||
}
|
||||
t.Cleanup(func() { runTheVerb = wasRun })
|
||||
// Registered after every variable cleanup above, so it runs before them (last in, first out): the look at the plans
|
||||
// has returned when checkEvents, inventory.PlanSaved and runTheVerb are restored (issue 1338).
|
||||
t.Cleanup(func() { stop(); wait() })
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -42,7 +42,7 @@ func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
|
||||
return s, heard
|
||||
}
|
||||
|
||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
|
||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { // 1157/AC3: the send keys stay until release 6 (ADR 0433 §5)
|
||||
s, heard := silentOnABus(t)
|
||||
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
|
||||
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// seatPassing is what passing seats on needs of the outside: the record's compare-and-set, and the hand act.
|
||||
type seatPassing struct {
|
||||
pass func(ctx context.Context, p catalogue.SeatPass) (bool, error)
|
||||
act func(ctx context.Context, p catalogue.SeatPass) error
|
||||
say io.Writer
|
||||
}
|
||||
|
||||
// passSeatsOn records, in the composition that finds it, the new holder of each seat held once whose holder on
|
||||
// record is assigned nowhere and which has exactly one claimant able to hold it, and writes the controller's own
|
||||
// hand act for it (novox/hq ADR 0449 §4, issue 1317). It answers the holdings as they are after it, so the
|
||||
// resolution that follows reads the record it just moved. A failed write is said and leaves the record as it was:
|
||||
// a composition is never refused for it.
|
||||
func passSeatsOn(ctx context.Context, sp seatPassing, holdings []catalogue.Held,
|
||||
assignments []catalogue.Assignment) []catalogue.Held {
|
||||
passes, said := catalogue.SeatsToPassOn(holdings, assignments)
|
||||
for _, line := range said {
|
||||
fmt.Fprintln(sp.say, line)
|
||||
}
|
||||
for _, p := range passes {
|
||||
moved, err := sp.pass(ctx, p)
|
||||
if err != nil {
|
||||
fmt.Fprintf(sp.say, "%s could not be passed from %s on %s to %s on %s: %v\n", p.Seat.Name,
|
||||
p.From.Module, p.From.Node, p.To.Module, p.To.Node, err)
|
||||
continue
|
||||
}
|
||||
if !moved {
|
||||
continue // another composition passed it first, and wrote the act
|
||||
}
|
||||
if err := sp.act(ctx, p); err != nil {
|
||||
fmt.Fprintf(sp.say, "%s passed from %s on %s to %s on %s, and the hand act could NOT be recorded: %v\n",
|
||||
p.Seat.Name, p.From.Module, p.From.Node, p.To.Module, p.To.Node, err)
|
||||
}
|
||||
var next []catalogue.Held
|
||||
for _, h := range holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == p.Seat.Name {
|
||||
continue
|
||||
}
|
||||
next = append(next, h)
|
||||
}
|
||||
p.To.Site = p.From.Site
|
||||
holdings = append(next, p.To)
|
||||
}
|
||||
return holdings
|
||||
}
|
||||
|
||||
// seatPassHandAct is the controller's own entry for a pass: it is the act that puts the holder on record,
|
||||
// where a person's `seat --to` would have, so the record of who held what stays complete (ADR 0449 §4).
|
||||
func seatPassHandAct(p catalogue.SeatPass) link.HandAct {
|
||||
return link.HandAct{
|
||||
By: "the controller",
|
||||
Verb: "seat",
|
||||
Args: []string{p.Seat.Name, "--to", p.To.Node + "/" + p.To.Module, "--was", p.From.Node + "/" + p.From.Module},
|
||||
Why: fmt.Sprintf("%s on %s, the holder on record of %s, is assigned on no machine, and %s on %s is the only "+
|
||||
"assignment that claims it and can hold it (ADR 0449 §4)", p.From.Module, p.From.Node, p.Seat.Name,
|
||||
p.To.Module, p.To.Node),
|
||||
Cause: "seat-passed",
|
||||
Outcome: "done",
|
||||
}
|
||||
}
|
||||
|
||||
// assignmentsOf are the assignments of every machine, as the passing of a seat reads them. **A module assigned and
|
||||
// missing from the shelf is still assigned**: it is an assignment with no claims, so its holder is not taken for gone.
|
||||
func assignmentsOf(shelf map[string]catalogue.Manifest, assigned map[string][]string) []catalogue.Assignment {
|
||||
var out []catalogue.Assignment
|
||||
for node, modules := range assigned {
|
||||
for _, module := range modules {
|
||||
m, ok := shelf[module]
|
||||
if !ok {
|
||||
m = catalogue.Manifest{Module: module}
|
||||
}
|
||||
out = append(out, catalogue.Assignment{Node: node, Manifest: m})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatPassingOn is how a composition on this inventory passes a seat: against the store's record with the hand act
|
||||
// on the bus, or, on a throwaway mesh (the merge gate's), in memory only, writing nothing anywhere.
|
||||
func seatPassingOn(inv *inventory.Inventory, w io.Writer) seatPassing {
|
||||
if inv.Throwaway() {
|
||||
return seatPassing{say: w,
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return true, nil },
|
||||
act: func(context.Context, catalogue.SeatPass) error { return nil }}
|
||||
}
|
||||
return seatPassing{
|
||||
say: w,
|
||||
pass: func(ctx context.Context, p catalogue.SeatPass) (bool, error) {
|
||||
return inv.PassSeat(ctx, p.Seat.Name, p.Seat.Scope, p.From, p.To)
|
||||
},
|
||||
act: func(ctx context.Context, p catalogue.SeatPass) error {
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
_, err := link.RecordHandAct(ctx, conn, seatPassHandAct(p))
|
||||
return err
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// theMeshPassesSeatsOn is passSeatsOn as the composition runs it. **Only when every machine's assignments were
|
||||
// read** (assignedErr nil): a machine whose read failed would look as if its modules were assigned nowhere, and
|
||||
// its holder would lose the seat. Said to where the composition says its other lines (sayLeftOut).
|
||||
func theMeshPassesSeatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
||||
assigned map[string][]string, assignedErr error, holdings []catalogue.Held) []catalogue.Held {
|
||||
var w io.Writer = os.Stderr
|
||||
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
||||
w = lw
|
||||
}
|
||||
if assignedErr != nil {
|
||||
fmt.Fprintf(w, "no seat is passed on in this composition: a machine's assignments could not be read, so a "+
|
||||
"holder on record might look assigned nowhere: %v\n", assignedErr)
|
||||
return holdings
|
||||
}
|
||||
return passSeatsOn(ctx, seatPassingOn(inv, w), holdings, assignmentsOf(shelf, assigned))
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aBrokerClaiming(module string) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: module,
|
||||
Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}
|
||||
}
|
||||
|
||||
func withBusSeat(t *testing.T) {
|
||||
t.Helper()
|
||||
was := catalogue.Seats()
|
||||
t.Cleanup(func() { catalogue.UseSeats(was) })
|
||||
catalogue.UseSeats([]catalogue.Seat{{Name: "mesh-broker", Scope: catalogue.ScopeMesh, Delivers: "mesh-bus", Decision: "test"}})
|
||||
}
|
||||
|
||||
// 1317/AC1: the composition records the claimant as the holder and writes one hand act by the controller naming
|
||||
// the seat, the former holder and the new one; the holdings it answers name the new holder.
|
||||
func TestACompositionPassesTheSeatAndWritesOneHandActByTheController(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
var passed []catalogue.SeatPass
|
||||
var acts []link.HandAct
|
||||
var said bytes.Buffer
|
||||
sp := seatPassing{say: &said,
|
||||
pass: func(_ context.Context, p catalogue.SeatPass) (bool, error) {
|
||||
passed = append(passed, p)
|
||||
return true, nil
|
||||
},
|
||||
act: func(_ context.Context, p catalogue.SeatPass) error {
|
||||
acts = append(acts, seatPassHandAct(p))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||
if len(got) != 1 || got[0].Module != "gate" || got[0].Node != "novox" {
|
||||
t.Fatalf("the holdings after the composition do not name the gate: %+v", got)
|
||||
}
|
||||
if len(passed) != 1 || len(acts) != 1 {
|
||||
t.Fatalf("passes=%d acts=%d, want one each", len(passed), len(acts))
|
||||
}
|
||||
a := acts[0]
|
||||
text := strings.Join(append([]string{a.By, a.Verb, a.Why}, a.Args...), " ")
|
||||
for _, want := range []string{"the controller", "mesh-broker", "novox/old-broker", "novox/gate"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("the hand act does not name %q: %+v", want, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1317/AC2: two claimants, nothing recorded, the refusal said.
|
||||
func TestACompositionWithTwoClaimantsRecordsNoneAndSaysSo(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
sp := seatPassing{say: &bytes.Buffer{},
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) {
|
||||
t.Fatal("a seat with two claimants was passed")
|
||||
return false, nil
|
||||
},
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("an act was written"); return nil },
|
||||
}
|
||||
var said bytes.Buffer
|
||||
sp.say = &said
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{
|
||||
{Node: "novox", Manifest: aBrokerClaiming("gate")}, {Node: "ace", Manifest: aBrokerClaiming("other")}})
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("the record changed: %+v", got)
|
||||
}
|
||||
if !strings.Contains(said.String(), "gate") || !strings.Contains(said.String(), "other") ||
|
||||
!strings.Contains(said.String(), "seat mesh-broker --to") {
|
||||
t.Fatalf("the refusal is not said with both and the remedy: %q", said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 1317/AC3 and AC4: nothing changes while the holder is assigned, nor with no claimant.
|
||||
func TestACompositionChangesNothingWhileTheHolderIsAssignedOrNobodyClaims(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
sp := seatPassing{say: &bytes.Buffer{},
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { t.Fatal("passed"); return false, nil },
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil },
|
||||
}
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{
|
||||
{Node: "novox", Manifest: aBrokerClaiming("old-broker")}, {Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("holder assigned: record changed: %+v", got)
|
||||
}
|
||||
got = passSeatsOn(t.Context(), sp, holdings, nil)
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("no claimant: record changed: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A composition that lost the compare-and-set writes no act, and a failed write leaves the record.
|
||||
func TestALostPassWritesNoActAndAFailedOneLeavesTheRecord(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
as := []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}}
|
||||
lost := seatPassing{say: &bytes.Buffer{},
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, nil },
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act without a pass"); return nil },
|
||||
}
|
||||
passSeatsOn(t.Context(), lost, holdings, as)
|
||||
var said bytes.Buffer
|
||||
sp := seatPassing{say: &said,
|
||||
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, context.DeadlineExceeded },
|
||||
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil },
|
||||
}
|
||||
got := passSeatsOn(t.Context(), sp, holdings, as)
|
||||
if got[0].Module != "old-broker" || !strings.Contains(said.String(), "could not be passed") {
|
||||
t.Fatalf("got %+v said %q", got, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A pass is the mesh working as ADR 0449 §4 decided, never a repair: two of them raise no healer-wanted and
|
||||
// count as no repair.
|
||||
func TestTwoSeatPassesAreNoRepairAndRaiseNoHealerWanted(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
now := time.Date(2026, 10, 19, 20, 0, 0, 0, time.UTC)
|
||||
mk := func(at time.Time) link.HandAct {
|
||||
a := seatPassHandAct(catalogue.SeatPass{Seat: catalogue.Seat{Name: "mesh-broker"},
|
||||
From: catalogue.Held{Node: "novox", Module: "old"}, To: catalogue.Held{Node: "novox", Module: "gate"}})
|
||||
a.At = at
|
||||
return a
|
||||
}
|
||||
acts := []link.HandAct{mk(now.Add(-26 * time.Hour)), mk(now.Add(-time.Hour))}
|
||||
if r := repairs(acts); len(r) != 0 {
|
||||
t.Fatalf("a seat pass counts as a repair: %+v", r)
|
||||
}
|
||||
f := calm(now)
|
||||
f.handActs = acts
|
||||
if got := watchHandActs(f); len(got) != 0 {
|
||||
t.Fatalf("two passes raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed read of one machine's assignments passes no seat: its holder would look assigned nowhere.
|
||||
func TestNoSeatIsPassedWhenAMachinesAssignmentsCouldNotBeRead(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
var said bytes.Buffer
|
||||
ctx := leftOutSaidBy(t.Context(), &said)
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "ace", Module: "old-broker"}}
|
||||
shelf := map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")}
|
||||
got := theMeshPassesSeatsOn(ctx, nil, shelf, map[string][]string{"novox": {"gate"}},
|
||||
errors.New("ace: the bus did not answer"), holdings)
|
||||
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||
t.Fatalf("the record moved: %+v", got)
|
||||
}
|
||||
if !strings.Contains(said.String(), "no seat is passed") || !strings.Contains(said.String(), "ace") {
|
||||
t.Errorf("not said: %q", said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A holder that is assigned but missing from the shelf is still assigned.
|
||||
func TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
as := assignmentsOf(map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")},
|
||||
map[string][]string{"novox": {"gate", "old-broker"}})
|
||||
passes, _ := catalogue.SeatsToPassOn(
|
||||
[]catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as)
|
||||
if len(passes) != 0 {
|
||||
t.Fatalf("a holder missing from the shelf lost its seat: %+v", passes)
|
||||
}
|
||||
}
|
||||
|
||||
// The merge gate's throwaway mesh passes the seat in memory and writes nothing: no record, no hand act.
|
||||
func TestAThrowawayMeshPassesTheSeatInMemoryAndWritesNothing(t *testing.T) {
|
||||
withBusSeat(t)
|
||||
inv := &inventory.Inventory{}
|
||||
inv.MarkThrowaway()
|
||||
var said bytes.Buffer
|
||||
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||
got := passSeatsOn(t.Context(), seatPassingOn(inv, &said), holdings,
|
||||
[]catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||
if len(got) != 1 || got[0].Module != "gate" {
|
||||
t.Fatalf("the gate's view does not hold the pass: %+v", got)
|
||||
}
|
||||
if said.Len() != 0 {
|
||||
t.Errorf("the throwaway composition said %q (a write was tried)", said.String())
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -351,6 +352,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if d := str("module-dirs"); d != "" {
|
||||
argv = append(argv, "--module-dirs", d)
|
||||
}
|
||||
// The plan as data, the document delivery-plan answers (novox/hq ADR 0450 §2, issue 1324).
|
||||
if str("data") == "true" {
|
||||
argv = append(argv, "--data")
|
||||
for _, flag := range []string{"head", "base", "removed"} {
|
||||
if v := str(flag); v != "" {
|
||||
argv = append(argv, "--"+flag, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
for _, act := range []string{"stop", "close", "retry", "go"} {
|
||||
@@ -408,14 +418,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if err := need("members"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"delivery", "order", "--members", str("members")}, nil
|
||||
members, err := membersFlags(str("members"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]string{"delivery", "order"}, members...), nil
|
||||
case "delivery-check":
|
||||
if err := need("members"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"delivery", "check", "--members", str("members")}
|
||||
members, err := membersFlags(str("members"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := append([]string{"delivery", "check"}, members...)
|
||||
if g := str("group"); g != "" {
|
||||
argv = []string{"delivery", "check", "--group", g, "--members", str("members")}
|
||||
argv = append([]string{"delivery", "check", "--group", g}, members...)
|
||||
}
|
||||
// Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check.
|
||||
if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" {
|
||||
@@ -1193,6 +1211,7 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
if handingOver.Load() {
|
||||
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
|
||||
}
|
||||
defer removeMembersFile(argv)
|
||||
cmd := selfCommand(ctx, argv)
|
||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||
caller := link.CallerIn(ctx)
|
||||
@@ -1858,3 +1877,41 @@ func terminalOnly(argv []string) error {
|
||||
"decide who may become root on it (novox/hq ADR 0266). A verb may run node list and node show. "+
|
||||
"Nothing was done", sub)
|
||||
}
|
||||
|
||||
// membersInArgument is how long a group's members may be as one argument of the command a verb re-executes: over
|
||||
// the kernel's 128 KiB for one argument the command cannot start (novox/hq issue 1272).
|
||||
const membersInArgument = 32 << 10
|
||||
|
||||
// membersDirPrefix names the private directory a long list of members is handed over in.
|
||||
const membersDirPrefix = "mesh-members-"
|
||||
|
||||
// membersFlags is a group's members as the flags of the command a verb re-executes: `--members <json>` while the
|
||||
// list is short, as a person would type it, and `--members-file <path>` to a file in a private directory of its own
|
||||
// when it is not. runVerb removes the directory once the command has ended.
|
||||
func membersFlags(members string) ([]string, error) {
|
||||
if len(members) <= membersInArgument {
|
||||
return []string{"--members", members}, nil
|
||||
}
|
||||
dir, err := os.MkdirTemp("", membersDirPrefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the members cannot be handed to the command: %w", err)
|
||||
}
|
||||
file := filepath.Join(dir, "members.json")
|
||||
if err := os.WriteFile(file, []byte(members), 0o600); err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return nil, fmt.Errorf("the members cannot be handed to the command: %w", err)
|
||||
}
|
||||
return []string{"--members-file", file}, nil
|
||||
}
|
||||
|
||||
// removeMembersFile removes the private directory membersFlags made for a command line, and only such a directory.
|
||||
func removeMembersFile(argv []string) {
|
||||
for i, a := range argv {
|
||||
if a == "--members-file" && i+1 < len(argv) {
|
||||
dir := filepath.Dir(argv[i+1])
|
||||
if strings.HasPrefix(filepath.Base(dir), membersDirPrefix) {
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and
|
||||
// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes
|
||||
// the key again; the look that writes it leaves it out of the set, and the condition clears.
|
||||
const (
|
||||
kindSendKeyRefused = "send-key-refused"
|
||||
sendKeyRefusedID = "DL"
|
||||
)
|
||||
|
||||
var refusedSendKeys = struct {
|
||||
mu sync.Mutex
|
||||
by map[silentWait]error
|
||||
}{}
|
||||
|
||||
// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found.
|
||||
func noteRefusedSendKeys(refused map[silentWait]error) {
|
||||
refusedSendKeys.mu.Lock()
|
||||
defer refusedSendKeys.mu.Unlock()
|
||||
refusedSendKeys.by = refused
|
||||
}
|
||||
|
||||
// observeRefusedSendKeys is one condition per plan and machine whose send key was refused.
|
||||
func observeRefusedSendKeys() []conditions.Observation {
|
||||
refusedSendKeys.mu.Lock()
|
||||
waits := make([]silentWait, 0, len(refusedSendKeys.by))
|
||||
why := map[silentWait]error{}
|
||||
for w, err := range refusedSendKeys.by {
|
||||
waits = append(waits, w)
|
||||
why[w] = err
|
||||
}
|
||||
refusedSendKeys.mu.Unlock()
|
||||
sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) })
|
||||
seen := map[[2]string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, w := range waits {
|
||||
if seen[[2]string{w.Plan, w.Node}] {
|
||||
continue // a plan sends a machine one tier at a time; one condition says it
|
||||
}
|
||||
seen[[2]string{w.Plan, w.Node}] = true
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node,
|
||||
Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+
|
||||
"names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node),
|
||||
Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeSendKeyRefused reads what the newest look of the plans could not write.
|
||||
func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
return observeRefusedSendKeys(), nil
|
||||
}
|
||||
@@ -234,6 +234,16 @@ var signalsTable = []signalRow{
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.batches, func(b batchFacts) time.Time { return b.created })
|
||||
}},
|
||||
{Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan",
|
||||
Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)",
|
||||
Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " +
|
||||
"words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " +
|
||||
"from the plan's last change, and ended when the plan is no longer failed",
|
||||
Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3,
|
||||
needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed,
|
||||
newest: func(f *signalFacts) time.Time {
|
||||
return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
|
||||
}},
|
||||
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
||||
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
||||
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
||||
@@ -318,6 +328,10 @@ func ago(d time.Duration) string {
|
||||
return d.Round(time.Minute).String()
|
||||
}
|
||||
|
||||
// silentBound is how long a machine may go unheard, where the watchman gives no verdict: three of the intervals nodes have
|
||||
// always used, fixed now that no machine says its own (issue 1183).
|
||||
const silentBound = heartbeatsMissed * heartbeatEvery
|
||||
|
||||
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||
func heartbeatBound(every time.Duration) time.Duration {
|
||||
if every <= 0 {
|
||||
@@ -333,9 +347,9 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||
continue
|
||||
}
|
||||
bound := heartbeatBound(m.every)
|
||||
bound := silentBound
|
||||
silent := f.now.Sub(m.lastHeard)
|
||||
if silent <= bound {
|
||||
if !m.silent(f.now) {
|
||||
continue
|
||||
}
|
||||
severity := conditions.Warning
|
||||
@@ -357,7 +371,7 @@ func watchReports(f *signalFacts) []conditions.Observation {
|
||||
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||
continue
|
||||
}
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
if m.silent(f.now) {
|
||||
continue // silent: S1 says it, and a silent machine reports nothing
|
||||
}
|
||||
bound := max(reportAtLeast, 3*m.lastApply)
|
||||
@@ -400,6 +414,34 @@ func watchPlans(f *signalFacts) []conditions.Observation {
|
||||
return out
|
||||
}
|
||||
|
||||
// kindPlanFailed is S23's kind: plan.<id>.plan-failed.
|
||||
const kindPlanFailed = "plan-failed"
|
||||
|
||||
// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no
|
||||
// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to
|
||||
// the operator). A plan failed longer ago than this is left to `plans`.
|
||||
const planFailedWithin = 7 * 24 * time.Hour
|
||||
|
||||
// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when
|
||||
// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2).
|
||||
// The condition ends when the plan is no longer failed, because it is no longer among the facts.
|
||||
func watchPlansFailed(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, p := range f.failed {
|
||||
if f.now.Sub(p.since) > planFailedWithin {
|
||||
continue
|
||||
}
|
||||
headline, explanation, needs, actions, resolved := planFailedWords(p)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed,
|
||||
Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit),
|
||||
p.failureSaid()),
|
||||
Said: p.failureSaid() + p.refusalSaid(),
|
||||
Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// kindWalkWaiting is S16's kind: plan.<id>.waiting.
|
||||
const kindWalkWaiting = "waiting"
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
func calm(now time.Time) *signalFacts {
|
||||
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||
sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||
@@ -160,6 +160,17 @@ var suppressions = map[string]suppression{
|
||||
created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}}
|
||||
},
|
||||
},
|
||||
// A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449).
|
||||
"S23": {
|
||||
inside: func(f *signalFacts) {
|
||||
f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
|
||||
since: f.now.Add(-planFailedWithin - time.Second)}}
|
||||
},
|
||||
past: func(f *signalFacts) {
|
||||
f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
|
||||
since: f.now.Add(-time.Second)}}
|
||||
},
|
||||
},
|
||||
// A batch queued behind an open walk past that walk's bound, named.
|
||||
"S19": {
|
||||
inside: func(f *signalFacts) {
|
||||
|
||||
@@ -171,7 +171,9 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT
|
||||
}}
|
||||
first, stopFirst := context.WithCancel(ctx)
|
||||
defer stopFirst()
|
||||
startRolloutEvents(first, open, js)
|
||||
waitFirst := startRolloutEvents(first, open, js)
|
||||
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
||||
t.Cleanup(func() { stopFirst(); waitFirst() })
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -414,18 +414,11 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.nodes, err = inv.Nodes(ctx)
|
||||
out.nodes, err = open.heardNodes(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
for _, n := range out.nodes {
|
||||
// Never heard from, or not lately. Different from failing: a machine that says nothing
|
||||
// may be new, switched off, or unreachable, and none of those is a machine that tried
|
||||
// and could not.
|
||||
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
|
||||
out.quiet = append(out.quiet, n)
|
||||
}
|
||||
}
|
||||
out.quiet = notHeardLately(out.nodes, time.Now())
|
||||
out.behind, err = inv.Behind(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// What stopped moving (novox/hq ADR 0433 §2, §3, issue 1156). The watchman writes `stuck.<bucket>.<key>` in its own
|
||||
// bucket for each key whose pulse expired, and again as moved when the key moves; it never deletes one. The controller
|
||||
// reads that bucket in two places and holds no timer for either: a probe of the self-check raises each key not moved as
|
||||
// a condition of the delivery scope, and the check that a machine is silent takes a node's presence key as its verdict.
|
||||
const (
|
||||
kindStuck = "stuck"
|
||||
probeStuckID = "DM"
|
||||
)
|
||||
|
||||
// presenceVerdict is the watchman's word on whether a machine is online, from its stuck presence key.
|
||||
type presenceVerdict int
|
||||
|
||||
const (
|
||||
// presenceUnsaid: the watchman has written nothing for the machine; the heartbeat rule judges it, as before.
|
||||
presenceUnsaid presenceVerdict = iota
|
||||
// presenceHeard: the key says moved, the machine is back online.
|
||||
presenceHeard
|
||||
// presenceSilent: the key says not moved, the machine was offline past its bound.
|
||||
presenceSilent
|
||||
)
|
||||
|
||||
// presenceEntry is presence.<node> as the watchman wrote it: whether the machine is online and when the entry was made.
|
||||
// The watchman writes it only when it changes, so for an online machine the time is when it connected.
|
||||
type presenceEntry struct {
|
||||
Online bool
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// applyPresence sets what the watchman says of a machine: the verdict of its stuck presence key (nil when there is none).
|
||||
// Last heard is lastHeard's: now when online, the entry's time when offline, the store's last report only without an entry.
|
||||
func applyPresence(m *machineFacts, stuck *sdkdelivery.Stuck, entry *presenceEntry, now time.Time) {
|
||||
switch {
|
||||
case stuck == nil:
|
||||
m.presence = presenceUnsaid
|
||||
case stuck.Moved:
|
||||
m.presence = presenceHeard
|
||||
default:
|
||||
m.presence = presenceSilent
|
||||
}
|
||||
m.lastHeard = lastHeard(m.lastHeard, entry, now)
|
||||
}
|
||||
|
||||
// silent says the machine is silent: the watchman's verdict where it gave one, else the heartbeat rule. A machine never
|
||||
// heard from is not silent, it has not joined.
|
||||
func (m machineFacts) silent(now time.Time) bool {
|
||||
switch m.presence {
|
||||
case presenceSilent:
|
||||
return true
|
||||
case presenceHeard:
|
||||
return false
|
||||
}
|
||||
return !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > silentBound
|
||||
}
|
||||
|
||||
// stuckObservations are the conditions of what stopped moving: one for each key not moved, in the delivery scope, kind
|
||||
// stuck, naming the key and what it waited for. A node's presence key is left to the silent check; a moved key raises
|
||||
// nothing, so its condition clears.
|
||||
func stuckObservations(entries map[string]sdkdelivery.Stuck) []conditions.Observation {
|
||||
keys := make([]string, 0, len(entries))
|
||||
for k := range entries {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []conditions.Observation
|
||||
for _, k := range keys {
|
||||
s := entries[k]
|
||||
if s.Moved {
|
||||
continue
|
||||
}
|
||||
if s.Bucket == sdkdelivery.PresenceBucket {
|
||||
if node, _, _, err := sdkdelivery.ParsePresenceKey(s.Key); err == nil && node != "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
waited := s.Waited
|
||||
if waited == "" {
|
||||
waited = "a move"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: k, Kind: kindStuck,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", k, s.Since.UTC().Format("2006-01-02 15:04 MST"),
|
||||
waited),
|
||||
Said: fmt.Sprintf("%s: waiting for %s since %s", k, waited, s.Since.UTC().Format(time.RFC3339))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readStuck is every key of the stuck bucket and its value; none when the bucket is not there yet (the watchman not
|
||||
// delivered). A value of another version is left out, said.
|
||||
func readStuck(ctx context.Context, conn *nats.Conn) (map[string]sdkdelivery.Stuck, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watchman's stuck bucket: %w", err)
|
||||
}
|
||||
keys, err := boundedCall(ctx, "listing the stuck keys", func(ctx context.Context) ([]string, error) { return kv.Keys(ctx) })
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]sdkdelivery.Stuck{}
|
||||
for _, key := range keys {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, err := sdkdelivery.DecodeStuck(e.Value())
|
||||
if err != nil {
|
||||
fmt.Printf("the stuck key %s is not read: %v\n", key, err)
|
||||
continue
|
||||
}
|
||||
out[key] = s
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// readPresence is, by node, presence.<node>'s entry; a node without an entry is left out.
|
||||
func readPresence(ctx context.Context, conn *nats.Conn, nodes []string) (map[string]presenceEntry, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watchman's presence bucket: %w", err)
|
||||
}
|
||||
out := map[string]presenceEntry{}
|
||||
for _, node := range nodes {
|
||||
key, err := sdkdelivery.PresenceNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := sdkdelivery.DecodePresence(e.Value())
|
||||
if err != nil {
|
||||
fmt.Printf("the presence key %s is not read: %v\n", key, err)
|
||||
continue
|
||||
}
|
||||
out[node] = presenceEntry{Online: p.Online, At: e.Created()}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// withPresence says of each machine what the watchman says: its stuck presence key and its presence entry's time.
|
||||
func withPresence(ctx context.Context, conn *nats.Conn, machines []machineFacts, now time.Time) error {
|
||||
stuck, err := readStuck(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, len(machines))
|
||||
for i, m := range machines {
|
||||
names[i] = m.name
|
||||
}
|
||||
entries, err := readPresence(ctx, conn, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range machines {
|
||||
var said *sdkdelivery.Stuck
|
||||
if key, err := sdkdelivery.StuckKey(sdkdelivery.PresenceBucket, machines[i].name); err == nil {
|
||||
if s, ok := stuck[key]; ok {
|
||||
said = &s
|
||||
}
|
||||
}
|
||||
var entry *presenceEntry
|
||||
if e, ok := entries[machines[i].name]; ok {
|
||||
entry = &e
|
||||
}
|
||||
applyPresence(&machines[i], said, entry, now)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeStuck is DM: every key the watchman says stopped moving is raised, but a node's presence, which the silent check
|
||||
// takes.
|
||||
func probeStuck(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, nil
|
||||
}
|
||||
entries, err := readStuck(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stuckObservations(entries), nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1156/AC4: on the embedded bus with the watchman's buckets, a stuck key written raises a condition and written as moved
|
||||
// clears it; a node's presence key not moved says the machine silent through the facts, and moved says it heard.
|
||||
func TestAStuckKeyWrittenAndThenMovedRaisesAndClearsACondition(t *testing.T) { // 1156/AC4
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if _, err := broker.RaiseBuckets(js, []broker.Bucket{
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "stuck"},
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(bucket, key string, moved bool) string {
|
||||
t.Helper()
|
||||
stuckKey, err := sdkdelivery.StuckKey(bucket, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := sdkdelivery.EncodeStuck(sdkdelivery.Stuck{Bucket: bucket, Key: key, Waited: "an answer", Since: time.Now().UTC(), Moved: moved})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, stuckKey, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return stuckKey
|
||||
}
|
||||
d := &doctor{js: js}
|
||||
stuckKey := write(sdkdelivery.PulseBucket, "builds.x", false)
|
||||
write(sdkdelivery.PresenceBucket, "ace", false)
|
||||
got, err := probeStuck(ctx, d)
|
||||
if err != nil || len(got) != 1 || got[0].ID != stuckKey || got[0].Kind != kindStuck {
|
||||
t.Fatalf("a stuck key written: %+v %v", got, err)
|
||||
}
|
||||
machines := []machineFacts{{name: "ace", lastHeard: time.Now()}}
|
||||
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !machines[0].silent(time.Now()) {
|
||||
t.Fatal("a presence key not moved did not say the machine silent")
|
||||
}
|
||||
write(sdkdelivery.PulseBucket, "builds.x", true)
|
||||
write(sdkdelivery.PresenceBucket, "ace", true)
|
||||
if got, err := probeStuck(ctx, d); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a stuck key written as moved: %+v %v", got, err)
|
||||
}
|
||||
machines = []machineFacts{{name: "ace", lastHeard: time.Now().Add(-time.Hour)}}
|
||||
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if machines[0].silent(time.Now()) {
|
||||
t.Fatal("a presence key that moved still says the machine silent")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aStuckKey(t *testing.T, bucket, key string) string {
|
||||
t.Helper()
|
||||
k, err := sdkdelivery.StuckKey(bucket, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// 1156/AC1: each not-moved stuck key that is not a node's presence is an observation of the delivery scope, kind stuck,
|
||||
// naming the key and what it waited for; a moved key raises nothing, so its condition clears.
|
||||
func TestEachStuckKeyThatHasNotMovedIsAConditionOfTheDeliveryScope(t *testing.T) { // 1156/AC1
|
||||
since := time.Date(2026, 10, 18, 19, 0, 0, 0, time.UTC)
|
||||
pulse := aStuckKey(t, sdkdelivery.PulseBucket, "x")
|
||||
moved := aStuckKey(t, sdkdelivery.HaveBucket, "ace.zsh")
|
||||
node := aStuckKey(t, sdkdelivery.PresenceBucket, "ace")
|
||||
part := aStuckKey(t, sdkdelivery.PresenceBucket, "part.mesh-controller.one")
|
||||
entries := map[string]sdkdelivery.Stuck{
|
||||
pulse: {Bucket: sdkdelivery.PulseBucket, Key: "x", Waited: "the build of zsh to be answered", Since: since},
|
||||
moved: {Bucket: sdkdelivery.HaveBucket, Key: "ace.zsh", Waited: "a want", Since: since, Moved: true},
|
||||
node: {Bucket: sdkdelivery.PresenceBucket, Key: "ace", Waited: "its return", Since: since},
|
||||
part: {Bucket: sdkdelivery.PresenceBucket, Key: "part.mesh-controller.one", Waited: "its return", Since: since},
|
||||
}
|
||||
got := stuckObservations(entries)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("observations %+v, want the pulse key and the part's presence", got)
|
||||
}
|
||||
byID := map[string]conditions.Observation{}
|
||||
for _, o := range got {
|
||||
if o.Scope != conditions.ScopeDelivery || o.Kind != kindStuck {
|
||||
t.Errorf("%+v is not a stuck condition of the delivery scope", o)
|
||||
}
|
||||
byID[o.ID] = o
|
||||
}
|
||||
o, ok := byID[pulse]
|
||||
if !ok || !strings.Contains(o.Summary, pulse) || !strings.Contains(o.Summary, "the build of zsh to be answered") {
|
||||
t.Fatalf("the condition does not name the key and what it waited for: %+v", got)
|
||||
}
|
||||
if _, ok := byID[part]; !ok {
|
||||
t.Errorf("a part's presence is left out: %v", byID)
|
||||
}
|
||||
// Moved again: the same key, written as moved, raises nothing.
|
||||
entries[pulse] = sdkdelivery.Stuck{Bucket: sdkdelivery.PulseBucket, Key: "x", Since: since, Moved: true}
|
||||
if got := stuckObservations(entries); len(got) != 1 {
|
||||
t.Fatalf("a moved key still raises: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func aMachineHeardAt(now time.Time, ago time.Duration) machineFacts {
|
||||
return machineFacts{name: "laptop", lastHeard: now.Add(-ago)}
|
||||
}
|
||||
|
||||
// 1156/AC2: a node's stuck presence key not moved makes the machine silent, unless it sleeps; moved, it is heard; no
|
||||
// timer is armed (the verdict is the key's, whatever the time of the last word).
|
||||
func TestAStuckPresenceKeyIsTheSilentVerdict(t *testing.T) { // 1156/AC2
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
f.machines = append(f.machines, aMachineHeardAt(now, 10*time.Second))
|
||||
notMoved := sdkdelivery.Stuck{Bucket: sdkdelivery.PresenceBucket, Key: "laptop", Waited: "its return", Since: now.Add(-3 * time.Minute)}
|
||||
applyPresence(&f.machines[1], ¬Moved, nil, now)
|
||||
got := watchHeartbeats(f)
|
||||
if len(got) != 1 || got[0].Machine != "laptop" || got[0].Kind != "silent" {
|
||||
t.Fatalf("a presence key not moved: %+v", got)
|
||||
}
|
||||
// Asleep: not silent.
|
||||
f.machines[1].power = link.PowerState{State: "sleeping", At: now.Add(-time.Hour)}
|
||||
if got := watchHeartbeats(f); len(got) != 0 {
|
||||
t.Fatalf("a sleeping machine raised %+v", got)
|
||||
}
|
||||
// Moved: heard, whatever the store's last word says.
|
||||
f.machines[1].power = link.PowerState{}
|
||||
f.machines[1].lastHeard = now.Add(-2 * time.Hour)
|
||||
moved := notMoved
|
||||
moved.Moved = true
|
||||
applyPresence(&f.machines[1], &moved, nil, now)
|
||||
if got := watchHeartbeats(f); len(got) != 0 {
|
||||
t.Fatalf("a presence key that moved raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1156/AC3: with a presence entry that says offline, last heard is the entry's time (1183/AC1); with none,
|
||||
// last_seen. An entry that says online counts as heard now: the watchman writes it only when it changes, so its time is
|
||||
// when the machine connected, and a machine online for hours with no stuck key is not silent, whatever last_seen says.
|
||||
func TestLastHeardFollowsThePresenceEntry(t *testing.T) { // 1156/AC3
|
||||
now := time.Now()
|
||||
seen := now.Add(-time.Hour)
|
||||
m := machineFacts{name: "laptop", lastHeard: seen}
|
||||
applyPresence(&m, nil, nil, now)
|
||||
if !m.lastHeard.Equal(seen) {
|
||||
t.Fatalf("with no presence entry last heard is %v, want last_seen %v", m.lastHeard, seen)
|
||||
}
|
||||
offlineAt := now.Add(-5 * time.Minute)
|
||||
applyPresence(&m, nil, &presenceEntry{Online: false, At: offlineAt}, now)
|
||||
if !m.lastHeard.Equal(offlineAt) {
|
||||
t.Fatalf("offline since %v: last heard is %v", offlineAt, m.lastHeard)
|
||||
}
|
||||
m = machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
|
||||
applyPresence(&m, nil, &presenceEntry{Online: false, At: now.Add(-time.Hour)}, now)
|
||||
if !m.lastHeard.Equal(now.Add(-time.Hour)) {
|
||||
t.Fatalf("an offline entry's time is last heard, not a report: %v", m.lastHeard)
|
||||
}
|
||||
for _, lastSeen := range []time.Duration{10 * time.Second, 5 * time.Hour} {
|
||||
m := machineFacts{name: "laptop", lastHeard: now.Add(-lastSeen)}
|
||||
applyPresence(&m, nil, &presenceEntry{Online: true, At: now.Add(-6 * time.Hour)}, now)
|
||||
if m.silent(now) {
|
||||
t.Fatalf("a machine online since hours ago (last_seen %v ago) with no stuck key is silent", lastSeen)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -406,7 +406,12 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
|
||||
if len(added) == 0 || len(from) == 0 {
|
||||
return nil
|
||||
}
|
||||
source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}
|
||||
return askBuilds(ctx, inv, m, buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}, added)
|
||||
}
|
||||
|
||||
// askBuilds asks the first build of each directory of a merge's repository, as askNewModules does, from the source given.
|
||||
func askBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, source buildSource,
|
||||
added []string) []string {
|
||||
var asked []string
|
||||
for _, dir := range added {
|
||||
path := dir
|
||||
@@ -419,7 +424,7 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
|
||||
request := inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: m.Base, Commit: m.Commit, For: "merge"}
|
||||
if err != nil {
|
||||
request.ID = fmt.Sprintf("not-asked-%s-%s", short(m.Commit), strings.ReplaceAll(dir, "/", "-"))
|
||||
request.ID = notAskedID(m.Commit, dir)
|
||||
request.NotAsked = err.Error()
|
||||
recordBuildRequest(ctx, inv, request)
|
||||
fmt.Printf(" %s is a new module in %s/%s and could not be built: %v — a hand `build` of it "+
|
||||
@@ -434,6 +439,15 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
|
||||
return asked
|
||||
}
|
||||
|
||||
// notAskedID is the id a merge's build that could not be asked is kept under: a key the bus takes, so a module at the
|
||||
// repository's root (".") is kept as one in a folder is.
|
||||
func notAskedID(commit, dir string) string {
|
||||
if dir == "." || dir == "" {
|
||||
return "not-asked-" + short(commit)
|
||||
}
|
||||
return fmt.Sprintf("not-asked-%s-%s", short(commit), strings.ReplaceAll(strings.Trim(dir, "/"), "/", "-"))
|
||||
}
|
||||
|
||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||
var actingOnMerges sync.Mutex
|
||||
|
||||
|
||||
@@ -55,6 +55,9 @@ type signalFacts struct {
|
||||
waits []waitFacts
|
||||
// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
|
||||
batches []batchFacts
|
||||
// failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449).
|
||||
failed []failedPlanFacts
|
||||
failedErr error
|
||||
// refusedSends are the sends a machine refused within the hour for the generation they were composed
|
||||
// from, each naming its sender (S20, novox/hq issue 234).
|
||||
refusedSends []inventory.Send
|
||||
@@ -142,9 +145,7 @@ type machineFacts struct {
|
||||
control bool
|
||||
// lastHeard is the store's last word from it, any word; zero when never.
|
||||
lastHeard time.Time
|
||||
// every is the heartbeat interval it said; zero when it said none.
|
||||
every time.Duration
|
||||
power link.PowerState
|
||||
power link.PowerState
|
||||
// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
|
||||
sentAt time.Time
|
||||
reportedCurrent bool
|
||||
@@ -155,6 +156,8 @@ type machineFacts struct {
|
||||
tools bool
|
||||
toolsHeard time.Time
|
||||
toolsEvery time.Duration
|
||||
// presence is the watchman's word on whether it is online (novox/hq ADR 0433 §3); unsaid, the heartbeat rule judges.
|
||||
presence presenceVerdict
|
||||
}
|
||||
|
||||
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||
@@ -353,6 +356,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
if f.plansErr == nil {
|
||||
f.batches, f.plansErr = gatherBatches(ctx, inv, now)
|
||||
}
|
||||
f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now)
|
||||
f.loop, f.loopErr = w.gatherLoop()
|
||||
f.asks, f.asksErr = w.gatherAsks(ctx, inv)
|
||||
if open, err := w.keeper.Open(ctx); err != nil {
|
||||
@@ -455,9 +459,6 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
for _, n := range nodes {
|
||||
m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
|
||||
lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
|
||||
if beat, ok := link.HostBeats.Of(n.Name); ok {
|
||||
m.every = beat.Every
|
||||
}
|
||||
if beat, ok := link.ToolsBeats.Of(n.Name); ok {
|
||||
m.toolsHeard, m.toolsEvery = beat.At, beat.Every
|
||||
}
|
||||
@@ -470,7 +471,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
m.reportedCurrent = r.Current
|
||||
}
|
||||
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
if m.silent(now) {
|
||||
anyPast = true
|
||||
}
|
||||
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||
@@ -478,6 +479,19 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
// What the watchman says of each machine: its stuck presence key is the silent verdict, and presence.<node>'s time when
|
||||
// it was last heard (ADR 0433 §3). Unreadable is said, and the heartbeat rule judges as before.
|
||||
if w.js != nil {
|
||||
if err := withPresence(ctx, w.js.Conn(), out, now); err != nil {
|
||||
fmt.Printf("what the watchman says of the machines' presence cannot be read: %v\n", err)
|
||||
} else {
|
||||
for _, m := range out {
|
||||
if m.silent(now) {
|
||||
anyPast = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||
// which is the louder mistake and the right one to make.
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018164924-2b420603fe83
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018164924-2b420603fe83 h1:BrHyE3ifFMUQLIza6I/iNlEgdcysB9/Y+1FO6Q9xws8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018164924-2b420603fe83/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4 h1:yYDxFRJoH59kiCoEulD+FBwctyf9CrtKIxlEAsVL39Q=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9 h1:qHpN/fpTDqijfMVoepxUT6mPkfcydb4LiPKsIokv6iM=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -97,27 +97,17 @@ func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Seats []struct {
|
||||
Name string `json:"name"`
|
||||
Emits []string `json:"emits"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
emitter, consumer, declared, err := eventsOfManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", e.Name(), err)
|
||||
}
|
||||
if len(m.Emits) > 0 {
|
||||
emitters = append(emitters, AnEmitter{Module: m.Module, Emits: m.Emits})
|
||||
if emitter != nil {
|
||||
emitters = append(emitters, *emitter)
|
||||
}
|
||||
if len(m.Consumes) > 0 {
|
||||
consumers = append(consumers, AConsumer{Module: m.Module, Consumes: m.Consumes})
|
||||
}
|
||||
for _, s := range m.Seats {
|
||||
seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits})
|
||||
if consumer != nil {
|
||||
consumers = append(consumers, *consumer)
|
||||
}
|
||||
seats = append(seats, declared...)
|
||||
}
|
||||
if len(emitters) == 0 {
|
||||
t.Fatalf("no module under %s emits anything, so this proved nothing", root)
|
||||
@@ -245,3 +235,41 @@ func TestTheBrokerAndTheCatalogueAgreeOnTheRuntimeModule(t *testing.T) {
|
||||
t.Fatalf("the broker calls the runtime %q and the catalogue %q", RuntimeModule, catalogue.RuntimeModule)
|
||||
}
|
||||
}
|
||||
|
||||
// eventsOfManifest is what one manifest says it emits and consumes for the agreement: its `emits`, and the events of
|
||||
// its own streams, which a consumer names as <module>.<event> as it names the others (novox/hq issue 1249: once
|
||||
// mesh-mail declared emits beside its stream, the stream's events were no longer counted).
|
||||
func eventsOfManifest(raw []byte) (*AnEmitter, *AConsumer, []DeclaredSeat, error) {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Streams []struct {
|
||||
Events []string `json:"events"`
|
||||
} `json:"streams"`
|
||||
Seats []struct {
|
||||
Name string `json:"name"`
|
||||
Emits []string `json:"emits"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
emits := append([]string{}, m.Emits...)
|
||||
for _, s := range m.Streams {
|
||||
emits = append(emits, s.Events...)
|
||||
}
|
||||
var emitter *AnEmitter
|
||||
if len(emits) > 0 {
|
||||
emitter = &AnEmitter{Module: m.Module, Emits: emits}
|
||||
}
|
||||
var consumer *AConsumer
|
||||
if len(m.Consumes) > 0 {
|
||||
consumer = &AConsumer{Module: m.Module, Consumes: m.Consumes}
|
||||
}
|
||||
var seats []DeclaredSeat
|
||||
for _, s := range m.Seats {
|
||||
seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits})
|
||||
}
|
||||
return emitter, consumer, seats, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mailShaped is mesh-mail's manifest as mesh-catalog main (b767aac8) declares its events: emits beside two streams of
|
||||
// its own (modules/mesh-mail/module.json), trimmed to the keys the agreement reads.
|
||||
const mailShaped = `{"module": "mesh-mail",
|
||||
"emits": ["mail.send.sent", "mail.send.failed"],
|
||||
"streams": [
|
||||
{"name": "mail", "events": ["mail.*.message.added", "mail.*.message.changed", "mail.*.message.removed"]},
|
||||
{"name": "mail-sends", "events": ["mail.send.approved"]}]}`
|
||||
|
||||
func TestAModulesStreamEventsCountBesideItsEmits(t *testing.T) { // 1249/AC1
|
||||
emitter, _, _, err := eventsOfManifest([]byte(mailShaped))
|
||||
if err != nil || emitter == nil {
|
||||
t.Fatalf("emitter %v, err %v", emitter, err)
|
||||
}
|
||||
bad := Disagreements([]AnEmitter{*emitter},
|
||||
[]AConsumer{{Module: "mesh-mail-notifier", Consumes: []string{"mesh-mail.mail.*.message.added"}}}, nil)
|
||||
if len(bad) != 0 {
|
||||
t.Fatalf("a consumer of mesh-mail's stream event was reported: %v", bad)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEventNeitherEmittedNorOnTheStreamIsStillCaught(t *testing.T) { // 1249/AC2
|
||||
emitter, _, _, err := eventsOfManifest([]byte(mailShaped))
|
||||
if err != nil || emitter == nil {
|
||||
t.Fatalf("emitter %v, err %v", emitter, err)
|
||||
}
|
||||
bad := Disagreements([]AnEmitter{*emitter},
|
||||
[]AConsumer{{Module: "mesh-mail-notifier", Consumes: []string{"mesh-mail.mail.*.message.renamed"}}}, nil)
|
||||
if len(bad) != 1 || !strings.Contains(bad[0], "mail.*.message.added") {
|
||||
t.Fatalf("want one disagreement naming the stream's events among what mesh-mail emits, got %v", bad)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1155/AC1: a module whose manifest reads mesh-controller.checks-running may watch and get every key of the bucket and
|
||||
// write none.
|
||||
func TestAModuleReadingChecksRunningReadsAndWritesNone(t *testing.T) { // 1155/AC1
|
||||
if !slices.Contains(ControllerMirrors(), "checks-running") || BucketName(ControllerSeat, "checks-running") != CheckRunningBucket {
|
||||
t.Fatalf("a module cannot name %s in `reads`: %v", CheckRunningBucket, ControllerMirrors())
|
||||
}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-watchman", PasswordHash: "x",
|
||||
Reads: []string{"mesh-controller.checks-running"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + CheckRunningBucket
|
||||
has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
|
||||
has(t, perms.Publish, "$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
for _, p := range perms.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+CheckRunningBucket+".any.key") {
|
||||
t.Errorf("the reading module may write %s through %q", CheckRunningBucket, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1155/AC3: the writers table names checks-running's writer as the build seat's holders, and the rule that a mirror is
|
||||
// written by the controller alone names it as the exception, with its reason.
|
||||
func TestChecksRunningIsAMirrorWrittenByTheBuildSeatsHolders(t *testing.T) { // 1155/AC3
|
||||
why := MirrorsWrittenByAnother["checks-running"]
|
||||
if why == "" {
|
||||
t.Fatal("the mirrors written by another do not name checks-running with a reason")
|
||||
}
|
||||
subject := "$KV." + CheckRunningBucket + ".novox.a.7"
|
||||
var rows int
|
||||
for _, row := range WritersTable {
|
||||
for _, s := range row.Subjects {
|
||||
if s != "$KV."+CheckRunningBucket+".>" {
|
||||
continue
|
||||
}
|
||||
rows++
|
||||
holder := Principal{Kind: KindModule, Module: "builder", Node: "one", Holds: []Seat{{Name: TheBuildSeat}}}
|
||||
other := Principal{Kind: KindModule, Module: "mesh-delivery-watchman", Node: "one"}
|
||||
if row.Writes == nil || !row.Writes(holder, subject) || row.Writes(other, subject) {
|
||||
t.Errorf("%q is not written by the build seat's holders alone", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("%d rows of the writers table write %s, want one", rows, CheckRunningBucket)
|
||||
}
|
||||
for m := range MirrorsWrittenByAnother {
|
||||
if m != "checks-running" {
|
||||
t.Errorf("the mirror %q is written by another and is not checks-running", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package broker
|
||||
|
||||
import "testing"
|
||||
|
||||
// The operator-channel seat as the messenger declares it once it takes a module's condition: copied from
|
||||
// mesh-catalog modules/messenger/module.json at 5b235a6e (issue 1217), the seat's accepts, emits and by-caller verbs.
|
||||
func operatorChannelWithCondition() Seat {
|
||||
return Seat{Name: "operator-channel", Scope: "mesh", Accepts: []string{"ask", "cancel", "condition"},
|
||||
Emits: []string{"decided"}, Serves: []string{"open", "history", "notify", "tell", "asks"},
|
||||
ByCaller: []string{"ask", "cancel", "condition", "decided"}, Records: []string{"asks"}}
|
||||
}
|
||||
|
||||
// 1216/AC1: a module that uses the seat may publish its condition under its own name.
|
||||
func TestAModuleUsingTheOperatorChannelMayPublishItsConditionUnderItsOwnName(t *testing.T) {
|
||||
x := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "web-app", Uses: []Seat{operatorChannelWithCondition()}})
|
||||
if !allowed(x.Publish, "mesh.seat.operator-channel.accept.condition.web-app") {
|
||||
t.Errorf("web-app may not say a condition as itself: %v", x.Publish)
|
||||
}
|
||||
}
|
||||
|
||||
// 1216/AC2: it may not publish a condition under another module's name, nor on the wildcard.
|
||||
func TestAModuleMayNotSayAConditionUnderAnotherName(t *testing.T) {
|
||||
x := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "web-app", Uses: []Seat{operatorChannelWithCondition()}})
|
||||
for _, subject := range []string{
|
||||
"mesh.seat.operator-channel.accept.condition.other-app",
|
||||
"mesh.seat.operator-channel.accept.condition.*",
|
||||
"mesh.seat.operator-channel.accept.condition.>",
|
||||
"mesh.seat.operator-channel.accept.condition",
|
||||
} {
|
||||
if allowed(x.Publish, subject) {
|
||||
t.Errorf("web-app may publish %s", subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1216/AC3: the seat's holder may take accept.condition.<any module> from the work queue SEAT_OPERATOR_CHANNEL.
|
||||
func TestTheHolderOfTheOperatorChannelTakesEveryModulesCondition(t *testing.T) {
|
||||
seat := operatorChannelWithCondition()
|
||||
holder := perms(t, Principal{Kind: KindModule, Node: "anchor", Module: "messenger", Holds: []Seat{seat}})
|
||||
if !allowed(holder.Subscribe, "mesh.seat.operator-channel.accept.condition.web-app") ||
|
||||
!allowed(holder.Subscribe, "mesh.seat.operator-channel.accept.condition.other-app") {
|
||||
t.Errorf("the holder does not take accept.condition.*: %v", holder.Subscribe)
|
||||
}
|
||||
if !allowed(holder.Publish, "$JS.API.CONSUMER.MSG.NEXT.SEAT_OPERATOR_CHANNEL.SEAT_OPERATOR_CHANNEL_worker") {
|
||||
t.Errorf("the holder may not pull the seat's work queue: %v", holder.Publish)
|
||||
}
|
||||
users, _ := Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": {
|
||||
{Module: "messenger", Holds: []Seat{seat}},
|
||||
{Module: "web-app", Uses: []Seat{seat}},
|
||||
}}})
|
||||
streams, workers := SeatTrafficObjects(users)
|
||||
kept := false
|
||||
for _, s := range streams {
|
||||
if s.Name == "SEAT_OPERATOR_CHANNEL" {
|
||||
for _, subject := range s.Subjects {
|
||||
kept = kept || subjectMatches(subject, "mesh.seat.operator-channel.accept.condition.web-app")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !kept {
|
||||
t.Error("the work queue SEAT_OPERATOR_CHANNEL does not keep accept.condition.<module>")
|
||||
}
|
||||
taken := false
|
||||
for _, w := range workers {
|
||||
if w.Name == "SEAT_OPERATOR_CHANNEL_worker" {
|
||||
for _, f := range w.Filters {
|
||||
taken = taken || subjectMatches(f, "mesh.seat.operator-channel.accept.condition.web-app")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !taken {
|
||||
t.Error("the seat's worker does not take accept.condition.<module>")
|
||||
}
|
||||
}
|
||||
@@ -88,3 +88,133 @@ func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
|
||||
t.Fatal("a work queue's consumer was reset")
|
||||
}
|
||||
}
|
||||
|
||||
// pendingOf polls a consumer's pending count until it is want or a second has passed, and returns what it saw last.
|
||||
func pendingOf(js *JetStream, stream, name string, want uint64) uint64 {
|
||||
var got uint64
|
||||
for deadline := time.Now().Add(time.Second); ; time.Sleep(20 * time.Millisecond) {
|
||||
if info, err := js.js.ConsumerInfo(stream, name); err == nil {
|
||||
got = info.NumPending
|
||||
}
|
||||
if got == want || time.Now().After(deadline) {
|
||||
return got
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1279/AC2 (ADR 0443 §1): on a bus whose EVENTS holds earlier events, the consumer the controller makes for a module
|
||||
// delivers none of them and delivers one published after its making; the same on another module's stream.
|
||||
func TestAModulesConsumerDeliversNothingOfTheStreamsPast(t *testing.T) { // 1279/AC2
|
||||
js := aLiveBus(t)
|
||||
if err := AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.Context().PurgeStream(EventsStream); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const subject = "mesh.mod.shop.event.order.placed"
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := js.Context().Publish(subject, []byte(fmt.Sprint(i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit", Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||
if !ok {
|
||||
t.Fatal("no consumer derived")
|
||||
}
|
||||
t.Cleanup(func() { _ = js.Context().DeleteConsumer(c.Stream, c.Name) })
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := pendingOf(js, c.Stream, c.Name, 0); got != 0 {
|
||||
t.Fatalf("a module's consumer made after 3 events holds %d of them", got)
|
||||
}
|
||||
if _, err := js.Context().Publish(subject, []byte("after")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := pendingOf(js, c.Stream, c.Name, 1); got != 1 {
|
||||
t.Fatalf("the event published after its making is not pending: %d", got)
|
||||
}
|
||||
|
||||
// Another module's stream.
|
||||
s := mailStream.Stream()
|
||||
if err := js.EnsureStream(s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.js.DeleteStream(s.Name) })
|
||||
for _, m := range []string{"added", "changed"} {
|
||||
if _, err := js.js.Publish(ModuleStreamSubject("mesh-mail", "mail.mesh.message."+m), []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
reader := Principal{Kind: KindModule, Node: "ace", Module: "reader", Hears: []Heard{mailHeard}}
|
||||
h := heardConsumer(reader, mailHeard)
|
||||
t.Cleanup(func() { _ = js.js.DeleteConsumer(h.Stream, h.Name) })
|
||||
if err := js.EnsureConsumer(h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := pendingOf(js, h.Stream, h.Name, 0); got != 0 {
|
||||
t.Fatalf("a module's consumer on another module's stream holds %d of its past", got)
|
||||
}
|
||||
if _, err := js.js.Publish(ModuleStreamSubject("mesh-mail", "mail.mesh.message.added"), []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := pendingOf(js, h.Stream, h.Name, 1); got != 1 {
|
||||
t.Fatalf("the event published after its making is not pending on the module's stream: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1279/AC3 (ADR 0443 §2): a consumer that exists, made from the start, asserted again as the controller now derives it,
|
||||
// keeps its deliver policy and delivers its next message from where it was.
|
||||
func TestAnExistingConsumerKeepsItsPositionWhenAssertedAgain(t *testing.T) { // 1279/AC3
|
||||
js := aLiveBus(t)
|
||||
if err := AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.Context().PurgeStream(EventsStream); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const subject = "mesh.mod.shop.event.order.placed"
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := js.Context().Publish(subject, []byte(fmt.Sprint(i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
c, _ := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit", Consumes: []string{"shop.order.placed"}, PasswordHash: "x"})
|
||||
t.Cleanup(func() { _ = js.Context().DeleteConsumer(c.Stream, c.Name) })
|
||||
old := c
|
||||
old.FromNow = false // as the controller made it before ADR 0443
|
||||
if err := js.EnsureConsumer(old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func() string {
|
||||
t.Helper()
|
||||
sub, err := js.Context().PullSubscribe("", c.Name, nats.Bind(c.Stream, c.Name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
|
||||
if err != nil || len(msgs) != 1 {
|
||||
t.Fatalf("fetch: %v (%d messages)", err, len(msgs))
|
||||
}
|
||||
_ = msgs[0].Ack()
|
||||
return string(msgs[0].Data)
|
||||
}
|
||||
if got := next(); got != "0" {
|
||||
t.Fatalf("the first message is %q", got)
|
||||
}
|
||||
if err := js.EnsureConsumer(c); err != nil { // now derived from now
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().ConsumerInfo(c.Stream, c.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Config.DeliverPolicy != nats.DeliverAllPolicy {
|
||||
t.Fatalf("asserting moved the deliver policy to %v", info.Config.DeliverPolicy)
|
||||
}
|
||||
if got := next(); got != "1" {
|
||||
t.Fatalf("after asserting again the next message is %q, want 1: the position moved", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@ var (
|
||||
// check ends; a key not renewed within the bucket's age is a builder that died, and is gone. A builder that takes a
|
||||
// waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by
|
||||
// design** — each the key of the check it runs.
|
||||
CheckRunningBucket = BucketName(ControllerSeat, "checks-running")
|
||||
CheckRunningBucket = BucketName(ControllerSeat, ChecksRunning)
|
||||
// DeliveryMergesBucket holds the merges mesh-delivery announced it made, each with the ask whose Approve it merged
|
||||
// on (novox/hq ADR 0420 §7, issue 1025): per merge a key `<owner>/<repo>/<number>/<head sha>` holding the
|
||||
// announcement, kept DeliveryMergesKeptFor. What lets the build of a moved module register it from its new
|
||||
@@ -133,9 +133,22 @@ func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, Ass
|
||||
// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the
|
||||
// chain's buckets.
|
||||
func ControllerMirrors() []string {
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts}
|
||||
return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning}
|
||||
}
|
||||
|
||||
// MirrorsWrittenByAnother are the controller's mirrors whose writer is not the controller, each with its reason: the rule
|
||||
// that a mirror is written by the controller alone (TestTheMirrorsListAndTheWritersTableAgree) has these exceptions and no
|
||||
// other.
|
||||
var MirrorsWrittenByAnother = map[string]string{
|
||||
ChecksRunning: "the checks running are said by the builders themselves: every machine holding the build seat puts the " +
|
||||
"key of the check it runs, at its start and at each heartbeat, and deletes it when the check ends, so the " +
|
||||
"controller asserts the bucket and writes none of its keys (ADR 0406 §3); it is readable through `reads` so " +
|
||||
"the watchman sees a check that stopped (ADR 0433 §6)",
|
||||
}
|
||||
|
||||
// ChecksRunning is the local name of the bucket of the checks running, as a reader names it in `reads`.
|
||||
const ChecksRunning = "checks-running"
|
||||
|
||||
// The mirrors' caps (design 54 §5): one value per key, no age, 64 KiB a value, 64 MiB a bucket.
|
||||
const (
|
||||
MirrorValueBytes = 64 << 10
|
||||
|
||||
@@ -135,6 +135,12 @@ func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) {
|
||||
for _, m := range ControllerMirrors() {
|
||||
bucket := BucketName(ControllerSeat, m)
|
||||
listed[bucket] = true
|
||||
if why, other := MirrorsWrittenByAnother[m]; other { // 1155/AC3: the exception, with its reason
|
||||
if why == "" {
|
||||
t.Errorf("the mirror %s is written by another and says no reason", bucket)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !rowed[bucket] {
|
||||
t.Errorf("the mirror %s is listed and no row of the writers table writes it", bucket)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
package broker
|
||||
|
||||
import "testing"
|
||||
|
||||
// The delivery's gate, composed as novox/hq ADR 0437 §3 declares it and ADR 0450 §1 ships it in release 3's step R3d
|
||||
// (issue 1326). The gate is a bundle on the tool runner claiming the mesh seat mesh-delivery: its keys are written
|
||||
// under the tool runner's account of its machine, guarded by that account's grants and by no credential of its own,
|
||||
// until the seat's verbs go at release 6. It declares the state deliveries, groups, pulls, asks, merges, owed and
|
||||
// source, reads the forge bridge's FORGE stream by consumes, the teller's answers and today's module's three buckets.
|
||||
// Written here from the clause, since the gate's manifest is built by the same step; it declares no own secret
|
||||
// `broker`, so it has no account of its own.
|
||||
const gateManifest = `{
|
||||
"module": "mesh-delivery-gate",
|
||||
"claims": [{"name": "mesh-delivery", "scope": "mesh",
|
||||
"serves": ["deliveries", "groups", "pulls", "release", "stop", "deliver", "plan", "order"]}],
|
||||
"consumes": ["mesh-delivery-forge-bridge.forge.**"],
|
||||
"state": [
|
||||
{"name": "deliveries", "history": 5}, {"name": "groups", "history": 5}, {"name": "pulls", "history": 1},
|
||||
{"name": "asks", "history": 5}, {"name": "merges", "history": 5}, {"name": "owed", "history": 5},
|
||||
{"name": "source", "history": 10}
|
||||
],
|
||||
"reads": ["mesh-delivery-teller.answers", "mesh-delivery.deliveries", "mesh-delivery.groups", "mesh-delivery.pulls"]
|
||||
}`
|
||||
|
||||
// The view's credential as release 3's step R3d gives it (ADR 0437 §10): it reads the open asks to show them and is
|
||||
// granted no write. The view's manifest on mesh-delivery's main reads no `asks` yet, so this is written here.
|
||||
const viewReadingAsksManifest = `{
|
||||
"module": "mesh-delivery-view",
|
||||
"reads": ["mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules"],
|
||||
"own-secrets": {"broker": "${dir:state}/broker"}
|
||||
}`
|
||||
|
||||
// the gate's seven buckets, on the bus.
|
||||
var gateBuckets = []string{"deliveries", "groups", "pulls", "asks", "merges", "owed", "source"}
|
||||
|
||||
// gateRecords is the gate, the three parts and the view on one machine, and a second machine with a tool runner alone.
|
||||
func gateRecords(t *testing.T) Records {
|
||||
t.Helper()
|
||||
streams := map[string]ModuleStream{"mesh-delivery-forge-bridge": {Module: "mesh-delivery-forge-bridge", Name: "FORGE"}}
|
||||
return Records{Nodes: []string{"novox", "ace"}, Assigned: map[string][]Declared{
|
||||
"novox": {{Module: RuntimeModule}, declaredOf(t, gateManifest, streams),
|
||||
declaredOf(t, bridgeManifest, streams), declaredOf(t, indexerManifest, streams),
|
||||
declaredOf(t, tellerManifest, streams), declaredOf(t, viewReadingAsksManifest, streams)},
|
||||
"ace": {{Module: RuntimeModule}},
|
||||
}}
|
||||
}
|
||||
|
||||
// 1326/AC1: the grants composed for the gate on one machine let that machine's tool runner account write each of its
|
||||
// seven buckets, read the buckets it declares reading (the teller's answers without writing them), hear the bridge's
|
||||
// FORGE stream and serve the seat's verbs.
|
||||
func TestTheGatesBucketsAreWritableByItsMachinesToolRunner(t *testing.T) {
|
||||
all := composed(t, gateRecords(t))
|
||||
runner, ok := all["runner@novox"]
|
||||
if !ok {
|
||||
t.Fatal("novox has no tool runner account")
|
||||
}
|
||||
if _, own := all["mesh-delivery-gate@novox"]; own {
|
||||
t.Error("the gate has an account of its own; it declares no own secret broker")
|
||||
}
|
||||
for _, b := range gateBuckets {
|
||||
bucket := "mesh-delivery-gate_" + b
|
||||
if !canWrite(runner, bucket, "some.key") || !canRead(runner, bucket, "some.key") {
|
||||
t.Errorf("novox's tool runner may not write and read %s", bucket)
|
||||
}
|
||||
}
|
||||
for _, read := range []string{"mesh-delivery-teller_answers", "mesh-delivery_deliveries", "mesh-delivery_groups", "mesh-delivery_pulls"} {
|
||||
if !canRead(runner, read, "k") {
|
||||
t.Errorf("novox's tool runner may not read %s, which the gate declares reading", read)
|
||||
}
|
||||
// 1329/AC3: the runner no longer carries the teller, so it may not write the teller's answers either.
|
||||
if canWrite(runner, read, "k") {
|
||||
t.Errorf("novox's tool runner may write %s, which the gate only reads", read)
|
||||
}
|
||||
}
|
||||
if !allowed(runner.Publish, "$JS.API.CONSUMER.MSG.NEXT.FORGE.novox_mesh-delivery-gate") {
|
||||
t.Errorf("the gate's FORGE consumer is not pulled by novox's tool runner")
|
||||
}
|
||||
if !allowed(runner.Subscribe, "mesh.seat.mesh-delivery.tool.deliveries") {
|
||||
t.Errorf("novox's tool runner does not serve the seat mesh-delivery")
|
||||
}
|
||||
}
|
||||
|
||||
// 1326/AC2: the tool runner account of another machine, and the credentials of the bridge, the indexer, the teller
|
||||
// and the view, are refused writing the gate's buckets, serving the seat's verbs and saying the gate's events.
|
||||
func TestNoOtherAccountWritesTheGatesBuckets(t *testing.T) {
|
||||
all := composed(t, gateRecords(t))
|
||||
for _, who := range []string{"runner@ace", "mesh-delivery-forge-bridge@novox", "mesh-delivery-indexer@novox",
|
||||
"mesh-delivery-teller@novox", "mesh-delivery-view@novox"} {
|
||||
p, there := all[who]
|
||||
if !there {
|
||||
t.Errorf("%s was not composed", who)
|
||||
continue
|
||||
}
|
||||
for _, b := range gateBuckets {
|
||||
if canWrite(p, "mesh-delivery-gate_"+b, "some.key") {
|
||||
t.Errorf("%s may write mesh-delivery-gate_%s", who, b)
|
||||
}
|
||||
}
|
||||
if allowed(p.Subscribe, "mesh.seat.mesh-delivery.tool.deliveries") {
|
||||
t.Errorf("%s serves the seat mesh-delivery's verbs", who)
|
||||
}
|
||||
if allowed(p.Publish, "mesh.mod.mesh-delivery-gate.event.merged") {
|
||||
t.Errorf("%s may say an event as the gate", who)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1326/AC3: each part reads the gate's buckets its manifest declares, and only those: the bridge merges and owed, the
|
||||
// indexer source, the teller and the view asks. The tool runner of another machine reads none.
|
||||
func TestEachPartReadsOnlyTheGateBucketsItDeclares(t *testing.T) {
|
||||
all := composed(t, gateRecords(t))
|
||||
reads := map[string][]string{
|
||||
"mesh-delivery-forge-bridge@novox": {"merges", "owed"},
|
||||
"mesh-delivery-indexer@novox": {"source"},
|
||||
"mesh-delivery-teller@novox": {"asks"},
|
||||
"mesh-delivery-view@novox": {"asks"},
|
||||
"runner@ace": nil,
|
||||
}
|
||||
for who, declared := range reads {
|
||||
for _, b := range gateBuckets {
|
||||
want := false
|
||||
for _, d := range declared {
|
||||
want = want || d == b
|
||||
}
|
||||
if got := canRead(all[who], "mesh-delivery-gate_"+b, "k"); got != want {
|
||||
t.Errorf("%s reads mesh-delivery-gate_%s: %v, declared %v", who, b, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,693 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The delivery's forge bridge, indexer and teller are processes on their module's own broker credential, granted from
|
||||
// the manifest alone (novox/hq ADR 0437 §4, issue 1243): `state` grants writing and watching the module's own buckets
|
||||
// and streams, `reads` watching another module's bucket and never writing it, `consumes` hearing the named events,
|
||||
// `invokes` calling the named tools, `uses` the seat operator-channel's by-caller subjects under the module's own name.
|
||||
//
|
||||
// The four manifests (the watchman's is the fourth, issue 1329) are copied from mesh-delivery's main at 9c15112d935a
|
||||
// (modules/<module>/module.json, read 2026-10-20; the first three are unchanged since 6a23627426c6), whole, so a change
|
||||
// to one of them there that changes its grant is a change to read here.
|
||||
const bridgeManifest = `{
|
||||
"module": "mesh-delivery-forge-bridge",
|
||||
"version": "1",
|
||||
"slug": "forge-bridge",
|
||||
"consumes": [
|
||||
"gitea.pull.updated",
|
||||
"gitea.pull.merged",
|
||||
"gitea.pull.closed"
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "forge", "events": ["forge.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "heads", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.merges",
|
||||
"mesh-delivery-gate.owed"
|
||||
],
|
||||
"invokes": [
|
||||
"gitea.gitea_merge_pull_request",
|
||||
"gitea.gitea_commit_status",
|
||||
"gitea.gitea_delivery_view",
|
||||
"gitea.gitea_note_append",
|
||||
"gitea.gitea_add_comment",
|
||||
"gitea.gitea_open_pulls",
|
||||
"gitea.gitea_commit_trees"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-forge-bridge",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-forge-bridge",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-forge-bridge"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-forge-bridge",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "bridge",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-forge-bridge",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-forge-bridge"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
},
|
||||
"user": "mesh-delivery-forge-bridge",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-forge-bridge",
|
||||
"binary": "mesh-delivery-forge-bridge"
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
const indexerManifest = `{
|
||||
"module": "mesh-delivery-indexer",
|
||||
"version": "1",
|
||||
"slug": "indexer",
|
||||
"consumes": [
|
||||
"mesh-delivery-forge-bridge.forge.**"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "candidates", "history": 10 },
|
||||
{ "name": "modules", "history": 20 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.builds",
|
||||
"mesh-delivery-gate.source"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-indexer",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-indexer",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-indexer"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-indexer",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "indexer",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-indexer",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-indexer"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
},
|
||||
"user": "mesh-delivery-indexer",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-indexer",
|
||||
"binary": "mesh-delivery-indexer"
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
const watchmanManifest = `{
|
||||
"module": "mesh-delivery-watchman",
|
||||
"version": "1",
|
||||
"slug": "watchman",
|
||||
"reads": [
|
||||
"mesh-controller.checks-running"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "presence", "history": 20 },
|
||||
{ "name": "pulse", "key-ttl": true },
|
||||
{ "name": "stuck", "history": 20 }
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-watchman",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-watchman",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-watchman"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-watchman",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "watchman",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-watchman",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-watchman"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
},
|
||||
"user": "mesh-delivery-watchman",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-watchman",
|
||||
"binary": "mesh-delivery-watchman"
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
const tellerManifest = `{
|
||||
"module": "mesh-delivery-teller",
|
||||
"version": "1",
|
||||
"slug": "teller",
|
||||
"state": [
|
||||
{ "name": "answers", "history": 5 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.asks"
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-teller",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-teller",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-teller"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-teller",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "teller",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-teller",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-teller"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
},
|
||||
"user": "mesh-delivery-teller",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-teller",
|
||||
"binary": "mesh-delivery-teller"
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
// manifestOf is the part of a module's manifest that composes a bus account, as the controller reads it.
|
||||
type manifestOf struct {
|
||||
Module string `json:"module"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Streams []struct {
|
||||
Name string `json:"name"`
|
||||
Events []string `json:"events"`
|
||||
} `json:"streams"`
|
||||
State []struct {
|
||||
Name string `json:"name"`
|
||||
History int `json:"history"`
|
||||
} `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Uses []string `json:"uses"`
|
||||
OwnSecrets map[string]json.RawMessage `json:"own-secrets"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
} `json:"claims"`
|
||||
}
|
||||
|
||||
// declaredOf is what the controller declares of a manifest to the broker (inventory's busrecords): its state as
|
||||
// buckets, its streams under the module's name in the bus's spelling, its claims as held seats, the module streams
|
||||
// its consumes name as heard, and no account when it declares no own secret `broker`. streamsOf are the module
|
||||
// streams the other manifests of the test own, by module.
|
||||
func declaredOf(t *testing.T, raw string, streamsOf map[string]ModuleStream) Declared {
|
||||
t.Helper()
|
||||
var m manifestOf
|
||||
if err := json.Unmarshal([]byte(raw), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := Declared{Module: m.Module, Consumes: m.Consumes, Reads: m.Reads, Invokes: m.Invokes}
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
d.NoAccount = true
|
||||
}
|
||||
for _, s := range m.State {
|
||||
d.State = append(d.State, Bucket{Module: m.Module, Name: s.Name, History: s.History})
|
||||
}
|
||||
for _, s := range m.Streams {
|
||||
var events []string
|
||||
for _, e := range s.Events {
|
||||
events = append(events, WirePattern(e))
|
||||
}
|
||||
d.Streams = append(d.Streams, ModuleStream{Module: m.Module, Name: strings.ToUpper(s.Name), Events: events,
|
||||
MaxAgeDays: 7, MaxMiB: 64})
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
d.Holds = append(d.Holds, Seat{Name: c.Name, Scope: c.Scope, Serves: c.Serves})
|
||||
}
|
||||
for _, name := range m.Uses {
|
||||
if name == "operator-channel" {
|
||||
d.Uses = append(d.Uses, operatorChannelWithCondition())
|
||||
}
|
||||
}
|
||||
for _, c := range m.Consumes {
|
||||
emitter, event, _ := strings.Cut(c, ".")
|
||||
if s, owned := streamsOf[emitter]; owned && strings.HasPrefix(event, strings.ToLower(s.Name)+".") {
|
||||
d.Hears = append(d.Hears, Heard{Stream: s.Name, Emitter: emitter,
|
||||
Filters: []string{ModuleStreamSubject(emitter, WirePattern(event))}})
|
||||
d.HearsAStream = true
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// canWrite says a principal may publish to a bucket's key; canRead says it may get one directly.
|
||||
func canWrite(p Permissions, bucket, key string) bool {
|
||||
return allowed(p.Publish, "$KV."+bucket+"."+key)
|
||||
}
|
||||
func canRead(p Permissions, bucket, key string) bool {
|
||||
return allowed(p.Publish, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+key)
|
||||
}
|
||||
|
||||
// composed is every module's and every machine's tool runner's grants the broker composes for these records,
|
||||
// as `<module>@<node>` and `runner@<node>`.
|
||||
func composed(t *testing.T, r Records) map[string]Permissions {
|
||||
t.Helper()
|
||||
users, err := Users(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]Permissions{}
|
||||
for _, u := range users {
|
||||
switch u.Kind {
|
||||
case KindNodeTools:
|
||||
out["runner@"+u.Node] = perms(t, u)
|
||||
case KindModule:
|
||||
out[u.Module+"@"+u.Node] = perms(t, u)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withPrefix is the patterns that start with any of the prefixes, sorted.
|
||||
func withPrefix(patterns []string, prefixes ...string) []string {
|
||||
var out []string
|
||||
for _, p := range patterns {
|
||||
for _, prefix := range prefixes {
|
||||
if strings.HasPrefix(p, prefix) {
|
||||
out = append(out, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// eventsSaid is the events a principal may publish, under a module's name or on a module's stream.
|
||||
func eventsSaid(p Permissions) []string {
|
||||
var out []string
|
||||
for _, s := range withPrefix(p.Publish, "mesh.mod.", "mesh.stream.") {
|
||||
if strings.Contains(s, ".event.") {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// toolsCalled is the tools of one module a principal may call, by name.
|
||||
func toolsCalled(p Permissions, module string) []string {
|
||||
var out []string
|
||||
for _, s := range withPrefix(p.Publish, "mesh.mod."+module+".tool.") {
|
||||
if name := strings.TrimPrefix(s, "mesh.mod."+module+".tool."); !strings.Contains(name, ".") {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// partsRecords is the three parts on one machine, beside the machine's tool runner.
|
||||
func partsRecords(t *testing.T) Records {
|
||||
t.Helper()
|
||||
streams := map[string]ModuleStream{"mesh-delivery-forge-bridge": {Module: "mesh-delivery-forge-bridge", Name: "FORGE"}}
|
||||
return Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {
|
||||
{Module: RuntimeModule},
|
||||
declaredOf(t, bridgeManifest, streams), declaredOf(t, indexerManifest, streams), declaredOf(t, tellerManifest, streams),
|
||||
}}}
|
||||
}
|
||||
|
||||
// 1243/AC1: each part's credential, composed from its manifest, may do exactly what its declaration names: it writes
|
||||
// its own buckets and stream, reads the buckets it reads (and cannot write them), hears what it consumes, calls what it
|
||||
// invokes, and says what the seat it uses lets it say under its own name.
|
||||
func TestEachPartsCredentialGrantsExactlyItsDeclaration(t *testing.T) {
|
||||
all := composed(t, partsRecords(t))
|
||||
bridge, indexer, teller := all["mesh-delivery-forge-bridge@novox"], all["mesh-delivery-indexer@novox"], all["mesh-delivery-teller@novox"]
|
||||
|
||||
// The forge bridge: state heads, the stream FORGE, reads merges and owed, invokes seven tools of the forge.
|
||||
if got, want := withPrefix(bridge.Publish, "$KV."), []string{"$KV.mesh-delivery-forge-bridge_heads.>"}; !slices.Equal(got, want) {
|
||||
t.Errorf("the bridge writes %v; want exactly %v", got, want)
|
||||
}
|
||||
if got, want := eventsSaid(bridge), []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}; !slices.Equal(got, want) {
|
||||
t.Errorf("the bridge says %v; want exactly %v", got, want)
|
||||
}
|
||||
for _, b := range []string{"merges", "owed"} {
|
||||
if !canRead(bridge, "mesh-delivery-gate_"+b, "g.1") || canWrite(bridge, "mesh-delivery-gate_"+b, "g.1") {
|
||||
t.Errorf("the bridge must read and not write the gate's %s", b)
|
||||
}
|
||||
}
|
||||
for _, event := range []string{"pull.updated", "pull.merged", "pull.closed"} {
|
||||
if !allowed(bridge.Subscribe, "mesh.mod.gitea.event."+event) {
|
||||
t.Errorf("the bridge does not hear gitea's %s", event)
|
||||
}
|
||||
}
|
||||
if allowed(bridge.Subscribe, "mesh.mod.gitea.event.pull.opened") {
|
||||
t.Error("the bridge hears an event of gitea's it does not consume")
|
||||
}
|
||||
called := []string{"gitea_add_comment", "gitea_commit_status", "gitea_commit_trees", "gitea_delivery_view",
|
||||
"gitea_merge_pull_request", "gitea_note_append", "gitea_open_pulls"}
|
||||
if got := toolsCalled(bridge, "gitea"); !slices.Equal(got, called) {
|
||||
t.Errorf("the bridge calls %v; want exactly %v", got, called)
|
||||
}
|
||||
if allowed(bridge.Publish, "mesh.mod.gitea.tool.gitea_delete_repo") {
|
||||
t.Error("the bridge may call a tool of gitea's it does not invoke")
|
||||
}
|
||||
|
||||
// The indexer: state candidates and modules, reads the controller's builds and the gate's source, hears FORGE.
|
||||
if got, want := withPrefix(indexer.Publish, "$KV."), []string{"$KV.mesh-delivery-indexer_candidates.>", "$KV.mesh-delivery-indexer_modules.>"}; !slices.Equal(got, want) {
|
||||
t.Errorf("the indexer writes %v; want exactly %v", got, want)
|
||||
}
|
||||
if got := eventsSaid(indexer); len(got) != 0 {
|
||||
t.Errorf("the indexer says %v; it declares no event", got)
|
||||
}
|
||||
for _, read := range [][2]string{{"mesh-controller_builds", "b.1"}, {"mesh-delivery-gate_source", "main"}} {
|
||||
if !canRead(indexer, read[0], read[1]) || canWrite(indexer, read[0], read[1]) {
|
||||
t.Errorf("the indexer must read and not write %s", read[0])
|
||||
}
|
||||
}
|
||||
if !allowed(indexer.Subscribe, "mesh.mod.mesh-delivery-forge-bridge.event.forge.merged") ||
|
||||
!allowed(indexer.Publish, "$JS.API.CONSUMER.MSG.NEXT.FORGE.novox_mesh-delivery-indexer") {
|
||||
t.Errorf("the indexer does not hear FORGE through its own consumer: %v / %v", indexer.Subscribe, indexer.Publish)
|
||||
}
|
||||
if got := withPrefix(indexer.Publish, "mesh.seat."); len(got) != 0 {
|
||||
t.Errorf("the indexer reaches the seats %v; it uses none", got)
|
||||
}
|
||||
|
||||
// The teller: state answers, reads the gate's asks, uses operator-channel under its own name.
|
||||
if got, want := withPrefix(teller.Publish, "$KV."), []string{"$KV.mesh-delivery-teller_answers.>"}; !slices.Equal(got, want) {
|
||||
t.Errorf("the teller writes %v; want exactly %v", got, want)
|
||||
}
|
||||
if !canRead(teller, "mesh-delivery-gate_asks", "merge.g.h") || canWrite(teller, "mesh-delivery-gate_asks", "merge.g.h") {
|
||||
t.Error("the teller must read and not write the gate's asks")
|
||||
}
|
||||
for _, verb := range []string{"ask", "cancel", "condition"} {
|
||||
if !allowed(teller.Publish, "mesh.seat.operator-channel.accept."+verb+".mesh-delivery-teller") {
|
||||
t.Errorf("the teller may not say %s as itself", verb)
|
||||
}
|
||||
}
|
||||
if !allowed(teller.Subscribe, "mesh.seat.operator-channel.event.decided.mesh-delivery-teller") {
|
||||
t.Errorf("the teller does not hear its own decided events: %v", teller.Subscribe)
|
||||
}
|
||||
}
|
||||
|
||||
// 1243/AC2: each part's credential is refused writing another module's bucket, publishing under another module's name
|
||||
// and answering a verb subject of a seat.
|
||||
func TestNoPartsCredentialReachesBeyondItsDeclaration(t *testing.T) {
|
||||
all := composed(t, partsRecords(t))
|
||||
parts := []string{"mesh-delivery-forge-bridge", "mesh-delivery-indexer", "mesh-delivery-teller"}
|
||||
buckets := map[string][]string{
|
||||
"mesh-delivery-forge-bridge": {"heads"}, "mesh-delivery-indexer": {"candidates", "modules"},
|
||||
"mesh-delivery-teller": {"answers"}, "mesh-delivery-gate": {"merges", "owed", "source", "asks", "deliveries"},
|
||||
"mesh-controller": {"builds", "plans"},
|
||||
}
|
||||
for _, part := range parts {
|
||||
p := all[part+"@novox"]
|
||||
for owner, names := range buckets {
|
||||
if owner == part {
|
||||
continue
|
||||
}
|
||||
for _, name := range names {
|
||||
if canWrite(p, owner+"_"+name, "some.key") {
|
||||
t.Errorf("%s may write %s_%s", part, owner, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, other := range append(append([]string(nil), parts...), "mesh-delivery-gate", "mesh-controller", "gitea", "mesh-delivery") {
|
||||
if other == part {
|
||||
continue
|
||||
}
|
||||
for _, subject := range []string{"mesh.mod." + other + ".event.merged", "mesh.stream." + other + ".event.forge.merged"} {
|
||||
if allowed(p.Publish, subject) {
|
||||
t.Errorf("%s may publish %s, under %s's name", part, subject, other)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, verb := range []string{"mesh.seat.mesh-delivery.tool.deliveries", "mesh.seat.mesh-delivery.tool.release",
|
||||
"mesh.seat.mesh-delivery.tool.stop", "mesh.seat.operator-channel.tool.tell"} {
|
||||
if allowed(p.Subscribe, verb) {
|
||||
t.Errorf("%s answers the verb subject %s", part, verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The teller says what it says as itself alone.
|
||||
teller := all["mesh-delivery-teller@novox"]
|
||||
for _, subject := range []string{"mesh.seat.operator-channel.accept.ask.mesh-delivery-gate",
|
||||
"mesh.seat.operator-channel.accept.condition.mesh-delivery-indexer", "mesh.seat.operator-channel.accept.ask.*",
|
||||
"mesh.seat.operator-channel.accept.ask.>"} {
|
||||
if allowed(teller.Publish, subject) {
|
||||
t.Errorf("the teller may publish %s", subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// toolServer is a module that serves a tool on its own broker credential: the runner carries it (ADR 0175).
|
||||
func toolServer() Declared {
|
||||
return Declared{Module: "some-tools", Serves: []string{"some_tool"},
|
||||
State: []Bucket{{Module: "some-tools", Name: "kept", History: 1}}}
|
||||
}
|
||||
|
||||
// 1329/AC1: composed from the manifests of mesh-delivery's main, the machine's tool runner may write no bucket of the
|
||||
// forge bridge, the indexer, the teller or the watchman, publish none of their stream events nor the teller's
|
||||
// operator-channel accept subjects, and pull none of their consumers; each part's own credential still may.
|
||||
func TestTheToolRunnerCarriesNoneOfTheDeliverysParts(t *testing.T) {
|
||||
streams := map[string]ModuleStream{"mesh-delivery-forge-bridge": {Module: "mesh-delivery-forge-bridge", Name: "FORGE"}}
|
||||
all := composed(t, Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {
|
||||
{Module: RuntimeModule}, declaredOf(t, bridgeManifest, streams), declaredOf(t, indexerManifest, streams),
|
||||
declaredOf(t, tellerManifest, streams), declaredOf(t, watchmanManifest, streams),
|
||||
}}})
|
||||
runner := all["runner@novox"]
|
||||
buckets := map[string][]string{
|
||||
"mesh-delivery-forge-bridge": {"heads"}, "mesh-delivery-indexer": {"candidates", "modules"},
|
||||
"mesh-delivery-teller": {"answers"}, "mesh-delivery-watchman": {"presence", "pulse", "stuck"},
|
||||
}
|
||||
for part, names := range buckets {
|
||||
own, there := all[part+"@novox"]
|
||||
if !there {
|
||||
t.Errorf("%s has no credential of its own", part)
|
||||
continue
|
||||
}
|
||||
for _, name := range names {
|
||||
if canWrite(runner, part+"_"+name, "some.key") {
|
||||
t.Errorf("novox's tool runner may write %s_%s", part, name)
|
||||
}
|
||||
if !canWrite(own, part+"_"+name, "some.key") {
|
||||
t.Errorf("%s may not write its own bucket %s", part, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, subject := range []string{
|
||||
"mesh.stream.mesh-delivery-forge-bridge.event.forge.merged",
|
||||
"mesh.seat.operator-channel.accept.ask.mesh-delivery-teller",
|
||||
"mesh.seat.operator-channel.accept.cancel.mesh-delivery-teller",
|
||||
"mesh.seat.operator-channel.accept.condition.mesh-delivery-teller",
|
||||
"$JS.API.CONSUMER.MSG.NEXT.FORGE.novox_mesh-delivery-indexer",
|
||||
} {
|
||||
if allowed(runner.Publish, subject) {
|
||||
t.Errorf("novox's tool runner may publish %s", subject)
|
||||
}
|
||||
}
|
||||
if !allowed(all["mesh-delivery-forge-bridge@novox"].Publish, "mesh.stream.mesh-delivery-forge-bridge.event.forge.merged") ||
|
||||
!allowed(all["mesh-delivery-teller@novox"].Publish, "mesh.seat.operator-channel.accept.ask.mesh-delivery-teller") ||
|
||||
!allowed(all["mesh-delivery-indexer@novox"].Publish, "$JS.API.CONSUMER.MSG.NEXT.FORGE.novox_mesh-delivery-indexer") {
|
||||
t.Error("a part's own credential lost what its manifest grants")
|
||||
}
|
||||
}
|
||||
|
||||
// 1329/AC2: a module with tools and a broker secret of its own stays carried by the tool runner, beside its own
|
||||
// credential (ADR 0175); a module with neither tools, a seat, waits nor a check is not.
|
||||
func TestTheToolRunnerStillCarriesAModuleWithToolsAndItsOwnCredential(t *testing.T) {
|
||||
process := Declared{Module: "only-process", State: []Bucket{{Module: "only-process", Name: "mine", History: 1}}}
|
||||
all := composed(t, Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {
|
||||
{Module: RuntimeModule}, toolServer(), process,
|
||||
}}})
|
||||
runner := all["runner@novox"]
|
||||
if !canWrite(runner, "some-tools_kept", "k") {
|
||||
t.Error("novox's tool runner does not carry a module with tools")
|
||||
}
|
||||
if !canWrite(all["some-tools@novox"], "some-tools_kept", "k") {
|
||||
t.Error("the module with tools lost its own credential")
|
||||
}
|
||||
if canWrite(runner, "only-process_mine", "k") {
|
||||
t.Error("novox's tool runner carries a module that has nothing to serve")
|
||||
}
|
||||
if !canWrite(all["only-process@novox"], "only-process_mine", "k") {
|
||||
t.Error("the process module lost its own credential")
|
||||
}
|
||||
for name, d := range map[string]Declared{
|
||||
"a held seat": {Module: "m", Holds: []Seat{{Name: "some-seat"}}},
|
||||
"a wait": {Module: "m", Waits: []Wait{{}}},
|
||||
"a check": {Module: "m", Checks: []string{"m.check"}},
|
||||
} {
|
||||
if !runnerServes(d) {
|
||||
t.Errorf("the tool runner does not serve a module with %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// routeProxyExcerpt is the part of mesh-control's modules/route-proxy/module.json at 70cb217601f9 (read 2026-10-20)
|
||||
// that decides its grants: its state, what it reads, its own broker secret, and the bundle `provider` that the machine's
|
||||
// runtime loads (`loads`) though the module declares no tools.
|
||||
const routeProxyExcerpt = `{
|
||||
"module": "route-proxy",
|
||||
"state": ["bound"],
|
||||
"reads": ["mesh-controller.assignments"],
|
||||
"own-secrets": { "broker": "${dir:mesh-state}/broker" },
|
||||
"build": { "artifacts": [
|
||||
{ "name": "provider", "kind": "bundle", "language": "go", "from": "cmd/route-provider", "binary": "route-provider", "loads": ["route-provider"] }
|
||||
] }
|
||||
}`
|
||||
|
||||
// 1329 (review): a module with no tools whose bundle the runtime loads stays carried, because its helper writes its state
|
||||
// and reads on the runner's account; the delivery's five parts are still dropped.
|
||||
func TestTheToolRunnerStillCarriesAModuleWhoseBundleItLoads(t *testing.T) {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
State []string `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Loads []string `json:"loads"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(routeProxyExcerpt), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy := Declared{Module: m.Module, Reads: m.Reads}
|
||||
for _, s := range m.State {
|
||||
proxy.State = append(proxy.State, Bucket{Module: m.Module, Name: s, History: 1})
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
proxy.Loads = proxy.Loads || len(a.Loads) > 0
|
||||
}
|
||||
streams := map[string]ModuleStream{"mesh-delivery-forge-bridge": {Module: "mesh-delivery-forge-bridge", Name: "FORGE"}}
|
||||
all := composed(t, Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {
|
||||
{Module: RuntimeModule}, proxy, declaredOf(t, bridgeManifest, streams), declaredOf(t, indexerManifest, streams),
|
||||
declaredOf(t, tellerManifest, streams), declaredOf(t, watchmanManifest, streams),
|
||||
}}})
|
||||
runner := all["runner@novox"]
|
||||
if !canWrite(runner, "route-proxy_bound", "k") {
|
||||
t.Error("novox's tool runner does not carry route-proxy, whose bundle it loads")
|
||||
}
|
||||
if !canWrite(all["route-proxy@novox"], "route-proxy_bound", "k") {
|
||||
t.Error("route-proxy lost its own credential")
|
||||
}
|
||||
for _, b := range []string{"mesh-delivery-teller_answers", "mesh-delivery-forge-bridge_heads", "mesh-delivery-indexer_modules", "mesh-delivery-watchman_stuck"} {
|
||||
if canWrite(runner, b, "k") {
|
||||
t.Errorf("novox's tool runner may write %s", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
+33
-10
@@ -171,6 +171,10 @@ func ConsumerFor(p Principal) (Consumer, bool) {
|
||||
Filters: filters,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: 5,
|
||||
// From now, as the controller's own consumers have been since issue 248: a module made on a stream that
|
||||
// keeps a week replays none of it. What a part needs of the present it reads from its owner
|
||||
// (novox/hq ADR 0443 §1, §3). A consumer that exists keeps its place whatever this says.
|
||||
FromNow: true,
|
||||
Why: "what " + p.Module + " declared it consumes; after max-deliver it gives an event up, and the " +
|
||||
"controller keeps it in DEAD_LETTERS until a person delivers it again or drops it",
|
||||
}, true
|
||||
@@ -190,18 +194,33 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
var out []ModuleConsumer
|
||||
seen := map[string]bool{}
|
||||
add := func(p Principal) {
|
||||
c, needed := ConsumerFor(p)
|
||||
if !needed || seen[p.Node+"/"+p.Module] {
|
||||
key := p.Node + "/" + p.Module
|
||||
if seen[key] {
|
||||
return
|
||||
}
|
||||
seen[p.Node+"/"+p.Module] = true
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: c})
|
||||
// And one on each module stream it hears (novox/hq ADR 0363 §5).
|
||||
for _, h := range p.Hears {
|
||||
if len(h.Filters) == 0 {
|
||||
continue
|
||||
c, needed := ConsumerFor(p)
|
||||
// **A module with a work-queue consumer of its own needs no other reason to be visited here**
|
||||
// (novox/hq ADR 0436 §1, §2, §6): mesh-mail's sender consumes and watches nothing of its own, so
|
||||
// ConsumerFor alone would have found nothing needed and skipped it before its one queue consumer
|
||||
// was ever composed.
|
||||
if !needed && len(p.Queues) == 0 {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
if needed {
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: c})
|
||||
// And one on each module stream it hears (novox/hq ADR 0363 §5).
|
||||
for _, h := range p.Hears {
|
||||
if len(h.Filters) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: heardConsumer(p, h)})
|
||||
}
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: heardConsumer(p, h)})
|
||||
}
|
||||
// And the one durable consumer of each of its own work-queue streams, for the one process its
|
||||
// bus.queues names (novox/hq ADR 0436 §1, §2, §6).
|
||||
for _, q := range p.Queues {
|
||||
out = append(out, ModuleConsumer{Node: p.Node, Module: p.Module, Consumer: QueueConsumer(p, q)})
|
||||
}
|
||||
}
|
||||
for _, p := range users {
|
||||
@@ -214,9 +233,13 @@ func ConsumersOf(users []Principal) []ModuleConsumer {
|
||||
continue
|
||||
}
|
||||
for _, d := range p.Carries {
|
||||
// **A carried module's work-queue consumer is still the controller's to make** (novox/hq ADR
|
||||
// 0436 §1, §2, §6), the same reason its ordinary consumer is (ADR 0198): mesh-mail has no
|
||||
// account of its own (NoAccount) and is carried here, but its process does, and the consumer
|
||||
// it reads through must exist and survive PruneHeard whether or not the module itself speaks.
|
||||
add(Principal{Kind: KindModule, Node: p.Node, Module: d.Module, Emits: d.Emits,
|
||||
Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches,
|
||||
Streams: d.Streams, Hears: ComposedHears(d)})
|
||||
Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues})
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -180,3 +180,35 @@ func TestAHoldersWorkerIsPulledByEveryHolder(t *testing.T) {
|
||||
t.Fatalf("the worker is %s on %s; one per seat, shared by its holders", c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
|
||||
// 1279/AC1 (ADR 0443 §1): every module principal's durable consumer on a stream that keeps its history is made from
|
||||
// now, the operator-channel holder's consumer of condition events among them; a seat's worker and a module's own work
|
||||
// queue are not, because work queued before its holder came is work to do.
|
||||
func TestAModulesConsumerIsMadeFromNow(t *testing.T) { // 1279/AC1
|
||||
for name, p := range map[string]Principal{
|
||||
"consumes": {Kind: KindModule, Node: "one", Module: "audit", Consumes: []string{"shop.order.placed"}, PasswordHash: "x"},
|
||||
"watches": {Kind: KindModule, Node: "one", Module: "mesh-catalog", PasswordHash: "x",
|
||||
Watches: []Seat{{Name: "mesh-build-machine", Emits: []string{"built"}}}},
|
||||
// The operator-channel holder (issue 549): a module that holds a seat and consumes the condition events.
|
||||
"a seat holder consuming conditions": {Kind: KindModule, Node: "one", Module: "messenger", PasswordHash: "x",
|
||||
Consumes: []string{"mesh-controller.condition.raised", "mesh-controller.condition.cleared"}},
|
||||
} {
|
||||
c, ok := ConsumerFor(p)
|
||||
if !ok {
|
||||
t.Fatalf("%s: no consumer derived", name)
|
||||
}
|
||||
if !c.FromNow {
|
||||
t.Errorf("%s: the consumer %s on %s is made from the start of the stream", name, c.Name, c.Stream)
|
||||
}
|
||||
}
|
||||
if c := heardConsumer(Principal{Kind: KindModule, Node: "ace", Module: "reader", Hears: []Heard{mailHeard}}, mailHeard); !c.FromNow {
|
||||
t.Errorf("a module's consumer on another module's stream is made from the start of %s", c.Stream)
|
||||
}
|
||||
if c, _ := HolderConsumerFor("one", "telegram", telegramSeat()); c.FromNow {
|
||||
t.Error("a seat's worker is made from now: work queued before its holder came would be lost")
|
||||
}
|
||||
q := QueueConsumer(Principal{Kind: KindModule, Node: "novox", Module: "mesh-mail"}, QueueOwner{Process: "send", Stream: "MAIL_SENDS", MaxDeliver: 5})
|
||||
if q.FromNow {
|
||||
t.Error("a module's own work queue is made from now: queued sends would be lost")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,16 +154,10 @@ func Borrow(j *JetStream) *JetStream {
|
||||
return &lent
|
||||
}
|
||||
|
||||
// EnsureStream creates the stream if it is absent and brings it to match if it is present.
|
||||
//
|
||||
// **Idempotent, because the controller asserts on every start** rather than creating once at
|
||||
// genesis: a stream somebody deleted, or a mesh raised from a restored backup, has to converge
|
||||
// rather than run without the guarantee its messages assume.
|
||||
//
|
||||
// An update, not a delete and recreate. Recreating would discard every message the stream holds
|
||||
// and every consumer's position in it — which for CONTROL means the pushes being held through a
|
||||
// store restart, exactly the guarantee the stream exists for.
|
||||
func (j *JetStream) EnsureStream(s Stream) error {
|
||||
// StreamConfigFor is the server's own config for one of the mesh's streams, pure so the shape it builds —
|
||||
// not least the restricted metadata a work-queue or a grant-restricted stream's dead letters age and hide
|
||||
// by (novox/hq ADR 0436 §6, ADR 0376 §3) — is testable without a server.
|
||||
func StreamConfigFor(s Stream) *nats.StreamConfig {
|
||||
want := &nats.StreamConfig{
|
||||
Name: s.Name,
|
||||
Subjects: s.Subjects,
|
||||
@@ -174,8 +168,14 @@ func (j *JetStream) EnsureStream(s Stream) error {
|
||||
}
|
||||
want.AllowDirect = s.Direct
|
||||
want.AllowMsgTTL = s.AllowMsgTTL
|
||||
if s.Restricted {
|
||||
want.Metadata = map[string]string{RestrictedMetadata: "true"}
|
||||
if s.Restricted || s.WorkQueue {
|
||||
want.Metadata = map[string]string{}
|
||||
if s.Restricted {
|
||||
want.Metadata[RestrictedMetadata] = "true"
|
||||
}
|
||||
if s.WorkQueue {
|
||||
want.Metadata[WorkQueueMetadata] = "true"
|
||||
}
|
||||
}
|
||||
if s.MaxBytes > 0 {
|
||||
want.MaxBytes = s.MaxBytes
|
||||
@@ -193,6 +193,20 @@ func (j *JetStream) EnsureStream(s Stream) error {
|
||||
want.MaxMsgsPerSubject = 1
|
||||
want.MaxAge = 0
|
||||
}
|
||||
return want
|
||||
}
|
||||
|
||||
// EnsureStream creates the stream if it is absent and brings it to match if it is present.
|
||||
//
|
||||
// **Idempotent, because the controller asserts on every start** rather than creating once at
|
||||
// genesis: a stream somebody deleted, or a mesh raised from a restored backup, has to converge
|
||||
// rather than run without the guarantee its messages assume.
|
||||
//
|
||||
// An update, not a delete and recreate. Recreating would discard every message the stream holds
|
||||
// and every consumer's position in it — which for CONTROL means the pushes being held through a
|
||||
// store restart, exactly the guarantee the stream exists for.
|
||||
func (j *JetStream) EnsureStream(s Stream) error {
|
||||
want := StreamConfigFor(s)
|
||||
|
||||
switch _, err := j.js.StreamInfo(s.Name); {
|
||||
case err == nil:
|
||||
@@ -436,30 +450,50 @@ func retentionOf(r Retention) nats.RetentionPolicy {
|
||||
// discards what the bucket holds, and what a module's state holds is data. The mesh's caps are
|
||||
// asserted with the owner's options, so a bucket made by hand converges to them.
|
||||
func (j *JetStream) EnsureBucket(b Bucket) error {
|
||||
history := b.History
|
||||
if history == 0 {
|
||||
history = 1
|
||||
}
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: b.Bucket(),
|
||||
Description: b.Why(),
|
||||
History: uint8(history),
|
||||
TTL: time.Duration(b.TTLSeconds) * time.Second,
|
||||
MaxValueSize: StateMaxValueBytes,
|
||||
MaxBytes: StateMaxBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
// A stream that allows a time to live per message cannot be made to disallow it (the server refuses the update),
|
||||
// so a bucket that has it keeps it whatever the owner now declares (novox/hq ADR 0433 §1).
|
||||
var held time.Duration
|
||||
if kv, err := js.KeyValue(ctx, b.Bucket()); err == nil {
|
||||
if st, err := kv.Status(ctx); err == nil {
|
||||
held = st.LimitMarkerTTL()
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, BucketConfig(b, held)); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", b.Bucket(), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BucketConfig is the options a module's bucket is asserted with. held is the marker age the bucket already has, zero
|
||||
// when it is absent or has none: **a bucket that allows a time to live per message keeps it** (the server refuses to
|
||||
// turn it off), so without key-ttl it is asserted with the age it holds, and with key-ttl with KeyTTLMarkerTTL.
|
||||
func BucketConfig(b Bucket, held time.Duration) jetstream.KeyValueConfig {
|
||||
history := b.History
|
||||
if history == 0 {
|
||||
history = 1
|
||||
}
|
||||
markers := held
|
||||
if b.KeyTTL {
|
||||
markers = KeyTTLMarkerTTL
|
||||
}
|
||||
return jetstream.KeyValueConfig{
|
||||
Bucket: b.Bucket(),
|
||||
Description: b.Why(),
|
||||
History: uint8(history),
|
||||
TTL: time.Duration(b.TTLSeconds) * time.Second,
|
||||
MaxValueSize: StateMaxValueBytes,
|
||||
MaxBytes: StateMaxBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
LimitMarkerTTL: markers,
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNames is every key-value bucket on the server, the mesh's and anybody else's.
|
||||
func (j *JetStream) BucketNames() ([]string, error) {
|
||||
js, err := jetstream.New(j.conn)
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package broker
|
||||
|
||||
import "testing"
|
||||
|
||||
// mailSendsStream is mesh-mail's future `mail-sends` stream (novox/hq ADR 0436 §7, §8), as the bus holds it:
|
||||
// work-queue retention, a 24-hour duplicate window, kept one day and at most 64 MiB.
|
||||
var mailSendsStream = ModuleStream{Module: "mesh-mail", Name: "MAIL_SENDS",
|
||||
Events: []string{"mail.send.approved"}, MaxAgeDays: 1, MaxMiB: 64, WorkQueue: true, Duplicates: 24}
|
||||
|
||||
// 1177/AC1: a work-queue stream is composed with work-queue retention, refusing a new message when full
|
||||
// (DiscardNew), a 24-hour duplicate window, a one-day maximum age and a 64 MiB cap.
|
||||
func TestAWorkQueueStreamIsComposedToRefuseWhenFull(t *testing.T) {
|
||||
s := mailSendsStream.Stream()
|
||||
if s.Retention != RetentionWorkQueue {
|
||||
t.Fatalf("retention: %v, want %v", s.Retention, RetentionWorkQueue)
|
||||
}
|
||||
if !s.DiscardNew {
|
||||
t.Fatal("a work-queue stream did not refuse a new message when full")
|
||||
}
|
||||
if s.DuplicatesSeconds != 24*60*60 {
|
||||
t.Fatalf("duplicate window: %d seconds, want %d (24 hours)", s.DuplicatesSeconds, 24*60*60)
|
||||
}
|
||||
if s.MaxAge != 24*60*60 {
|
||||
t.Fatalf("max age: %d seconds, want %d (1 day)", s.MaxAge, 24*60*60)
|
||||
}
|
||||
if s.MaxBytes != 64<<20 {
|
||||
t.Fatalf("max bytes: %d, want %d (64 MiB)", s.MaxBytes, 64<<20)
|
||||
}
|
||||
}
|
||||
|
||||
// 1177/AC3: a stream with no work-queue clause composes exactly as main does: limits retention, dropping its
|
||||
// oldest, never refusing a new message.
|
||||
func TestANonWorkQueueStreamStillDropsItsOldest(t *testing.T) {
|
||||
s := mailStream.Stream()
|
||||
if s.Retention != RetentionLimits || s.DiscardNew || s.DuplicatesSeconds != 0 {
|
||||
t.Fatalf("composed as %+v; want limits retention, never discarding new, no duplicate window", s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1178/AC1: the process a work-queue stream's bus.queues names gets exactly one durable consumer,
|
||||
// <node>_<module>_<process id>, with the stream's declared max-deliver — composed even though mesh-mail's
|
||||
// sender otherwise consumes and watches nothing of its own, which alone would give ConsumerFor nothing
|
||||
// needed. Explicit acknowledgement is EnsureConsumer's own rule for every consumer it makes (jetstream.go),
|
||||
// so it is not re-asserted here.
|
||||
func TestTheOneProcessAWorkQueuesBusNamesGetsItsOwnConsumer(t *testing.T) {
|
||||
sender := Principal{Kind: KindModule, Node: "novox", Module: "mesh-mail", PasswordHash: "x",
|
||||
Queues: []QueueOwner{{Process: "send", Stream: "MAIL_SENDS", MaxDeliver: 5}}}
|
||||
got := ConsumersOf([]Principal{sender})
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("mesh-mail's consumers: %+v", got)
|
||||
}
|
||||
c := got[0].Consumer
|
||||
if c.Name != "novox_mesh-mail_send" || c.Stream != "MAIL_SENDS" || c.MaxDeliver != 5 {
|
||||
t.Fatalf("the queue's consumer: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// review round 1, finding 2 (1178): mesh-mail's real shape has no account of its own (NoAccount) and is
|
||||
// carried by the machine's tool runner — so its queue consumer has to come from the carried-module path in
|
||||
// ConsumersOf, not only from a module with its own principal, or PruneHeard would remove what issueProcess
|
||||
// made for it (novox/hq ADR 0436 §1, §2, §6).
|
||||
func TestAWorkQueueConsumerSurvivesForAModuleTheRuntimeCarries(t *testing.T) {
|
||||
users, err := Users(Records{Nodes: []string{"novox"}, Assigned: map[string][]Declared{"novox": {
|
||||
{Module: RuntimeModule},
|
||||
{Module: "mesh-mail", NoAccount: true,
|
||||
Queues: []QueueOwner{{Process: "send", Stream: "MAIL_SENDS", MaxDeliver: 5}}},
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := ConsumersOf(users)
|
||||
var found bool
|
||||
for _, c := range got {
|
||||
if c.Module == "mesh-mail" && c.Consumer.Name == "novox_mesh-mail_send" && c.Consumer.Stream == "MAIL_SENDS" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("mesh-mail's queue consumer is missing from the carried module's composition: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,12 @@ type ModuleStream struct {
|
||||
MaxMiB int
|
||||
// Restricted says a trusted config value of the module grants who hears it (ADR 0363 §4).
|
||||
Restricted bool
|
||||
// WorkQueue says this stream holds ADR 0436 §6's work-queue retention: a message is removed once a
|
||||
// consumer acknowledges it, and the bus refuses a new one when full rather than dropping its oldest.
|
||||
WorkQueue bool
|
||||
// Duplicates is the work-queue's duplicate window in hours (novox/hq ADR 0436 §6); meaningless unless
|
||||
// WorkQueue.
|
||||
Duplicates int
|
||||
}
|
||||
|
||||
// Subjects are the subjects the stream takes.
|
||||
@@ -70,7 +76,11 @@ func (s ModuleStream) Stream() Stream {
|
||||
if s.Restricted {
|
||||
why += "; heard only by the consumers its trusted config value grants"
|
||||
}
|
||||
return Stream{
|
||||
if s.WorkQueue {
|
||||
why += "; a work-queue (novox/hq ADR 0436 §6): a message is removed once its one consumer " +
|
||||
"acknowledges it, and a full stream refuses a new one rather than dropping what it holds"
|
||||
}
|
||||
out := Stream{
|
||||
Name: s.Name,
|
||||
Subjects: s.Subjects(),
|
||||
Retention: RetentionLimits,
|
||||
@@ -80,6 +90,50 @@ func (s ModuleStream) Stream() Stream {
|
||||
// Said on the bus, so a dead letter kept from it ages out with it (novox/hq ADR 0376 §3).
|
||||
Restricted: s.Restricted,
|
||||
}
|
||||
if s.WorkQueue {
|
||||
out.Retention = RetentionWorkQueue
|
||||
out.DiscardNew = true
|
||||
out.DuplicatesSeconds = s.Duplicates * 60 * 60
|
||||
// Said on the bus too, reusing the one flag a restricted stream's dead letters already age and hide
|
||||
// by (novox/hq ADR 0436 §6, ADR 0376 §3-4): a work-queue stream's grant is always empty (ADR 0436
|
||||
// §6 refuses one that carries a grant), so this never collides with a trusted-config restriction.
|
||||
out.Restricted = true
|
||||
// And its own metadata, apart from the bus's work-queue retention SEAT_* and other mesh-internal
|
||||
// streams share without being one of this clause's: only this flag says a dead letter kept from the
|
||||
// stream is drop-only (novox/hq ADR 0436 §6), never the stream's Retention alone.
|
||||
out.WorkQueue = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// QueueOwner is the one process of a module that is the durable consumer of one of its own work-queue
|
||||
// streams (novox/hq ADR 0436 §1, §2, §6): named in that process's bus.queues, and checked at registration
|
||||
// to be the only one (catalogue.WorkQueueStreamProblems).
|
||||
type QueueOwner struct {
|
||||
// Process is the process resource's id — its bus account's name, with the node and module (novox/hq ADR
|
||||
// 0436 §2).
|
||||
Process string
|
||||
// Stream is the work-queue stream's name on the bus.
|
||||
Stream string
|
||||
// MaxDeliver bounds how many times a message is handed over before it is given up as a dead letter
|
||||
// (novox/hq ADR 0436 §6).
|
||||
MaxDeliver int
|
||||
}
|
||||
|
||||
// QueueConsumer is the durable consumer the one process a module's bus.queues names is granted on its own
|
||||
// work-queue stream (novox/hq ADR 0436 §1, §2, §6): no filter, since it is the stream's only consumer —
|
||||
// unlike heardConsumer, which is one of possibly several hearing a module's ordinary stream.
|
||||
func QueueConsumer(p Principal, q QueueOwner) Consumer {
|
||||
return Consumer{
|
||||
Name: p.Node + "_" + p.Module + "_" + q.Process,
|
||||
Stream: q.Stream,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: q.MaxDeliver,
|
||||
Why: fmt.Sprintf("%s's process %s is the one consumer %s's bus.queues names for its work-queue "+
|
||||
"stream %s (novox/hq ADR 0436 §6); after max-deliver %d it gives a message up, and the "+
|
||||
"controller keeps it in %s, offered only drop", p.Module, q.Process, p.Module, q.Stream,
|
||||
q.MaxDeliver, DeadLettersStream),
|
||||
}
|
||||
}
|
||||
|
||||
// ModuleStreams is every declared module stream as the controller asserts it, in name order.
|
||||
@@ -169,6 +223,8 @@ func heardConsumer(p Principal, h Heard) Consumer {
|
||||
Filters: filters,
|
||||
AckWaitSeconds: 30,
|
||||
MaxDeliver: 5,
|
||||
// From now, like a module's consumer on EVENTS (novox/hq ADR 0443 §1).
|
||||
FromNow: true,
|
||||
Why: "what " + p.Module + " consumes of " + h.Emitter + "'s stream " + h.Stream + " and is granted " +
|
||||
"(novox/hq ADR 0363); after max-deliver it gives an event up, and the controller keeps it in " +
|
||||
"DEAD_LETTERS until a person delivers it again or drops it",
|
||||
|
||||
@@ -42,6 +42,16 @@ func TestAModuleStreamOnARealBus(t *testing.T) {
|
||||
if err := js.EnsureConsumer(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Made from now (novox/hq ADR 0443 §1): the two mails published above are its past, not its news.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if info, _ := js.js.ConsumerInfo("MAIL", c.Name); info.NumPending != 0 {
|
||||
t.Fatalf("the reader made after two mails has %d of them pending, want 0", info.NumPending)
|
||||
}
|
||||
for _, account := range []string{"mesh", "work"} {
|
||||
if _, err := js.js.Publish(ModuleStreamSubject("mesh-mail", "mail."+account+".message.added"), []byte("{}")); err != nil {
|
||||
t.Fatalf("publishing %s's mail: %v", account, err)
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if info, _ := js.js.ConsumerInfo("MAIL", c.Name); info.NumPending != 1 {
|
||||
t.Fatalf("the reader granted `mesh` has %d pending, want 1", info.NumPending)
|
||||
|
||||
@@ -154,7 +154,7 @@ func TestTheToolRunnerNeverHearsARestrictedStream(t *testing.T) {
|
||||
records := Records{Nodes: []string{"ace", "novox"}, Assigned: map[string][]Declared{
|
||||
"novox": {{Module: "mesh-mail", Streams: []ModuleStream{mailStream}}, {Module: "news", Streams: []ModuleStream{newsStream}}},
|
||||
"ace": {{Module: RuntimeModule},
|
||||
{Module: "carried", Hears: []Heard{mailHeard, newsHeard}, HearsAStream: true},
|
||||
{Module: "carried", NoAccount: true, Hears: []Heard{mailHeard, newsHeard}, HearsAStream: true},
|
||||
{Module: "mailer", RunsAs: "mailer", Hears: []Heard{mailHeard}, HearsAStream: true}},
|
||||
}}
|
||||
users, err := Users(records)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user