Compare commits
126
Commits
aa02e7bf4e
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c2a8d6210 | ||
|
|
c07f604047 | ||
|
|
b893cc829c | ||
|
|
70cb217601 | ||
|
|
1d0df24a0d | ||
|
|
202b453de1 | ||
|
|
7a0b275020 | ||
|
|
40350d82fa | ||
|
|
1f708e2d8d | ||
|
|
7f23c537cc | ||
|
|
0aac89f4e2 | ||
|
|
d8c13d97ca | ||
|
|
f1775281e2 | ||
|
|
4083755ca1 | ||
|
|
7b83aaa949 | ||
|
|
23a209f3d0 | ||
|
|
3448f98000 | ||
|
|
12db1d3748 | ||
|
|
7770d193da | ||
|
|
248fe521a3 | ||
|
|
666dcff79b | ||
|
|
caa3fc8995 | ||
|
|
a0c971cea9 | ||
|
|
8802076499 | ||
|
|
2d2e736b45 | ||
|
|
fdfda0e839 | ||
|
|
540cda11df | ||
|
|
ad753c70f6 | ||
|
|
da20fea549 | ||
|
|
2f4f50b9ca | ||
|
|
14f405ece5 | ||
|
|
0d1d780d43 | ||
|
|
5c807808e8 | ||
|
|
da5a5c9b67 | ||
|
|
07a8f04e65 | ||
|
|
dd802c81a7 | ||
|
|
a5a43c8f31 | ||
|
|
3e7f34d041 | ||
|
|
e29e855bdf | ||
|
|
da3c01089e | ||
|
|
253283d68e | ||
|
|
3f038b989e | ||
|
|
3834c72747 | ||
|
|
68088ec782 | ||
|
|
8da2619920 | ||
|
|
59c8cf2f24 | ||
|
|
3ac8657a49 | ||
|
|
230f0a1a18 | ||
|
|
6e487fa01f | ||
|
|
7a74cee83f | ||
|
|
378e65780d | ||
|
|
296e292f55 | ||
|
|
791b92c875 | ||
|
|
1490c3b075 | ||
|
|
4104d49ec8 | ||
|
|
1991fe931b | ||
|
|
d8249dad8b | ||
|
|
df4d1204c7 | ||
|
|
e8c8f0488b | ||
|
|
cfb6a8c531 | ||
|
|
9b7f9a7fd2 | ||
|
|
65632e7999 | ||
|
|
f6a48ac5ef | ||
|
|
93a82cb032 | ||
|
|
de651e0fd1 | ||
|
|
97ee33e8f4 | ||
|
|
19e3eac746 | ||
|
|
863db82737 | ||
|
|
a069898c0c | ||
|
|
ffad53aad2 | ||
|
|
89cfa9431d | ||
|
|
5939bbdefe | ||
|
|
ec85cdf70c | ||
|
|
cd303f68ed | ||
|
|
2cbebbc011 | ||
|
|
ac79e29319 | ||
|
|
ef0ca69cb0 | ||
|
|
cec26d8596 | ||
|
|
e621e15d2f | ||
|
|
a08498b240 | ||
|
|
39ae7de6d3 | ||
|
|
4e12fc19b7 | ||
|
|
19f15d8ea0 | ||
|
|
b1cfb85f86 | ||
|
|
75a27ec1a4 | ||
|
|
85af245f1a | ||
|
|
1ec27e6ba1 | ||
|
|
425134784a | ||
|
|
fb66608a13 | ||
|
|
7be2df627f | ||
|
|
cb006fc8dd | ||
|
|
0273609090 | ||
|
|
3b49d7044f | ||
|
|
ba59283b4b | ||
|
|
461fa9c7b4 | ||
|
|
06bf8cb2fb | ||
|
|
222c9c1d72 | ||
|
|
0520773412 | ||
|
|
eba864dc5f | ||
|
|
2d3d5ab616 | ||
|
|
dc65fe2319 | ||
|
|
fc43238267 | ||
|
|
8c8bc5021e | ||
|
|
557c11af1e | ||
|
|
661e8eff2f | ||
|
|
8c0fa40e65 | ||
|
|
6746ab6492 | ||
|
|
48e9656c67 | ||
|
|
a825a4c904 | ||
|
|
315c622386 | ||
|
|
d3556f452e | ||
|
|
68db18bdfc | ||
|
|
cbc6bac8c0 | ||
|
|
0b4cd46b3f | ||
|
|
7f074d9c94 | ||
|
|
e76ba966b3 | ||
|
|
da6d038e07 | ||
|
|
c795915315 | ||
|
|
4eeba8fb20 | ||
|
|
d095a3dcef | ||
|
|
5b08eb09be | ||
|
|
472744bbaa | ||
|
|
9859923664 | ||
|
|
fdab58ae4f | ||
|
|
3b989d8bb3 | ||
|
|
88ea361bf6 |
@@ -432,6 +432,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
}
|
}
|
||||||
result.Against = built.Against
|
result.Against = built.Against
|
||||||
result.SourceFingerprint = built.Source
|
result.SourceFingerprint = built.Source
|
||||||
|
result.SourceInputs = built.SourceInputs
|
||||||
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||||
result.ReuseKey = built.ReuseKey
|
result.ReuseKey = built.ReuseKey
|
||||||
if built.Reused != nil {
|
if built.Reused != nil {
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ const packageManagerKey = "the package manager ("
|
|||||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||||
heard := map[string]bool{}
|
heard := map[string]bool{}
|
||||||
for _, m := range f.machines {
|
for _, m := range f.machines {
|
||||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) {
|
||||||
heard[m.name] = true
|
heard[m.name] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -208,6 +208,9 @@ type asker struct {
|
|||||||
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
// raise keeps the asker's own condition (sourceAsker): which conditions needing the operator could not be
|
||||||
// asked, and why. Nil raises nothing (a test that does not look).
|
// asked, and why. Nil raises nothing (a test that does not look).
|
||||||
raise func(ctx context.Context, obs []conditions.Observation) error
|
raise func(ctx context.Context, obs []conditions.Observation) error
|
||||||
|
// tell tells the operator one plain notice that needs no answer (operator-channel.tell, novox/hq ADR 0380):
|
||||||
|
// the refusal of an answer to a plan's ask (ADR 0449 §3). Nil tells nothing (a test that does not look).
|
||||||
|
tell func(ctx context.Context, headline string, lines []string) error
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
logf func(string, ...any)
|
logf func(string, ...any)
|
||||||
|
|
||||||
@@ -406,8 +409,10 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
for _, r := range all {
|
for _, r := range all {
|
||||||
k := partKey(r.Condition, r.Part)
|
k := partKey(r.Condition, r.Part)
|
||||||
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
if r.State == string(asks.OutcomeChosen) && now.Sub(r.Ended) < askAgainAfterAnswer {
|
||||||
|
if prior, has := answered[k]; !has || r.Ended.After(prior.Ended) {
|
||||||
answered[k] = r
|
answered[k] = r
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if r.State == string(asks.OutcomeRefused) {
|
if r.State == string(asks.OutcomeRefused) {
|
||||||
if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) {
|
if prior, has := refused[k]; !has || r.Opened.After(prior.Opened) {
|
||||||
refused[k] = r
|
refused[k] = r
|
||||||
@@ -497,7 +502,9 @@ func (a *asker) reconcile(ctx context.Context) error {
|
|||||||
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
continue // refused lately, and nothing it was refused for has changed: asked again after the wait
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) {
|
// **A condition raised after the answered ask ended is a new one** (novox/hq ADR 0449 §2): a retried
|
||||||
|
// plan that fails again is asked again, not held back for the hour its earlier answer holds.
|
||||||
|
if r, done := answered[key]; done && sameAsked(r.Actions, p.actions) && !c.Raised.After(r.Ended) {
|
||||||
if _, held := byCondition[key]; !held {
|
if _, held := byCondition[key]; !held {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -636,6 +643,8 @@ func doesWords(act conditions.Action) string {
|
|||||||
return "the delivery starts"
|
return "the delivery starts"
|
||||||
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["stop"] != "":
|
||||||
return "the delivery is stopped"
|
return "the delivery is stopped"
|
||||||
|
case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "":
|
||||||
|
return "the failed builds are asked again"
|
||||||
case strings.HasSuffix(act.Verb, ".restart"):
|
case strings.HasSuffix(act.Verb, ".restart"):
|
||||||
return "its service is restarted on " + act.Machine
|
return "its service is restarted on " + act.Machine
|
||||||
}
|
}
|
||||||
@@ -884,6 +893,7 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
|||||||
ended := a.now()
|
ended := a.now()
|
||||||
if acted != nil {
|
if acted != nil {
|
||||||
outcome = "failed: " + acted.Error()
|
outcome = "failed: " + acted.Error()
|
||||||
|
a.tellRefused(ctx, *r, act, acted)
|
||||||
}
|
}
|
||||||
r.Ended, r.Acted = ended, outcome
|
r.Ended, r.Acted = ended, outcome
|
||||||
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
if _, err := a.store.Change(ctx, r.ID, func(x *asked) bool {
|
||||||
@@ -918,6 +928,55 @@ func (a *asker) Decided(ctx context.Context, body []byte) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tellRefused tells the operator what became of the answer they gave to a plan's ask, when it was not done (novox/hq
|
||||||
|
// ADR 0449 §3): the controller's `plans` verb refuses a Retry, Start or Stop when the plan moved on between the ask and
|
||||||
|
// the answer, and the refusal was only recorded, so the operator who pressed the answer heard nothing. **A refusal is
|
||||||
|
// told as one, and nothing else is**: a verb that said no changed nothing, but a call that got no answer (a timeout, a
|
||||||
|
// refused grant) may have run, and is told as not known. One plain notice with the ask's headline, beside the hand act
|
||||||
|
// the caller still records. An answer to any other ask is not told here. A notice that cannot be told is said in the
|
||||||
|
// log and changes nothing.
|
||||||
|
func (a *asker) tellRefused(ctx context.Context, r asked, act conditions.Action, failure error) {
|
||||||
|
if a.tell == nil || act.Verb != broker.ControllerSeat+".plans" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
var refused verbRefused
|
||||||
|
if errors.As(failure, &refused) {
|
||||||
|
lines = []string{
|
||||||
|
fmt.Sprintf("Your answer, %s, could not be done: %s.", act.Label, strings.TrimRight(refusalWords(refused.words, act), ".")),
|
||||||
|
"Nothing was changed.",
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
lines = []string{
|
||||||
|
fmt.Sprintf("Your answer, %s, was given, and it is not known whether it was done.", act.Label),
|
||||||
|
"Look at the delivery before you answer again.",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := a.tell(ctx, r.Ask.Headline, lines); err != nil {
|
||||||
|
a.logf("the operator could NOT be told that %s was refused (ask %s): %v", act.Label, r.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusalWords is a verb's refusal as the operator reads it: the plans named as "the plan" (plainRefusal), and, where
|
||||||
|
// the words still carry an identifier the operator's channel withholds, one sentence of the answer's own.
|
||||||
|
func refusalWords(words string, act conditions.Action) string {
|
||||||
|
return plainRefusalOr(words, refusalFallback(act))
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusalFallback is what an answer's refusal says when its own words cannot be said: one sentence for each answer
|
||||||
|
// to a plan's ask, never another answer's.
|
||||||
|
func refusalFallback(act conditions.Action) string {
|
||||||
|
switch {
|
||||||
|
case act.Arguments["retry"] != "":
|
||||||
|
return "a retry would not run now; the details say why"
|
||||||
|
case act.Arguments["go"] != "":
|
||||||
|
return "the delivery could not be started now; the details say why"
|
||||||
|
case act.Arguments["stop"] != "":
|
||||||
|
return "the delivery could not be stopped now; the details say why"
|
||||||
|
}
|
||||||
|
return "the controller refused it; the details say why"
|
||||||
|
}
|
||||||
|
|
||||||
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
// handActWarrant is the verb an act the operator chose on a warrant is recorded under: a person's decision,
|
||||||
// never a repair (handActVerbs).
|
// never a repair (handActVerbs).
|
||||||
const handActWarrant = "warrant"
|
const handActWarrant = "warrant"
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
|
||||||
|
// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
|
||||||
|
func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
|
||||||
|
twoTiers(t, open)
|
||||||
|
before := time.Now().UTC().Add(-time.Hour)
|
||||||
|
ended := before.Add(time.Minute)
|
||||||
|
failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||||
|
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Note: "a failed to build in tier 0",
|
||||||
|
Delivery: &inventory.PlanDelivery{},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||||
|
Why: link.KilledByHand}},
|
||||||
|
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, err := nats.Connect(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
// The controller's seat answers `plans` as it does: by running the verb.
|
||||||
|
sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
|
||||||
|
var args map[string]string
|
||||||
|
_ = json.Unmarshal(m.Data, &args)
|
||||||
|
answer := link.Answer{Result: json.RawMessage(`"ok"`)}
|
||||||
|
if len(args) != 1 || args["retry"] == "" {
|
||||||
|
answer = link.Answer{Error: "the retry was asked with more than the plan"}
|
||||||
|
} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
|
||||||
|
answer = link.Answer{Error: err.Error()}
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(answer)
|
||||||
|
_ = m.Respond(body)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
|
||||||
|
// What S23 raises for it, read from the store as the watchdog reads it.
|
||||||
|
raised := func(context.Context) ([]conditions.Condition, error) {
|
||||||
|
facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []conditions.Condition
|
||||||
|
for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
|
||||||
|
out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||||
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
store := memAskedStore{}
|
||||||
|
var sent []published
|
||||||
|
var acts []link.HandAct
|
||||||
|
a := &asker{open: raised, store: store,
|
||||||
|
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
||||||
|
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
||||||
|
sent = append(sent, published{subject, id, body})
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
call: callAction(conn),
|
||||||
|
record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
|
||||||
|
now: func() time.Time { return now },
|
||||||
|
logf: t.Logf}
|
||||||
|
cs, _ := raised(ctx)
|
||||||
|
if len(cs) != 1 || len(cs[0].Actions) != 1 {
|
||||||
|
t.Fatalf("the failed plan raised %+v", cs)
|
||||||
|
}
|
||||||
|
if err := a.reconcile(ctx); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var w asks.Warrant
|
||||||
|
for _, r := range store {
|
||||||
|
for _, o := range r.Ask.Options {
|
||||||
|
if o.Label == "Retry" {
|
||||||
|
w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
|
||||||
|
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
||||||
|
AskDigest: r.Ask.Digest(),
|
||||||
|
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if w.Ask == "" {
|
||||||
|
t.Fatalf("no ask offers Retry: %+v", store)
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(w)
|
||||||
|
if err := a.Decided(ctx, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := p.Modules["a"]
|
||||||
|
if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" {
|
||||||
|
t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("the build was asked %d times: %v", len(*asked), *asked)
|
||||||
|
}
|
||||||
|
if got := store[w.Ask]; got.Acted != "done" {
|
||||||
|
t.Fatalf("the ask ended %q", got.Acted)
|
||||||
|
}
|
||||||
|
if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
|
||||||
|
t.Fatalf("the warrant was recorded as %+v", acts)
|
||||||
|
}
|
||||||
|
if cs, _ = raised(ctx); len(cs) != 0 {
|
||||||
|
t.Fatalf("the plan-failed condition did not end: %+v", cs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -77,6 +77,15 @@ type askerRig struct {
|
|||||||
silenced []string
|
silenced []string
|
||||||
acts []link.HandAct
|
acts []link.HandAct
|
||||||
now time.Time
|
now time.Time
|
||||||
|
// refuses, when set, is what every verb the asker calls answers; told is every notice the operator was told.
|
||||||
|
refuses error
|
||||||
|
told []told
|
||||||
|
}
|
||||||
|
|
||||||
|
// told is one plain notice the asker told the operator.
|
||||||
|
type told struct {
|
||||||
|
headline string
|
||||||
|
lines []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newAskerRig(t *testing.T) *askerRig {
|
func newAskerRig(t *testing.T) *askerRig {
|
||||||
@@ -102,6 +111,10 @@ func newAskerRig(t *testing.T) *askerRig {
|
|||||||
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
call: func(_ context.Context, a conditions.Action, args map[string]string) error {
|
||||||
raw, _ := json.Marshal(args)
|
raw, _ := json.Marshal(args)
|
||||||
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
r.called = append(r.called, a.Verb+"@"+a.Machine+" "+string(raw))
|
||||||
|
return r.refuses
|
||||||
|
},
|
||||||
|
tell: func(_ context.Context, headline string, lines []string) error {
|
||||||
|
r.told = append(r.told, told{headline, lines})
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
record: func(_ context.Context, act link.HandAct) error { r.acts = append(r.acts, act); return nil },
|
||||||
@@ -1013,3 +1026,215 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
|||||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
|
||||||
|
func failedPlanCondition(plan string) conditions.Condition {
|
||||||
|
p := aFailedPlan(plan, nil)
|
||||||
|
f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||||
|
o := watchPlansFailed(f)[0]
|
||||||
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||||
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
|
||||||
|
func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
c := failedPlanCondition("plan-1792433168676870614")
|
||||||
|
r.open = []conditions.Condition{c}
|
||||||
|
if err := r.a.reconcile(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sent := r.asksSent(t)
|
||||||
|
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
||||||
|
t.Fatalf("asked %+v", sent)
|
||||||
|
}
|
||||||
|
o := sent[0].Options[0]
|
||||||
|
if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
|
||||||
|
t.Errorf("the option is %+v", o)
|
||||||
|
}
|
||||||
|
if err := sent[0].Check(r.now); err != nil {
|
||||||
|
t.Errorf("the ask is refused as made: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(sent[0].Explanation, "mesh MCP server") {
|
||||||
|
t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
|
||||||
|
// the same warrant heard again does nothing.
|
||||||
|
func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
|
||||||
|
plan := "plan-1792433168676870614"
|
||||||
|
r := newAskerRig(t)
|
||||||
|
c := failedPlanCondition(plan)
|
||||||
|
r.open = []conditions.Condition{c}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
w := r.warrantFor(t, c.Key, "Retry")
|
||||||
|
answerWith(t, r, w)
|
||||||
|
answerWith(t, r, w)
|
||||||
|
if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
|
||||||
|
t.Fatalf("called %v, want exactly %s", r.called, want)
|
||||||
|
}
|
||||||
|
if len(r.acts) != 1 {
|
||||||
|
t.Fatalf("hand-acts %+v", r.acts)
|
||||||
|
}
|
||||||
|
act := r.acts[0]
|
||||||
|
if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
|
||||||
|
act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
|
||||||
|
t.Errorf("the hand-act is %+v", act)
|
||||||
|
}
|
||||||
|
if got := r.store[w.Ask]; got.Acted != "done" {
|
||||||
|
t.Errorf("kept %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is
|
||||||
|
// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before.
|
||||||
|
func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) {
|
||||||
|
plan := "plan-1792433168676870614"
|
||||||
|
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
||||||
|
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
||||||
|
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
c conditions.Condition
|
||||||
|
label string
|
||||||
|
}{
|
||||||
|
{"retry", failedPlanCondition(plan), "Retry"},
|
||||||
|
{"stop", waiting, "Stop"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"}
|
||||||
|
r.open = []conditions.Condition{tc.c}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
w := r.warrantFor(t, tc.c.Key, tc.label)
|
||||||
|
answerWith(t, r, w)
|
||||||
|
answerWith(t, r, w) // heard again: told once
|
||||||
|
if len(r.told) != 1 {
|
||||||
|
t.Fatalf("told %+v", r.told)
|
||||||
|
}
|
||||||
|
got := r.told[0]
|
||||||
|
if got.headline != tc.c.Headline {
|
||||||
|
t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline)
|
||||||
|
}
|
||||||
|
said := strings.Join(got.lines, " ")
|
||||||
|
if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) {
|
||||||
|
t.Errorf("the lines do not carry the answer and the refusal's words: %q", said)
|
||||||
|
}
|
||||||
|
if why, ok := conditions.Plain(said); !ok {
|
||||||
|
t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said)
|
||||||
|
}
|
||||||
|
if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") {
|
||||||
|
t.Errorf("the failure is not recorded as before: %+v", r.acts)
|
||||||
|
}
|
||||||
|
if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") {
|
||||||
|
t.Errorf("kept %+v", kept)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here.
|
||||||
|
func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
c := failedPlanCondition("plan-1792433168676870614")
|
||||||
|
r.open = []conditions.Condition{c}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||||
|
if len(r.told) != 0 {
|
||||||
|
t.Fatalf("a retry that was done told %+v", r.told)
|
||||||
|
}
|
||||||
|
|
||||||
|
r = newAskerRig(t)
|
||||||
|
r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"}
|
||||||
|
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
||||||
|
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
||||||
|
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
||||||
|
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
||||||
|
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
||||||
|
r.open = []conditions.Condition{module}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
answerWith(t, r, r.warrantFor(t, module.Key, "Restart"))
|
||||||
|
if len(r.told) != 0 || len(r.acts) != 1 {
|
||||||
|
t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was
|
||||||
|
// changed"; and the sentence a refusal falls back to fits the answer given.
|
||||||
|
func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) {
|
||||||
|
plan := "plan-1792433168676870614"
|
||||||
|
r := newAskerRig(t)
|
||||||
|
r.refuses = errors.New("mesh-controller did not answer plans within 30s")
|
||||||
|
c := failedPlanCondition(plan)
|
||||||
|
r.open = []conditions.Condition{c}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||||
|
if len(r.told) != 1 {
|
||||||
|
t.Fatalf("told %+v", r.told)
|
||||||
|
}
|
||||||
|
said := strings.Join(r.told[0].lines, " ")
|
||||||
|
if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") {
|
||||||
|
t.Errorf("a call that may have run is told as %q", said)
|
||||||
|
}
|
||||||
|
if why, ok := conditions.Plain(said); !ok {
|
||||||
|
t.Errorf("not plain: %s: %q", why, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) {
|
||||||
|
plan := "plan-1792433168676870614"
|
||||||
|
for _, tc := range []struct {
|
||||||
|
act conditions.Action
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{retryAction(plan), "a retry would not run now"},
|
||||||
|
{conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"},
|
||||||
|
{conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"},
|
||||||
|
} {
|
||||||
|
// Words the operator's channel withholds: a hash and a dotted name.
|
||||||
|
got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act)
|
||||||
|
if !strings.Contains(got, tc.want) {
|
||||||
|
t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want)
|
||||||
|
}
|
||||||
|
if _, ok := conditions.Plain(got); !ok {
|
||||||
|
t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the
|
||||||
|
// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not.
|
||||||
|
func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) {
|
||||||
|
r := newAskerRig(t)
|
||||||
|
c := failedPlanCondition("plan-1792433168676870614")
|
||||||
|
c.Raised = r.now.Add(-time.Minute)
|
||||||
|
r.open = []conditions.Condition{c}
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||||
|
// Still open right after the answer: not asked again at once.
|
||||||
|
r.now = r.now.Add(time.Minute)
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
if n := len(r.asksSent(t)); n != 1 {
|
||||||
|
t.Fatalf("asked again while the same condition stayed open: %d", n)
|
||||||
|
}
|
||||||
|
// The retry ran, the plan left failed and the condition ended.
|
||||||
|
r.now = r.now.Add(time.Minute)
|
||||||
|
r.open = nil
|
||||||
|
_ = r.a.reconcile(context.Background())
|
||||||
|
// It fails again, ten minutes after the answer: a condition raised after the answered ask ended.
|
||||||
|
r.now = r.now.Add(8 * time.Minute)
|
||||||
|
again := failedPlanCondition("plan-1792433168676870614")
|
||||||
|
again.Raised = r.now
|
||||||
|
r.open = []conditions.Condition{again}
|
||||||
|
if err := r.a.reconcile(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n := len(r.asksSent(t)); n != 2 {
|
||||||
|
t.Fatalf("the second failure was asked %d times in all, want 2", n)
|
||||||
|
}
|
||||||
|
if len(r.called) != 1 {
|
||||||
|
t.Fatalf("something retried without an answer: %v", r.called)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -135,6 +135,12 @@ func (b busAsked) All(ctx context.Context) ([]asked, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// verbRefused is a verb's own refusal of an act: the seat answered, and said no. Any other error of callAction (a bus
|
||||||
|
// that did not answer, a grant that is refused) is not one: the verb may have run.
|
||||||
|
type verbRefused struct{ verb, words string }
|
||||||
|
|
||||||
|
func (e verbRefused) Error() string { return e.verb + " refused: " + e.words }
|
||||||
|
|
||||||
// callAction performs an action's verb as the controller, through the grant that names it.
|
// callAction performs an action's verb as the controller, through the grant that names it.
|
||||||
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||||
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
return func(ctx context.Context, a conditions.Action, args map[string]string) error {
|
||||||
@@ -168,7 +174,23 @@ func callAction(conn *nats.Conn) func(ctx context.Context, a conditions.Action,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if answer.Error != "" {
|
if answer.Error != "" {
|
||||||
return fmt.Errorf("%s refused: %s", a.Verb, answer.Error)
|
return verbRefused{verb: a.Verb, words: answer.Error}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tellThrough tells the operator one plain notice through the operator channel's `tell`, over the controller's own
|
||||||
|
// connection and grant (novox/hq ADR 0380); a holder's refusal is an error naming it.
|
||||||
|
func tellThrough(conn *nats.Conn) func(ctx context.Context, headline string, lines []string) error {
|
||||||
|
return func(ctx context.Context, headline string, lines []string) error {
|
||||||
|
answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell",
|
||||||
|
map[string]any{"headline": headline, "lines": lines}, tellWithin)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if answer.Error != "" {
|
||||||
|
return fmt.Errorf("operator-channel.tell refused: %s", answer.Error)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -201,7 +223,7 @@ func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.
|
|||||||
}
|
}
|
||||||
|
|
||||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
||||||
func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
func asksRecords(ctx context.Context, inv catalogueReader) (string, error) {
|
||||||
declared, err := inv.Catalogue(ctx)
|
declared, err := inv.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -576,6 +598,7 @@ func servingAsker(open *stores, conn *nats.Conn, js jetstream.JetStream, keeper
|
|||||||
_, err := link.RecordHandAct(ctx, conn, act)
|
_, err := link.RecordHandAct(ctx, conn, act)
|
||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
|
tell: tellThrough(conn),
|
||||||
routerRecord: routerRecordOf(conn, open.inventory),
|
routerRecord: routerRecordOf(conn, open.inventory),
|
||||||
routerHere: routerHereIn(open.inventory),
|
routerHere: routerHereIn(open.inventory),
|
||||||
grantHeld: grantHeldIn(open, conn),
|
grantHeld: grantHeldIn(open, conn),
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/overlay"
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -146,8 +147,18 @@ func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.
|
|||||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
register(t, open, aKeeper())
|
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
|
||||||
pushedBy(t, open, "laptop")
|
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
|
||||||
|
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,7 +46,21 @@ func pushedBy(t *testing.T, open *stores, node string) string {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
digest := digestOf(body)
|
digest := digestOf(body)
|
||||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
|
||||||
|
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
|
||||||
|
builds := declared.Builds
|
||||||
|
if builds == nil {
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(plan.Modules))
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
names = append(names, m.Module)
|
||||||
|
}
|
||||||
|
builds = carriedBuilds(names, nil, current, nil)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return digest
|
return digest
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||||
|
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
||||||
|
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
||||||
|
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
||||||
|
|
||||||
|
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
||||||
|
// is away once presence says so). A machine
|
||||||
|
// never heard from has not joined and is not away.
|
||||||
|
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
for _, m := range machines {
|
||||||
|
if m.silent(now) {
|
||||||
|
out[m.name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
||||||
|
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
||||||
|
// before: a machine is not passed on a reading that was not made.
|
||||||
|
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
machines := make([]machineFacts, len(nodes))
|
||||||
|
for i, n := range nodes {
|
||||||
|
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
||||||
|
}
|
||||||
|
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return awayOf(machines, now), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
||||||
|
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
||||||
|
for _, n := range running {
|
||||||
|
if away[n] && !slices.Contains(p.Away, n) {
|
||||||
|
p.Away = append(p.Away, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(p.Away)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
||||||
|
// those it did not pass as away.
|
||||||
|
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
||||||
|
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
||||||
|
if !slices.Contains(p.Away, n) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, sent := s.Rest[n]
|
||||||
|
return !sent
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
||||||
|
func awaySaying(p inventory.Plan) string {
|
||||||
|
var said []string
|
||||||
|
for _, n := range p.Away {
|
||||||
|
said = append(said, n+" was away: sent when it is heard again")
|
||||||
|
}
|
||||||
|
return strings.Join(said, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
||||||
|
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
||||||
|
away, err := judgeAway(ctx, open, now)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
||||||
|
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
||||||
|
}
|
||||||
|
return away
|
||||||
|
}
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it
|
||||||
|
// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard
|
||||||
|
// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's
|
||||||
|
// builds, so the send it gets when it is heard again carries them (AC3).
|
||||||
|
func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
// laptop is away as every tier is sent; anchor is heard.
|
||||||
|
wasAway := judgeAway
|
||||||
|
judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) {
|
||||||
|
return map[string]bool{"laptop": true}, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { judgeAway = wasAway })
|
||||||
|
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacy, err := js.Conn().JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||||
|
t.Cleanup(func() { checkEvents = nil })
|
||||||
|
sentHeard := make(chan *nats.Msg, 32)
|
||||||
|
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
|
||||||
|
wasSaved := inventory.PlanSaved
|
||||||
|
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||||
|
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||||
|
|
||||||
|
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||||
|
var mu sync.Mutex
|
||||||
|
var asks []string
|
||||||
|
var sends [][]string
|
||||||
|
wasAsk := askABuild
|
||||||
|
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
||||||
|
asks = append(asks, source.Repository+"#"+id)
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = wasAsk })
|
||||||
|
digests := 0
|
||||||
|
wasSend := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
sends = append(sends, append([]string(nil), names...))
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
builds := map[string]string{}
|
||||||
|
for _, m := range []string{"a", "b"} {
|
||||||
|
builds[m] = current[m].Commit
|
||||||
|
}
|
||||||
|
for _, node := range names {
|
||||||
|
digests++
|
||||||
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = wasSend })
|
||||||
|
|
||||||
|
register := func(m, commit string, asked time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||||
|
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||||
|
Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record := func(id, m, commit string, at time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||||
|
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
for _, m := range []string{"a", "b"} {
|
||||||
|
record("build-"+m+"-1", m, "c1", old)
|
||||||
|
register(m, "c1", old)
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||||
|
register("a", "c2", time.Now().Add(-time.Minute))
|
||||||
|
|
||||||
|
// A machine's report on what it was sent, heard as the controller hears it.
|
||||||
|
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||||
|
seq := int64(0)
|
||||||
|
reports := func(node string) {
|
||||||
|
t.Helper()
|
||||||
|
last, found, err := inv.SentTo(ctx, node)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||||
|
}
|
||||||
|
seq++
|
||||||
|
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||||
|
"report_sequence": seq})
|
||||||
|
var report link.Report
|
||||||
|
if err := json.Unmarshal(body, &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := listener.Heard(ctx, report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plan := func() inventory.Plan {
|
||||||
|
t.Helper()
|
||||||
|
p, err := inv.PlanByID(ctx, "plan-1160")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
built := time.Now().UTC()
|
||||||
|
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||||
|
"b": {},
|
||||||
|
}}
|
||||||
|
first, stopFirst := context.WithCancel(ctx)
|
||||||
|
defer stopFirst()
|
||||||
|
waitFirst := startRolloutEvents(first, open, js)
|
||||||
|
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
||||||
|
t.Cleanup(func() { stopFirst(); waitFirst() })
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "tier 0 sent to anchor alone", func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor"
|
||||||
|
})
|
||||||
|
reports("anchor")
|
||||||
|
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
||||||
|
var moved string
|
||||||
|
eventually(t, "tier 1 reached on anchor's report alone", func() bool {
|
||||||
|
if q := plan(); q.Tier == 1 {
|
||||||
|
moved = q.Note
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
|
||||||
|
t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
|
||||||
|
}
|
||||||
|
// The ask for tier 1's build is the next step, and records its build id on the module.
|
||||||
|
eventually(t, "tier 1 asked", func() bool { m := plan().Modules["b"]; return m != nil && m.Build != "" })
|
||||||
|
b := plan().Modules["b"]
|
||||||
|
askedAt, _ := link.BuildAskedAt(b.Build)
|
||||||
|
record(b.Build, "b", "c2", askedAt)
|
||||||
|
register("b", "c2", askedAt)
|
||||||
|
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
||||||
|
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
||||||
|
})
|
||||||
|
// 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again.
|
||||||
|
was, known, err := inv.SentBuilds(ctx, "laptop")
|
||||||
|
if err != nil || !known {
|
||||||
|
t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known)
|
||||||
|
}
|
||||||
|
current, err := inv.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if was["a"] == current["a"].Commit || was["a"] != "c1" {
|
||||||
|
t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit)
|
||||||
|
}
|
||||||
|
if got := plan().Away; strings.Join(got, ",") != "laptop" {
|
||||||
|
t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||||
|
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
||||||
|
|
||||||
|
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
||||||
|
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
machines := []machineFacts{
|
||||||
|
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
||||||
|
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
||||||
|
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
||||||
|
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
||||||
|
{name: "new"},
|
||||||
|
}
|
||||||
|
got := awayOf(machines, now)
|
||||||
|
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
||||||
|
"new": false} {
|
||||||
|
if got[node] != want {
|
||||||
|
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
||||||
|
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
||||||
|
p := &inventory.Plan{}
|
||||||
|
away := map[string]bool{"g14": true, "ace": true}
|
||||||
|
rememberAway(p, []string{"anchor", "g14"}, away)
|
||||||
|
rememberAway(p, []string{"g14", "ace"}, away)
|
||||||
|
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
||||||
|
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1161/AC2: the plan says it, where its other passed machines are said.
|
||||||
|
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
||||||
|
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
||||||
|
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
||||||
|
t.Fatalf("1161/AC2: the plan says %q", say)
|
||||||
|
}
|
||||||
|
if got := awaySaying(inventory.Plan{}); got != "" {
|
||||||
|
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
||||||
|
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
||||||
|
p := inventory.Plan{Away: []string{"g14"}}
|
||||||
|
none := &inventory.PlanModule{}
|
||||||
|
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
||||||
|
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
||||||
|
}
|
||||||
|
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
||||||
|
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
||||||
|
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -176,6 +176,11 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
|||||||
repository, m.Base, m.Commit, already)
|
repository, m.Base, m.Commit, already)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if already == 0 {
|
||||||
|
if done, err := f.firstBuilds(ctx, m, entries, read, now); err != nil || done {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
fmt.Print(nothingReadsIt(repository, m))
|
fmt.Print(nothingReadsIt(repository, m))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -362,7 +367,7 @@ func buildsEvery(p inventory.Plan, modules []inventory.Entry) bool {
|
|||||||
// ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word.
|
// ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word.
|
||||||
func ownPath(touches []inventory.Entry) bool {
|
func ownPath(touches []inventory.Entry) bool {
|
||||||
for _, e := range touches {
|
for _, e := range touches {
|
||||||
if _, own := onTheControllersPath[e.Manifest.Module]; own {
|
if onTheOwnPath(e.Manifest.Module, e.Manifest) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -742,6 +747,9 @@ func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inven
|
|||||||
type keptMerge struct {
|
type keptMerge struct {
|
||||||
link.SourceMoved
|
link.SourceMoved
|
||||||
Moves []string `json:"moves,omitempty"`
|
Moves []string `json:"moves,omitempty"`
|
||||||
|
// FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first
|
||||||
|
// build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146).
|
||||||
|
FirstBuild bool `json:"first_build,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||||
@@ -937,7 +945,11 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
|||||||
var read map[string][]inventory.ReadRepository
|
var read map[string][]inventory.ReadRepository
|
||||||
var edges []inventory.Edge
|
var edges []inventory.Edge
|
||||||
judged := false
|
judged := false
|
||||||
openWalks, err := openWalksOf(ctx, inv, started)
|
onPath, err := ownPathNow(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
openWalks, err := openWalksOf(ctx, inv, started, onPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -970,7 +982,7 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
|||||||
var fold, keep []inventory.Plan
|
var fold, keep []inventory.Plan
|
||||||
if !b.OwnPath() {
|
if !b.OwnPath() {
|
||||||
for _, w := range waiting {
|
for _, w := range waiting {
|
||||||
ww, err := openWalkOf(ctx, inv, w)
|
ww, err := openWalkOf(ctx, inv, w, onPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1005,7 +1017,7 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
|||||||
if !cut.Open() {
|
if !cut.Open() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
w, err := openWalkOf(ctx, inv, cut)
|
w, err := openWalkOf(ctx, inv, cut, onPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1383,8 +1395,9 @@ func classOfModules(modules []string, entries []inventory.Entry) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
onPath := ownPathOf(entries)
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
if _, own := onPath[m]; own || resolvers[m] {
|
||||||
return inventory.ClassCore
|
return inventory.ClassCore
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -152,6 +153,7 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
|||||||
Path: result.Path,
|
Path: result.Path,
|
||||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
||||||
SourceFingerprint: result.SourceFingerprint,
|
SourceFingerprint: result.SourceFingerprint,
|
||||||
|
SourceInputs: result.SourceInputs,
|
||||||
// And its reuse key (novox/hq ADR 0350 §1), which the chain names (ADR 0408 §1).
|
// And its reuse key (novox/hq ADR 0350 §1), which the chain names (ADR 0408 §1).
|
||||||
ReuseKey: result.ReuseKey,
|
ReuseKey: result.ReuseKey,
|
||||||
}
|
}
|
||||||
@@ -236,6 +238,14 @@ func buildsCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, b := range builds {
|
for _, b := range builds {
|
||||||
|
printBuild(os.Stdout, b)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// printBuild says one build: what, its outcome, where and when, what it made and published, its reuse key, and its
|
||||||
|
// source fingerprint (novox/hq issue 1204: the one tool that reads what the builder recorded it was made from).
|
||||||
|
func printBuild(w io.Writer, b inventory.Build) {
|
||||||
what := b.Module
|
what := b.Module
|
||||||
if what == "" {
|
if what == "" {
|
||||||
// It failed before knowing what it was building, which is most of the interesting
|
// It failed before knowing what it was building, which is most of the interesting
|
||||||
@@ -246,35 +256,36 @@ func buildsCommand(ctx context.Context, args []string) error {
|
|||||||
if !b.Worked() {
|
if !b.Worked() {
|
||||||
outcome = "failed"
|
outcome = "failed"
|
||||||
}
|
}
|
||||||
fmt.Printf("%-18s %-14s %-10s %s %s\n",
|
fmt.Fprintf(w, "%-18s %-14s %-10s %s %s\n",
|
||||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||||
fmt.Printf(" %s", b.Repository)
|
fmt.Fprintf(w, " %s", b.Repository)
|
||||||
if b.Ref != "" {
|
if b.Ref != "" {
|
||||||
fmt.Printf(" at %s", b.Ref)
|
fmt.Fprintf(w, " at %s", b.Ref)
|
||||||
}
|
}
|
||||||
fmt.Println()
|
fmt.Fprintln(w)
|
||||||
for _, made := range b.Made {
|
for _, made := range b.Made {
|
||||||
fmt.Printf(" %-10s %s\n", made.Kind, made.Reference)
|
fmt.Fprintf(w, " %-10s %s\n", made.Kind, made.Reference)
|
||||||
}
|
}
|
||||||
// What it published and under what reuse key (novox/hq ADR 0408 §1).
|
// What it published and under what reuse key (novox/hq ADR 0408 §1).
|
||||||
if p := b.Published; p != nil {
|
if p := b.Published; p != nil {
|
||||||
if p.Artifact != "" {
|
if p.Artifact != "" {
|
||||||
fmt.Printf(" published %s\n", p.Artifact)
|
fmt.Fprintf(w, " published %s\n", p.Artifact)
|
||||||
}
|
}
|
||||||
if p.Package != nil {
|
if p.Package != nil {
|
||||||
fmt.Printf(" published %s@%s\n", p.Package.Name, p.Package.Version)
|
fmt.Fprintf(w, " published %s@%s\n", p.Package.Name, p.Package.Version)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if b.ReuseKey != "" {
|
if b.ReuseKey != "" {
|
||||||
fmt.Printf(" reuse key %s\n", b.ReuseKey)
|
fmt.Fprintf(w, " reuse key %s\n", b.ReuseKey)
|
||||||
|
}
|
||||||
|
if b.SourceFingerprint != "" {
|
||||||
|
fmt.Fprintf(w, " source %s\n", b.SourceFingerprint)
|
||||||
}
|
}
|
||||||
if !b.Worked() {
|
if !b.Worked() {
|
||||||
// The builder's own first line. The whole failure is often a build log, and printing
|
// The builder's own first line. The whole failure is often a build log, and printing
|
||||||
// it here would bury every other row.
|
// it here would bury every other row.
|
||||||
fmt.Printf(" %s\n", firstLine(b.Failed))
|
fmt.Fprintf(w, " %s\n", firstLine(b.Failed))
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// builderCommand issues a build machine its own broker credential.
|
// builderCommand issues a build machine its own broker credential.
|
||||||
@@ -473,7 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
||||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -812,7 +823,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
ident, err := openIdentity(ctx)
|
ident, err := openIdentity(ctx)
|
||||||
|
|||||||
@@ -389,10 +389,31 @@ func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, o
|
|||||||
// least one required status, no administrator merging past one.
|
// least one required status, no administrator merging past one.
|
||||||
//
|
//
|
||||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
||||||
// repository and path. path is the module's directory as built; branch the trunk it is registered from.
|
// repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of
|
||||||
|
// that head (novox/hq ADR 0420 §7 for a moved module, ADR 0432 for a new one), the forge's checks still holding. path is the module's directory as built; branch the trunk it is registered from.
|
||||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
||||||
buildID, path, branch string) (int64, error) {
|
buildID, path, branch string) (int64, error) {
|
||||||
forms := newSourceForms(ctx, inv)
|
return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, buildID, path, branch)
|
||||||
|
}
|
||||||
|
|
||||||
|
// catalogueReader is the catalogue's manifests, as the router's records are found in them.
|
||||||
|
type catalogueReader interface {
|
||||||
|
Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// registrations is what a registration's decision reads of the mesh: *inventory.Inventory, or a test's stand-in.
|
||||||
|
type registrations interface {
|
||||||
|
catalogueReader
|
||||||
|
SourceOf(ctx context.Context, module string) (inventory.Source, error)
|
||||||
|
SourceIdentity(ctx context.Context, module string) (int64, error)
|
||||||
|
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
||||||
|
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
||||||
|
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||||
|
func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, module string, built inventory.Source,
|
||||||
|
buildID, path, branch string) (int64, error) {
|
||||||
was, err := inv.SourceOf(ctx, module)
|
was, err := inv.SourceOf(ctx, module)
|
||||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
||||||
if err != nil && !isNew {
|
if err != nil && !isNew {
|
||||||
@@ -404,7 +425,8 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
|||||||
}
|
}
|
||||||
refuse := func(why string) (int64, error) {
|
refuse := func(why string) (int64, error) {
|
||||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
||||||
"controller's terminal, never through a verb: a registered module is what the next push sends, and whoever "+
|
"controller's terminal, or by the build of a merge mesh-delivery announced on the operator's Approve of that "+
|
||||||
|
"head, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||||
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -438,8 +460,11 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
|||||||
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
||||||
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
||||||
// A verb alone, or a merge with no such warrant, is refused as before.
|
// A verb alone, or a merge with no such warrant, is refused as before.
|
||||||
|
// **Or a new module from a repository nothing registered is built from, on the same warrant** (novox/hq ADR 0432,
|
||||||
|
// issue 1146): the build of a merge mesh-delivery announced on the operator's Approve of that head registers it.
|
||||||
|
// The forge's protection and the repository's identity are checked below as for every registration.
|
||||||
movedWhy := ""
|
movedWhy := ""
|
||||||
if why != "" && !terminal && !isNew {
|
if why != "" && !terminal {
|
||||||
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
||||||
switch {
|
switch {
|
||||||
case err == nil:
|
case err == nil:
|
||||||
@@ -507,7 +532,10 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
|||||||
owner, name, m, facts.ID, m, id))
|
owner, name, m, facts.ID, m, id))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if movedWhy != "" {
|
if movedWhy != "" && isNew {
|
||||||
|
fmt.Printf("%s: %s is new, from %s/%s — registered at the operator's Approve: %s\n", buildID, module, owner, name,
|
||||||
|
movedWhy)
|
||||||
|
} else if movedWhy != "" {
|
||||||
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
||||||
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
||||||
}
|
}
|
||||||
@@ -516,7 +544,7 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
|||||||
|
|
||||||
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
||||||
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
||||||
func askedHereFor(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
func askedHereFor(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
||||||
built inventory.Source, path string) (bool, error) {
|
built inventory.Source, path string) (bool, error) {
|
||||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||||
if err != nil || !found || !r.AtTerminal {
|
if err != nil || !found || !r.AtTerminal {
|
||||||
@@ -539,13 +567,49 @@ func sourceWords(repository, seat string) string {
|
|||||||
return buildSource{Repository: repository, Seat: seat}.String()
|
return buildSource{Repository: repository, Seat: seat}.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// firstBuildWarrant is what the controller checked before it asks the first build of the modules a merge adds to a
|
||||||
|
// repository nothing is built from: the merge was made on the operator's Approve of its head (approvedAnnounced). It
|
||||||
|
// travels in the context of that ask alone, in this process, so no verb carries one.
|
||||||
|
type firstBuildWarrant struct {
|
||||||
|
repository string // owner/name on the git seat
|
||||||
|
paths map[string]bool
|
||||||
|
why string
|
||||||
|
}
|
||||||
|
|
||||||
|
type firstBuildWarrantKey struct{}
|
||||||
|
|
||||||
|
// withFirstBuildWarrant is ctx carrying the warrant for asking the first builds of paths in repository.
|
||||||
|
func withFirstBuildWarrant(ctx context.Context, repository string, paths []string, why string) context.Context {
|
||||||
|
w := &firstBuildWarrant{repository: repository, paths: map[string]bool{}, why: why}
|
||||||
|
for _, p := range paths {
|
||||||
|
w.paths[strings.Trim(p, "/")] = true
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, firstBuildWarrantKey{}, w)
|
||||||
|
}
|
||||||
|
|
||||||
|
func firstBuildWarrantIn(ctx context.Context) *firstBuildWarrant {
|
||||||
|
w, _ := ctx.Value(firstBuildWarrantKey{}).(*firstBuildWarrant)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// covers says the warrant is for exactly this ask: the repository on the git seat, and a module path the merge holds.
|
||||||
|
func (w *firstBuildWarrant) covers(source buildSource, path string) bool {
|
||||||
|
return w != nil && source.Seat == gitSeat && strings.EqualFold(strings.Trim(source.Repository, "/"), w.repository) &&
|
||||||
|
w.paths[strings.Trim(path, "/")]
|
||||||
|
}
|
||||||
|
|
||||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
||||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
||||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
||||||
func verbMayAsk(ctx context.Context, source buildSource, url string) error {
|
func verbMayAsk(ctx context.Context, source buildSource, url, path string) error {
|
||||||
if startedAtTheTerminal() {
|
if startedAtTheTerminal() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
// **Or the first build of a module a merge adds on the operator's Approve** (novox/hq ADR 0432, issue 1146): the
|
||||||
|
// controller's own ask, carrying the warrant it checked, for that repository and the module paths the merge holds.
|
||||||
|
if firstBuildWarrantIn(ctx).covers(source, path) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -134,18 +134,18 @@ func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T)
|
|||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
t.Setenv(verbVar, "build")
|
t.Setenv(verbVar, "build")
|
||||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git", "")
|
||||||
var policy *heldAtTheTerminal
|
var policy *heldAtTheTerminal
|
||||||
if !errors.As(err, &policy) {
|
if !errors.As(err, &policy) {
|
||||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||||
}
|
}
|
||||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
"http://forge.internal:20000/novox/mesh-catalog.git", ""); err != nil {
|
||||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||||
}
|
}
|
||||||
t.Setenv(verbVar, "")
|
t.Setenv(verbVar, "")
|
||||||
t.Setenv(link.CallerVar, "")
|
t.Setenv(link.CallerVar, "")
|
||||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "", ""); err != nil {
|
||||||
t.Fatalf("the terminal was refused: %v", err)
|
t.Fatalf("the terminal was refused: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -317,7 +317,7 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
|||||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
||||||
}
|
}
|
||||||
routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
routerAskOf = func(_ context.Context, _ catalogueReader, _, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
if id != a.merge.AskID {
|
if id != a.merge.AskID {
|
||||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
t.Errorf("%s: the router was asked for %q", c.what, id)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -32,3 +37,43 @@ func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 1204: builds says the source fingerprint the builder recorded, the one tool that reads it.
|
||||||
|
func TestBuildsSaysTheSourceFingerprint(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 19, 10, 40, 0, 0, time.UTC)
|
||||||
|
built := inventory.Build{ID: "build-1", Module: "mailu", Commit: "b4527d9091e972f28cb365623d9280045e7741a3", On: "ace",
|
||||||
|
Repository: "novox/mesh-catalog", At: at, ReuseKey: "r1:abc",
|
||||||
|
SourceFingerprint: "src1:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}
|
||||||
|
|
||||||
|
// 1204/AC1
|
||||||
|
var out bytes.Buffer
|
||||||
|
printBuild(&out, built)
|
||||||
|
lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n")
|
||||||
|
if last := lines[len(lines)-1]; last != " source "+built.SourceFingerprint {
|
||||||
|
t.Errorf("last line %q; want the source line after the reuse key:\n%s", last, out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(lines[len(lines)-2], "reuse key r1:abc") {
|
||||||
|
t.Errorf("the source line does not follow the reuse key:\n%s", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1204/AC2
|
||||||
|
without := built
|
||||||
|
without.SourceFingerprint = ""
|
||||||
|
var plain bytes.Buffer
|
||||||
|
printBuild(&plain, without)
|
||||||
|
if strings.Contains(plain.String(), "source") {
|
||||||
|
t.Errorf("a build with no fingerprint says a source line:\n%s", plain.String())
|
||||||
|
}
|
||||||
|
if want := strings.Join(lines[:len(lines)-1], "\n") + "\n"; plain.String() != want {
|
||||||
|
t.Errorf("the other lines changed:\n%s\nwant\n%s", plain.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1215/AC4: what the builder said of the inputs is what the controller keeps of the build.
|
||||||
|
func TestTheControllerKeepsTheInputsABuilderSaid(t *testing.T) {
|
||||||
|
in := &sdkdelivery.SourceInputs{Module: "mailu", Tree: "t", Toolchains: map[string]string{"code": "ts sha256:aa 01"}}
|
||||||
|
kept := buildFrom(link.BuildResult{ID: "build-1", Repository: "r", Commit: "c", SourceFingerprint: "src1:x", SourceInputs: in})
|
||||||
|
if !reflect.DeepEqual(kept.SourceInputs, in) {
|
||||||
|
t.Errorf("the controller keeps %+v, the builder said %+v", kept.SourceInputs, in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,9 +13,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
|
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
|
||||||
// bus's key-value store, written by the controller that holds the lease. The tables of the controller's store are
|
// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them.
|
||||||
// copied into them once, by the serving controller at its start, and a kind is read from its bucket only once the move
|
|
||||||
// has recorded it (inventory.ChainReads).
|
|
||||||
|
|
||||||
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
|
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
|
||||||
func chainHolds(ctx context.Context) error {
|
func chainHolds(ctx context.Context) error {
|
||||||
@@ -33,97 +31,7 @@ func openTheChain(ctx context.Context) (*chain.Chain, error) {
|
|||||||
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
|
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
|
||||||
}
|
}
|
||||||
|
|
||||||
// kindChainMoveRefused is the condition of a move that stopped without switching a kind.
|
// attachTheChain gives the inventory of the serving controller the chain on its connection.
|
||||||
const kindChainMoveRefused = "chain-move-refused"
|
|
||||||
|
|
||||||
// chainMoveRetry is how long a move that failed for want of the bus waits before it tries again.
|
|
||||||
const chainMoveRetry = 30 * time.Second
|
|
||||||
|
|
||||||
// movingTheChain copies the tables into the buckets and records it, at the serving controller's start: after the buckets
|
|
||||||
// exist and the lease is held, before anything reads a kind that has not moved. **The first try is made here, before the
|
|
||||||
// controller serves**, so what it serves is read from one place. A move that fails for want of the bus is tried again
|
|
||||||
// every half minute while the controller serves from the tables; one that is refused — a count that differs, a value
|
|
||||||
// over the bus's largest — is raised as an urgent condition, and the tables stay what is read: nothing is half-switched.
|
|
||||||
func movingTheChain(ctx context.Context, inv *inventory.Inventory) {
|
|
||||||
say := func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }
|
|
||||||
// The first try is made here, before the controller serves; the tries after it, while it serves.
|
|
||||||
if movedOrStopped(ctx, inv, say) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(chainMoveRetry):
|
|
||||||
}
|
|
||||||
if movedOrStopped(ctx, inv, say) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
// movedOrStopped makes one try at the move and says whether there is nothing more to try: moved, refused, or the controller
|
|
||||||
// ending. False for a failure of the kind that is tried again (the bus away, a write not granted yet, the lease not held).
|
|
||||||
func movedOrStopped(ctx context.Context, inv *inventory.Inventory, say func(string, ...any)) bool {
|
|
||||||
_, err := inv.MoveChain(ctx, say)
|
|
||||||
var refusal *inventory.Refusal
|
|
||||||
switch {
|
|
||||||
case err == nil:
|
|
||||||
go clearChainMove(ctx)
|
|
||||||
go askAgainSoon(ctx, inv)
|
|
||||||
return true
|
|
||||||
case errors.As(err, &refusal):
|
|
||||||
say("%v", err)
|
|
||||||
go raiseChainMove(ctx, refusal)
|
|
||||||
return true
|
|
||||||
case ctx.Err() != nil:
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
say("the move to the bus did not finish, and the tables are read until it does: %v", err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// chainMoveObservation is the urgent condition for a refused move.
|
|
||||||
func chainMoveObservation(r *inventory.Refusal) conditions.Observation {
|
|
||||||
return conditions.Observation{Scope: conditions.ScopeBus, ID: "chain." + r.Kind, Token: kindChainMoveRefused,
|
|
||||||
Kind: kindChainMoveRefused, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
|
||||||
Summary: "the controller's " + r.Kind + " were not moved to the bus: " + r.Why, Source: "chain move"}
|
|
||||||
}
|
|
||||||
|
|
||||||
// whenKept runs f once the serving controller keeps conditions: the move runs before they are kept.
|
|
||||||
func whenKept(ctx context.Context, f func(k *conditions.Keeper)) {
|
|
||||||
for conditionsFrom == nil {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
f(conditionsFrom)
|
|
||||||
}
|
|
||||||
|
|
||||||
func raiseChainMove(ctx context.Context, r *inventory.Refusal) {
|
|
||||||
whenKept(ctx, func(k *conditions.Keeper) {
|
|
||||||
if _, err := k.Observe(ctx, chainMoveObservation(r)); err != nil {
|
|
||||||
fmt.Printf("chain: the condition for the refused move could not be raised: %v\n", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func clearChainMove(ctx context.Context) {
|
|
||||||
whenKept(ctx, func(k *conditions.Keeper) {
|
|
||||||
for _, kind := range []string{inventory.KindBuilds, inventory.KindRollouts, inventory.KindAssignments} {
|
|
||||||
key := chainMoveObservation(&inventory.Refusal{Kind: kind}).Key()
|
|
||||||
if _, err := k.Clear(ctx, key, "the "+kind+" are on the bus"); err != nil {
|
|
||||||
fmt.Printf("chain: the condition for the move of the %s could not be cleared: %v\n", kind, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// attachTheChain gives the inventory of the serving controller the chain on its connection, then moves what has not moved.
|
|
||||||
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
|
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
|
||||||
bus, ok := server.Bus().(link.OverNATS)
|
bus, ok := server.Bus().(link.OverNATS)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -134,7 +42,19 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
inv.UseChain(c)
|
inv.UseChain(c)
|
||||||
movingTheChain(ctx, inv)
|
inv.RequireLayout()
|
||||||
|
// And the builds laid out under their modules, once, before they are read (issue 1121) — **before this returns**, so
|
||||||
|
// nothing the start goes on to read meets a chain not laid out yet: on 2026-10-18 the layout ran beside the start, a
|
||||||
|
// read of the start was refused, and the controller exited and was started again every five seconds. The layout of
|
||||||
|
// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
|
||||||
|
if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
|
||||||
|
fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
|
||||||
|
go layBuildsOutSoon(ctx, inv)
|
||||||
|
}
|
||||||
|
// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
|
||||||
|
go askAgainSoon(ctx, inv)
|
||||||
|
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||||
|
go copySentBuildsSoon(ctx, inv)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,3 +99,79 @@ func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
|
|||||||
askAgain(ctx, inv)
|
askAgain(ctx, inv)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
|
||||||
|
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||||
|
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||||
|
|
||||||
|
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
||||||
|
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
||||||
|
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
||||||
|
// that names a commit and empties each column it has copied.
|
||||||
|
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||||
|
failed := false
|
||||||
|
for {
|
||||||
|
_, err := inv.CopySentBuilds(ctx)
|
||||||
|
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
||||||
|
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||||
|
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
||||||
|
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||||
|
o.Summary += err.Error()
|
||||||
|
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if keeper := conditionsFrom; keeper != nil && failed {
|
||||||
|
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(chainMoveRetry):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// chainMoveRetry is how long the copy waits before it tries again.
|
||||||
|
const chainMoveRetry = 30 * time.Second
|
||||||
|
|
||||||
|
// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it
|
||||||
|
// has, no build is read, and a send that needs one is refused with the reason.
|
||||||
|
const kindBuildsNotLaidOut = "builds-not-laid-out"
|
||||||
|
|
||||||
|
// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a
|
||||||
|
// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an
|
||||||
|
// urgent condition until it has worked. It is idempotent (LayBuildsOut).
|
||||||
|
func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||||
|
failed := false
|
||||||
|
for {
|
||||||
|
err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
|
||||||
|
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut,
|
||||||
|
Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||||
|
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
||||||
|
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||||
|
o.Summary += err.Error()
|
||||||
|
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if keeper := conditionsFrom; keeper != nil && failed {
|
||||||
|
_, _ = keeper.Clear(ctx, o.Key(), "laid out")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(chainMoveRetry):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -176,12 +176,12 @@ func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T
|
|||||||
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
||||||
t.Fatalf("the assignment names %+v", on)
|
t.Fatalf("the assignment names %+v", on)
|
||||||
}
|
}
|
||||||
// A declaration sent by hand with no builds known says nothing of them.
|
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
|
||||||
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c1" {
|
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
|
||||||
t.Fatalf("a send of unknown builds changed %+v", on)
|
t.Fatalf("a send of unknown builds left %+v", on)
|
||||||
}
|
}
|
||||||
// A module sent a commit no build is kept of is named by its commit alone.
|
// A module sent a commit no build is kept of is named by its commit alone.
|
||||||
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
||||||
@@ -207,3 +207,39 @@ func runOf(on []chain.Assignment, module string) chain.Assignment {
|
|||||||
a, _ := assignmentOf(on, module)
|
a, _ := assignmentOf(on, module)
|
||||||
return a
|
return a
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
|
||||||
|
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
|
||||||
|
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
|
||||||
|
// and the table of pending assignments is read by nobody. No key is written by hand.
|
||||||
|
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
|
||||||
|
open := aCatalogueMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
asksWithPaths(t)
|
||||||
|
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||||
|
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||||
|
t.Fatalf("an assignment while its build runs was refused: %v", err)
|
||||||
|
}
|
||||||
|
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||||
|
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
|
||||||
|
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
|
||||||
|
!strings.HasPrefix(waiting.Pending.ID, "p-") {
|
||||||
|
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
|
||||||
|
}
|
||||||
|
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
|
||||||
|
t.Fatalf("a module only waiting is assigned: %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||||
|
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
|
||||||
|
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
|
||||||
|
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||||
|
t.Fatal("the build registered sensors and it was not assigned")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
|
||||||
|
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
|
||||||
|
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
|
||||||
|
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
|
||||||
|
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
|
||||||
|
open := windowed(t)
|
||||||
|
everyHeadWaits(t)
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
||||||
|
cutAt(t, open, t0.Add(91*time.Second))
|
||||||
|
ws, _ := walks(t, open)
|
||||||
|
if len(ws) != 1 {
|
||||||
|
t.Fatalf("%d walk(s) after the cut", len(ws))
|
||||||
|
}
|
||||||
|
rollout := ws[0].ID
|
||||||
|
asking, err := inv.PlanByID(ctx, rollout)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buildID := asking.Modules["notes"].Build
|
||||||
|
if buildID == "" || len(*asked) != 1 {
|
||||||
|
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
||||||
|
}
|
||||||
|
digest := "sha256:" + strings.Repeat("b", 64)
|
||||||
|
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
|
||||||
|
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
|
||||||
|
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the live path made is in the chain.
|
||||||
|
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
|
||||||
|
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
|
||||||
|
}
|
||||||
|
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
|
||||||
|
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
|
||||||
|
}
|
||||||
|
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
} else if _, ok := assignmentOf(on, "notes"); !ok {
|
||||||
|
t.Fatalf("the assignment is not read from the chain: %+v", on)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the catalog of the store holds none of them.
|
||||||
|
have := map[string]bool{}
|
||||||
|
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||||
|
have[strings.Trim(row[0], `"`)] = true
|
||||||
|
}
|
||||||
|
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
|
||||||
|
if have[table] {
|
||||||
|
t.Errorf("the table %s is in the controller's store", table)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, table := range []string{"build_gate", "assignment_generation"} {
|
||||||
|
if !have[table] {
|
||||||
|
t.Errorf("the table %s, which stays, is gone", table)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
||||||
|
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1212/AC1 and AC2: the check request names the pulls bucket of each module claiming the delivery seat, in the order
|
||||||
|
// of the claimants' names.
|
||||||
|
func TestACheckNamesThePullsBucketsOfTheSeatsClaimants(t *testing.T) {
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
entries []inventory.Entry
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")}, []string{"mesh-delivery-gate_pulls"}},
|
||||||
|
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery"), namedOnly("gitea")}, []string{"mesh-delivery_pulls"}},
|
||||||
|
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
||||||
|
"none": {[]inventory.Entry{namedOnly("gitea")}, nil},
|
||||||
|
} {
|
||||||
|
if got := pullsBucketsOf(c.entries); !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("%s: %v, want %v", name, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1212/AC1 and AC2: the check a request carries names those buckets (the wiring, not only the helper).
|
||||||
|
func TestTheCheckRequestCarriesThePullsBuckets(t *testing.T) {
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
entries []inventory.Entry
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}, []string{"mesh-delivery-gate_pulls"}},
|
||||||
|
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls"}},
|
||||||
|
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
||||||
|
} {
|
||||||
|
got := checkOf(link.PullUpdated{Owner: "novox", Repo: "a", Number: 7}, checkScope{}, c.entries, nil, nil, nil, nil)
|
||||||
|
if !reflect.DeepEqual(got.Pulls, c.want) || got.Owner != "novox" || got.Number != 7 {
|
||||||
|
t.Errorf("%s: %+v, want pulls %v", name, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"regexp"
|
"regexp"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
@@ -12,6 +14,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -140,18 +143,26 @@ func sourceOwner(repository string) string {
|
|||||||
|
|
||||||
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
|
// PullUpdated decides a pull request's merge check, and asks for it when there is something to run.
|
||||||
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
||||||
|
_, err := f.pullUpdated(ctx, p, os.Stdout)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// pullUpdated is PullUpdated saying what it did to w. It returns the id of the check it asked of the build seat, or
|
||||||
|
// "" when it said a verdict itself and asked nothing. A command that answers JSON on standard output (`delivery
|
||||||
|
// check`) gives it standard error, so its output stays the document alone (novox/hq issue 1311).
|
||||||
|
func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Writer) (string, error) {
|
||||||
inv := f.open.inventory
|
inv := f.open.inventory
|
||||||
entries, err := inv.Catalogued(ctx)
|
entries, err := inv.Catalogued(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
read, err := readForPlanning(ctx, inv)
|
read, err := readForPlanning(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
edges, err := inv.Dependencies(ctx)
|
edges, err := inv.Dependencies(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
scope := pullScope(p, entries, read, edges)
|
scope := pullScope(p, entries, read, edges)
|
||||||
// **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull
|
// **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull
|
||||||
@@ -160,12 +171,12 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA}
|
Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA}
|
||||||
if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone},
|
if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone},
|
||||||
map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil {
|
map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil {
|
||||||
return err
|
return "", err
|
||||||
} else if cyclic {
|
} else if cyclic {
|
||||||
failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA)
|
failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA)
|
||||||
fmt.Printf("%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary)
|
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary)
|
||||||
sayChecked(ctx, failing)
|
sayChecked(ctx, w, failing)
|
||||||
return nil
|
return "", nil
|
||||||
}
|
}
|
||||||
// **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403
|
// **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403
|
||||||
// clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is
|
// clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is
|
||||||
@@ -178,7 +189,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
u, err := inv.UpgradeOf(ctx, module)
|
u, err := inv.UpgradeOf(ctx, module)
|
||||||
return u, err == nil
|
return u, err == nil
|
||||||
})
|
})
|
||||||
fmt.Print(planText(plan))
|
fmt.Fprint(w, planText(plan))
|
||||||
direct := directVerdict(p, plan)
|
direct := directVerdict(p, plan)
|
||||||
if composed {
|
if composed {
|
||||||
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
||||||
@@ -186,19 +197,19 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
case !scope.gated() && !scope.Mesh:
|
case !scope.gated() && !scope.Mesh:
|
||||||
fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||||
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
|
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
|
||||||
sayChecked(ctx, direct)
|
sayChecked(ctx, w, direct)
|
||||||
return nil
|
return "", nil
|
||||||
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
|
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
|
||||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
||||||
fmt.Printf("%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
||||||
sayChecked(ctx, direct)
|
sayChecked(ctx, w, direct)
|
||||||
return nil
|
return "", nil
|
||||||
}
|
}
|
||||||
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
|
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
request.Check.Plan = &plan
|
request.Check.Plan = &plan
|
||||||
request.Check.Composed = composed
|
request.Check.Composed = composed
|
||||||
@@ -207,11 +218,11 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
seat := buildSeatHeld(ctx)
|
seat := buildSeatHeld(ctx)
|
||||||
ask, err := askOverOn(seat)
|
ask, err := askOverOn(seat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
defer ask.Close()
|
defer ask.Close()
|
||||||
if err := ask.Ask(ctx, request); err != nil {
|
if err := ask.Ask(ctx, request); err != nil {
|
||||||
return err
|
return "", err
|
||||||
}
|
}
|
||||||
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
||||||
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
||||||
@@ -225,9 +236,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error {
|
|||||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||||
scope.Width, scope.Tiers)
|
scope.Width, scope.Tiers)
|
||||||
}
|
}
|
||||||
fmt.Printf("%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
fmt.Fprintf(w, "%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
||||||
p.Commit, seat, what, request.ID)
|
p.Commit, seat, what, request.ID)
|
||||||
return nil
|
return request.ID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1).
|
// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1).
|
||||||
@@ -305,13 +316,19 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
|||||||
Held: heldBy(ctx),
|
Held: heldBy(ctx),
|
||||||
Seats: seatBases(ctx),
|
Seats: seatBases(ctx),
|
||||||
Source: sourceOnSeat(source),
|
Source: sourceOnSeat(source),
|
||||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
|
Check: checkOf(p, scope, entries, beside, builds, namedForTheCheck(ctx, p, builds), runningBy(ctx)),
|
||||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
|
||||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
|
||||||
Running: runningBy(ctx)},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkOf is the check a request carries: the pull request, what the change touches, and the buckets of the pull
|
||||||
|
// requests' keys of the modules claiming the delivery seat (novox/hq ADR 0437 §3, issue 1212).
|
||||||
|
func checkOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry, beside map[string]link.CheckedOut,
|
||||||
|
builds []link.CheckBuild, named []link.CheckedBuild, running map[string]string) *link.CheckRequest {
|
||||||
|
return &link.CheckRequest{Pulls: pullsBucketsOf(entries), Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||||
|
Trunk: p.BaseSHA, Ready: p.Ready, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents,
|
||||||
|
New: scope.New, Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: named, Running: running}
|
||||||
|
}
|
||||||
|
|
||||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||||
// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module.
|
// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module.
|
||||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||||
@@ -521,7 +538,7 @@ func checked(ctx context.Context, result link.BuildResult) {
|
|||||||
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
fmt.Printf("%s: stopped, its pull request's key moved on; the newer check will judge\n", result.ID)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sayChecked(ctx, checkedOf(result))
|
sayChecked(ctx, os.Stdout, checkedOf(result))
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
|
// checkedOf is what a check's outcome says: each layer, and an error — never a pass — for a check that
|
||||||
@@ -637,7 +654,7 @@ func prefixedAll(prefix string, items []string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
||||||
func sayChecked(ctx context.Context, c link.Checked) {
|
func sayChecked(ctx context.Context, w io.Writer, c link.Checked) {
|
||||||
repo := "none"
|
repo := "none"
|
||||||
if c.RepoCheck != nil {
|
if c.RepoCheck != nil {
|
||||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||||
@@ -646,7 +663,7 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
|||||||
if c.Trunk != "" {
|
if c.Trunk != "" {
|
||||||
judged = " merged into trunk " + c.Trunk
|
judged = " merged into trunk " + c.Trunk
|
||||||
}
|
}
|
||||||
fmt.Printf("%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
fmt.Fprintf(w, "%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||||
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||||
if checkEvents == nil {
|
if checkEvents == nil {
|
||||||
return
|
return
|
||||||
@@ -658,6 +675,16 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
|||||||
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
defer stop()
|
defer stop()
|
||||||
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
|
if err := checkEvents.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyChecked, body); err != nil {
|
||||||
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
fmt.Fprintf(w, "%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pullsBucketsOf are the buckets of the pull requests' keys a check names for the build agent: one per module
|
||||||
|
// claiming the delivery seat, in the order of their names (novox/hq ADR 0437 §3, issue 1212).
|
||||||
|
func pullsBucketsOf(entries []inventory.Entry) []string {
|
||||||
|
var out []string
|
||||||
|
for _, m := range claimantsOf(entries, catalogue.DeliverySeat) {
|
||||||
|
out = append(out, broker.BucketName(m, "pulls"))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -56,13 +56,19 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
|||||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
// Built from a commit, so that a send carries them: a holder is asked only of what the machine was sent (issue 1109).
|
||||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
built := inventory.Source{BuiltFrom: "c1"}
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
register(t, open, house)
|
if err := inv.RegisterModule(ctx, house, built); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||||
}
|
}
|
||||||
|
|||||||
+140
-28
@@ -42,13 +42,53 @@ import (
|
|||||||
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
|
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
|
||||||
// go`, `delivery stop` and `delivery walks`.
|
// go`, `delivery stop` and `delivery walks`.
|
||||||
|
|
||||||
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
|
// onTheControllersPath are the modules, by name, whose walk never waits for the delivery's owner, and what each is.
|
||||||
|
// The delivery's owner is not among them: it is whichever module claims the seat (ownPathOf, novox/hq ADR 0437 §3).
|
||||||
var onTheControllersPath = map[string]string{
|
var onTheControllersPath = map[string]string{
|
||||||
"mesh-controller": "the controller",
|
"mesh-controller": "the controller",
|
||||||
"mesh-host": "the node-engine",
|
"mesh-host": "the node-engine",
|
||||||
"node-tools": "the node tools",
|
"node-tools": "the node tools",
|
||||||
"nats": "the bus",
|
"nats": "the bus",
|
||||||
catalogue.DeliverySeat: "the delivery's owner",
|
}
|
||||||
|
|
||||||
|
// claimantsOf are the modules among the catalogued entries that claim a seat, each once, in name order.
|
||||||
|
func claimantsOf(entries []inventory.Entry, seat string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.ClaimsSeat(seat) && !slices.Contains(out, e.Manifest.Module) {
|
||||||
|
out = append(out, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownPathOf is the controller's own path among these entries: the modules by name, and the modules that claim
|
||||||
|
// the delivery seat, which is how the delivery's owner is found (novox/hq ADR 0437 §3).
|
||||||
|
func ownPathOf(entries []inventory.Entry) map[string]string {
|
||||||
|
own := make(map[string]string, len(onTheControllersPath)+1)
|
||||||
|
for m, what := range onTheControllersPath {
|
||||||
|
own[m] = what
|
||||||
|
}
|
||||||
|
for _, m := range claimantsOf(entries, catalogue.DeliverySeat) {
|
||||||
|
own[m] = "the delivery's owner"
|
||||||
|
}
|
||||||
|
return own
|
||||||
|
}
|
||||||
|
|
||||||
|
// onTheOwnPath is whether one module, with its manifest at hand, is on the controller's own path.
|
||||||
|
func onTheOwnPath(module string, m catalogue.Manifest) bool {
|
||||||
|
_, named := onTheControllersPath[module]
|
||||||
|
return named || m.ClaimsSeat(catalogue.DeliverySeat)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownPathNow is ownPathOf the catalogue as it is.
|
||||||
|
func ownPathNow(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) {
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ownPathOf(entries), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
|
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
|
||||||
@@ -69,8 +109,9 @@ func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDeliv
|
|||||||
if !deliverySeatHeld(entries) {
|
if !deliverySeatHeld(entries) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
onPath := ownPathOf(entries)
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
if _, own := onTheControllersPath[m]; own {
|
if _, own := onPath[m]; own {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -327,16 +368,17 @@ func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Pla
|
|||||||
// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds.
|
// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds.
|
||||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge,
|
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge,
|
||||||
now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) {
|
now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) {
|
||||||
own := false
|
|
||||||
for _, n := range movesOf(again) {
|
|
||||||
if _, on := onTheControllersPath[n]; on {
|
|
||||||
own = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
entries, read, edges, err := theGraph(ctx, inv)
|
entries, read, edges, err := theGraph(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return inventory.Plan{}, nil, nil, err
|
return inventory.Plan{}, nil, nil, err
|
||||||
}
|
}
|
||||||
|
own := false
|
||||||
|
onPath := ownPathOf(entries)
|
||||||
|
for _, n := range movesOf(again) {
|
||||||
|
if _, on := onPath[n]; on {
|
||||||
|
own = true
|
||||||
|
}
|
||||||
|
}
|
||||||
joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges)
|
joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return inventory.Plan{}, nil, nil, err
|
return inventory.Plan{}, nil, nil, err
|
||||||
@@ -742,6 +784,27 @@ func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[s
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// changeMember is one change as the planner is asked about it, the same for `delivery plan` and `plans --what-if`.
|
||||||
|
func changeMember(repository, base, head string, paths, removed []string, moduleDirs *[]string) orderMember {
|
||||||
|
m := orderMember{ID: repository + "@" + head, Repository: repository, Base: base, Head: head,
|
||||||
|
Paths: paths, Removed: removed}
|
||||||
|
if moduleDirs != nil {
|
||||||
|
m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliveryPlanDocument is the answer of `delivery plan` and of `plans --what-if … --data`: one change's
|
||||||
|
// change plan with its reach, the mesh's own part in it and whether the gate would run. One function, so
|
||||||
|
// the two verbs cannot differ (novox/hq ADR 0450 §2, issue 1324).
|
||||||
|
func deliveryPlanDocument(m orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
|
||||||
|
edges []inventory.Edge, policy policyOf) map[string]any {
|
||||||
|
s := pullScope(m.pull(), entries, read, edges)
|
||||||
|
plan := changePlanOf(m.Repository, m.Base, m.Head, s.Reach, entries, policy)
|
||||||
|
return map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
|
||||||
|
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()}
|
||||||
|
}
|
||||||
|
|
||||||
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
|
// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner:
|
||||||
//
|
//
|
||||||
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
|
// delivery plan --repository owner/repo --head <commit> [--base main] --paths a,b [--module-dirs …] [--removed …]
|
||||||
@@ -763,6 +826,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
||||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||||
|
membersFile := set.String("members-file", "", "a file of a group's members, as JSON: the same as --members, for a list "+
|
||||||
|
"too long for an argument (novox/hq issue 1272)")
|
||||||
group := set.String("group", "", "a delivery group's id")
|
group := set.String("group", "", "a delivery group's id")
|
||||||
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
|
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
|
||||||
"(novox/hq ADR 0406 §3)")
|
"(novox/hq ADR 0406 §3)")
|
||||||
@@ -777,6 +842,13 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var members []orderMember
|
var members []orderMember
|
||||||
|
if *membersFile != "" {
|
||||||
|
raw, err := os.ReadFile(*membersFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("--members-file: %w", err)
|
||||||
|
}
|
||||||
|
*membersJSON = string(raw)
|
||||||
|
}
|
||||||
if *membersJSON != "" {
|
if *membersJSON != "" {
|
||||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||||
return fmt.Errorf("the members are not readable: %w", err)
|
return fmt.Errorf("the members are not readable: %w", err)
|
||||||
@@ -804,22 +876,15 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if *repository == "" || *head == "" && *paths == "" {
|
if *repository == "" || *head == "" && *paths == "" {
|
||||||
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
|
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
|
||||||
}
|
}
|
||||||
m := orderMember{ID: *repository + "@" + *head, Repository: *repository, Base: *base, Head: *head,
|
m := changeMember(*repository, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs))
|
||||||
Paths: splitList(*paths), Removed: splitList(*removed)}
|
|
||||||
if d := moduleDirsOf(*moduleDirs); d != nil {
|
|
||||||
m.ModuleDirs, m.ModuleDirsSaid = *d, true
|
|
||||||
}
|
|
||||||
entries, read, edges, err := theGraph(ctx, inv)
|
entries, read, edges, err := theGraph(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
s := pullScope(m.pull(), entries, read, edges)
|
return answer(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) {
|
||||||
plan := changePlanOf(*repository, *base, *head, s.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
|
||||||
u, err := inv.UpgradeOf(ctx, module)
|
u, err := inv.UpgradeOf(ctx, module)
|
||||||
return u, err == nil
|
return u, err == nil
|
||||||
})
|
}))
|
||||||
return answer(map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents,
|
|
||||||
New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()})
|
|
||||||
case "order":
|
case "order":
|
||||||
if len(members) == 0 {
|
if len(members) == 0 {
|
||||||
return errors.New("delivery order --members <json>: a group has members")
|
return errors.New("delivery order --members <json>: a group has members")
|
||||||
@@ -837,10 +902,14 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
if *group == "" && len(members) == 1 {
|
if *group == "" && len(members) == 1 {
|
||||||
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
// One head, checked again as the forge's announcement would have it (a recheck): the controller's
|
||||||
// own path for a pull request, run because the delivery's owner asked.
|
// own path for a pull request, run because the delivery's owner asked.
|
||||||
if err := sayingOnTheBus(ctx, inv, func() error { return (following{open: open, oneShot: true}).PullUpdated(ctx, members[0].pull()) }); err != nil {
|
return recheckOneHead(members[0].ID, answer, func() (string, error) {
|
||||||
|
var asked string
|
||||||
|
err := sayingOnTheBus(ctx, inv, func() (err error) {
|
||||||
|
asked, err = (following{open: open, oneShot: true}).pullUpdated(ctx, members[0].pull(), os.Stderr)
|
||||||
return err
|
return err
|
||||||
}
|
})
|
||||||
return answer(map[string]any{"rechecked": members[0].ID})
|
return asked, err
|
||||||
|
})
|
||||||
}
|
}
|
||||||
if *group == "" || len(members) < 2 {
|
if *group == "" || len(members) < 2 {
|
||||||
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
|
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
|
||||||
@@ -935,7 +1004,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
|||||||
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
||||||
}
|
}
|
||||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||||
"own-path": sortedKeysOf(ownPathWords())})
|
"own-path": sortedKeysOf(ownPathOf(entries))})
|
||||||
}
|
}
|
||||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub)
|
return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub)
|
||||||
}
|
}
|
||||||
@@ -1006,9 +1075,6 @@ func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Ti
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ownPathWords is the controller's own path as words, for an answer.
|
|
||||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
|
||||||
|
|
||||||
// theGraph is what the planner reads.
|
// theGraph is what the planner reads.
|
||||||
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
|
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
|
||||||
[]inventory.Edge, error) {
|
[]inventory.Edge, error) {
|
||||||
@@ -1094,7 +1160,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
|
|||||||
if failing, cyclic, err := cycleVerdict(ctx, group, members, reach, edges, entries); err != nil {
|
if failing, cyclic, err := cycleVerdict(ctx, group, members, reach, edges, entries); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
} else if cyclic {
|
} else if cyclic {
|
||||||
sayChecked(ctx, failing)
|
sayChecked(ctx, os.Stderr, failing)
|
||||||
return failing.ID, nil
|
return failing.ID, nil
|
||||||
}
|
}
|
||||||
primary := groupPrimary(members, reach)
|
primary := groupPrimary(members, reach)
|
||||||
@@ -1181,8 +1247,32 @@ func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
|||||||
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
||||||
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
||||||
// the walk recorded.
|
// the walk recorded.
|
||||||
|
//
|
||||||
|
// The plan the announcement is made from is a copy that shares nothing with the walk, taken before the goroutine
|
||||||
|
// starts: the walk goes on writing its steps' maps and pointers while the announcement is marshalled and its
|
||||||
|
// touches read (novox/hq issue 1260), and it says the plan as it was when this was called.
|
||||||
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||||
go publishPlanMoved(ctx, bus, facts, p)
|
kept, err := copyOfPlan(p)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go publishPlanMoved(ctx, bus, facts, kept)
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyOfPlan is the plan with nothing shared with the original: its JSON round trip, which is all of it but what
|
||||||
|
// the plan never keeps (WalkSaid), carried over.
|
||||||
|
func copyOfPlan(p inventory.Plan) (inventory.Plan, error) {
|
||||||
|
raw, err := json.Marshal(p)
|
||||||
|
if err != nil {
|
||||||
|
return inventory.Plan{}, err
|
||||||
|
}
|
||||||
|
var kept inventory.Plan
|
||||||
|
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||||
|
return inventory.Plan{}, err
|
||||||
|
}
|
||||||
|
kept.WalkSaid = p.WalkSaid
|
||||||
|
return kept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||||
@@ -1290,3 +1380,25 @@ func overNATS(conn *nats.Conn) (link.Bus, error) {
|
|||||||
}
|
}
|
||||||
return link.OverNATS{Conn: conn, JS: js}, nil
|
return link.OverNATS{Conn: conn, JS: js}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// recheckAnswer is what `delivery check` answers for one head: the id of the check asked of the build seat (empty when
|
||||||
|
// the controller said the verdict itself and asked nothing), which mesh-delivery reads, and the head rechecked.
|
||||||
|
func recheckAnswer(asked, member string) map[string]any {
|
||||||
|
return map[string]any{"asked": asked, "rechecked": member}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recheckOneHead runs a one-head `delivery check` and answers it as data. Standard output is the answer's JSON alone,
|
||||||
|
// or the seat verb reads no answer from it (novox/hq issue 1311): while the check runs, whatever it prints to standard
|
||||||
|
// output (the controller's own path prints its prose there, as do the stores it reads) goes to standard error.
|
||||||
|
func recheckOneHead(member string, answer func(any) error, ask func() (string, error)) error {
|
||||||
|
real := os.Stdout
|
||||||
|
os.Stdout = os.Stderr
|
||||||
|
asked, err := func() (string, error) {
|
||||||
|
defer func() { os.Stdout = real }()
|
||||||
|
return ask()
|
||||||
|
}()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return answer(recheckAnswer(asked, member))
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `delivery check` with one head answers the id of the check it asked, as data (novox/hq issue 1311): the prose
|
||||||
|
// the check says goes to standard error, so standard output is the document alone and answerOf reads it.
|
||||||
|
func TestDeliveryCheckOfOneHeadAnswersTheRequestID(t *testing.T) {
|
||||||
|
argv := []string{"delivery", "check", "--members", "[]"}
|
||||||
|
doc, err := json.Marshal(recheckAnswer("check-42", "novox/x@abc"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Seen failing first: the prose the command used to print on standard output before the document.
|
||||||
|
polluted := append([]byte("novox/x#7 (abc): asked the build seat as check-42\n"), doc...)
|
||||||
|
if got := answerOf(argv, polluted, "", true); got.Answer != nil {
|
||||||
|
t.Fatalf("prose before the document should leave no answer; got %v", got.Answer)
|
||||||
|
}
|
||||||
|
got := answerOf(argv, doc, "novox/x#7 (abc): asked the build seat as check-42\n", true)
|
||||||
|
m, ok := got.Answer.(map[string]any)
|
||||||
|
if !ok || m["asked"] != "check-42" || m["rechecked"] != "novox/x@abc" {
|
||||||
|
t.Fatalf("the answer is %v; want asked check-42 and rechecked novox/x@abc", got.Answer)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got.Output, "asked the build seat") {
|
||||||
|
t.Fatalf("the prose is still said in the text: %q", got.Output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The text of the other callers does not change: sayChecked writes the same line to the writer it is given, and
|
||||||
|
// the controller's own path still gives PullUpdated standard output.
|
||||||
|
func TestSayCheckedWritesTheSameLineToItsWriter(t *testing.T) {
|
||||||
|
var w bytes.Buffer
|
||||||
|
sayChecked(context.Background(), &w, link.Checked{ID: "c1", Owner: "novox", Repo: "x", Number: 7, Commit: "0123456789abcdef",
|
||||||
|
Verdict: "pass", Summary: "fine"})
|
||||||
|
want := "c1: novox/x#7 at 01234567 checked on a machine that did not say which: gate PASS — fine; repository none\n"
|
||||||
|
if w.String() != want {
|
||||||
|
t.Fatalf("sayChecked said %q; want %q", w.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whatever the check prints to standard output while it runs (checks.go namedForTheCheck, the inventory's pointer
|
||||||
|
// repair) must not reach the answer's standard output: it is exactly one JSON document (novox/hq issue 1311).
|
||||||
|
func TestOneHeadRecheckLeavesStandardOutputTheAnswerAlone(t *testing.T) {
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
e, ew, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
realOut, realErr := os.Stdout, os.Stderr
|
||||||
|
os.Stdout, os.Stderr = w, ew
|
||||||
|
runErr := recheckOneHead("novox/x@abc", func(v any) error { return json.NewEncoder(os.Stdout).Encode(v) },
|
||||||
|
func() (string, error) {
|
||||||
|
fmt.Println(" the earlier checks' builds could not be read, so the check builds")
|
||||||
|
fmt.Fprintln(os.Stdout, "repaired a pointer")
|
||||||
|
return "check-42", nil
|
||||||
|
})
|
||||||
|
os.Stdout, os.Stderr = realOut, realErr
|
||||||
|
w.Close()
|
||||||
|
ew.Close()
|
||||||
|
if runErr != nil {
|
||||||
|
t.Fatal(runErr)
|
||||||
|
}
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
_, _ = out.ReadFrom(r)
|
||||||
|
_, _ = errOut.ReadFrom(e)
|
||||||
|
var doc map[string]any
|
||||||
|
if err := json.Unmarshal(out.Bytes(), &doc); err != nil || doc["asked"] != "check-42" || doc["rechecked"] != "novox/x@abc" {
|
||||||
|
t.Fatalf("standard output %q is not the answer alone (%v)", out.String(), err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(errOut.String(), "repaired a pointer") {
|
||||||
|
t.Fatalf("the prose should be on standard error: %q", errOut.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -55,6 +55,17 @@ type stalledLine struct {
|
|||||||
Bound string `json:"bound"`
|
Bound string `json:"bound"`
|
||||||
H2 string `json:"h2"`
|
H2 string `json:"h2"`
|
||||||
Says string `json:"says"`
|
Says string `json:"says"`
|
||||||
|
// Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty
|
||||||
|
// or anything else is a warning, as before.
|
||||||
|
Severity string `json:"severity,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise.
|
||||||
|
func (l stalledLine) severity() conditions.Severity {
|
||||||
|
if l.Severity == string(conditions.Urgent) {
|
||||||
|
return conditions.Urgent
|
||||||
|
}
|
||||||
|
return conditions.Warning
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||||
@@ -165,7 +176,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
|
|||||||
out := make([]conditions.Observation, 0, len(lines))
|
out := make([]conditions.Observation, 0, len(lines))
|
||||||
for _, l := range lines {
|
for _, l := range lines {
|
||||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
||||||
Severity: conditions.Warning,
|
Severity: l.severity(),
|
||||||
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
||||||
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
||||||
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
func severityRaisedFor(t *testing.T, line string) conditions.Severity {
|
||||||
|
t.Helper()
|
||||||
|
var l stalledLine
|
||||||
|
if err := json.Unmarshal([]byte(line), &l); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := stalledObservations([]stalledLine{l})
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("%d conditions", len(got))
|
||||||
|
}
|
||||||
|
return got[0].Severity
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnUrgentStalledLineRaisesAnUrgentCondition(t *testing.T) { // 1278/AC1
|
||||||
|
line := `{"id":"novox/mesh-delivery#5","state":"held","for":"3m0s","bound":"0s","h2":"close","says":"held","severity":"urgent"}`
|
||||||
|
if got := severityRaisedFor(t, line); got != conditions.Urgent {
|
||||||
|
t.Errorf("an urgent line raised %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStalledLineWithoutOrWithAnUnknownSeverityRaisesAWarning(t *testing.T) { // 1278/AC2
|
||||||
|
for _, extra := range []string{``, `,"severity":""`, `,"severity":"critical"`, `,"severity":"warning"`} {
|
||||||
|
line := `{"id":"novox/a#1","state":"unanswered","for":"3m0s","bound":"2m0s","h2":"close","says":"x"` + extra + `}`
|
||||||
|
if got := severityRaisedFor(t, line); got != conditions.Warning {
|
||||||
|
t.Errorf("the line %s raised %q", extra, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -120,6 +120,16 @@ var probeRegistry = []probe{
|
|||||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||||
|
// What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a
|
||||||
|
// node's presence, which the silent check takes.
|
||||||
|
{ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " +
|
||||||
|
"not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156",
|
||||||
|
Kind: kindStuck, Phase: 3, run: probeStuck},
|
||||||
|
// A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the
|
||||||
|
// operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8).
|
||||||
|
{ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " +
|
||||||
|
"minutes for mesh-delivery's announcement of the operator's Approve of its head", From: "ADR 0432, issue 1146",
|
||||||
|
Kind: kindFirstBuildUnasked, Phase: 3, run: probeFirstBuilds},
|
||||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
||||||
@@ -178,6 +188,10 @@ var probeRegistry = []probe{
|
|||||||
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
|
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
|
||||||
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
|
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
|
||||||
run: probeBackupHeld},
|
run: probeBackupHeld},
|
||||||
|
// A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it.
|
||||||
|
{ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " +
|
||||||
|
"bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1,
|
||||||
|
run: probeSendKeyRefused},
|
||||||
}
|
}
|
||||||
|
|
||||||
// probeVerdict is one probe's outcome in a run.
|
// probeVerdict is one probe's outcome in a run.
|
||||||
|
|||||||
@@ -72,8 +72,6 @@ func summarise(ds []inventory.Duration) []durationSummary {
|
|||||||
switch k.kind {
|
switch k.kind {
|
||||||
case inventory.DurationApply:
|
case inventory.DurationApply:
|
||||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||||
case inventory.DurationHeartbeatGap:
|
|
||||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
|
||||||
}
|
}
|
||||||
out = append(out, s)
|
out = append(out, s)
|
||||||
}
|
}
|
||||||
@@ -138,7 +136,7 @@ func durationsCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if len(summary) == 0 {
|
if len(summary) == 0 {
|
||||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+
|
||||||
"plan-tier and build durations as it hears them\n", *days)
|
"plan-tier and build durations as it hears them\n", *days)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// aForgeAndABuildSeat gives a test what a real ask of a build needs: the forge holding the git seat, assigned to anchor, so
|
||||||
|
// the clone URL composes; the identity store; the build seat's queue on a real bus; and the environment of the serving controller, which is
|
||||||
|
// never the controller's terminal (a verb's, in effect: startedAtTheTerminal is false).
|
||||||
|
func aForgeAndABuildSeat(t *testing.T, open *stores) *broker.JetStream {
|
||||||
|
t.Helper()
|
||||||
|
js, _ := aBuildQueue(t)
|
||||||
|
// The ask signs what it sends with the controller's identity, as the serving controller does.
|
||||||
|
identity.ForTest(t)
|
||||||
|
forge, err := catalogue.ParseManifest([]byte(`{"module":"gitea","version":"1",
|
||||||
|
"claims":[{"name":"git","scope":"mesh"}],"provides":[{"name":"git","scope":"mesh"}],
|
||||||
|
"serves":{"git":{"scheme":"http","port":3000}}}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, forge)
|
||||||
|
if _, err := assign(t.Context(), open, "anchor", "gitea"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv(servedVar, "1")
|
||||||
|
return js
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1146/AC9, through the real ask (askABuild, buildOneAsked, verbMayAsk) and a real bus: the first build of a module a
|
||||||
|
// merge adds on the operator's Approve is queued on the build seat; the same ask without that warrant, and the warrant's
|
||||||
|
// ask for a path the merge does not hold, are refused by the guard of a build through a verb.
|
||||||
|
func TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued(t *testing.T) {
|
||||||
|
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
js := aForgeAndABuildSeat(t, open)
|
||||||
|
|
||||||
|
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||||
|
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||||
|
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||||
|
return a.merge, true, nil
|
||||||
|
}
|
||||||
|
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
|
return &a.ask, &a.warrant, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||||
|
|
||||||
|
// Without the warrant the guard refuses, whatever the repository holds.
|
||||||
|
_, err := askABuild(ctx, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", "main")
|
||||||
|
var policy *heldAtTheTerminal
|
||||||
|
if !errors.As(err, &policy) {
|
||||||
|
t.Fatalf("1146/AC9: a first build asked without the warrant was not refused: %v", err)
|
||||||
|
}
|
||||||
|
// And a path the merge does not hold is refused on the warrant.
|
||||||
|
warranted := withFirstBuildWarrant(ctx, "novox/mesh-delivery", []string{""}, "the Approve")
|
||||||
|
if _, err := askABuild(warranted, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", "main"); !errors.As(err, &policy) {
|
||||||
|
t.Fatalf("1146/AC9: the warrant covered a path the merge does not hold: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||||
|
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||||
|
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||||
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
queue, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(queue.Asks) != 1 {
|
||||||
|
t.Fatalf("1146/AC9: the build queue holds %+v, want the one first build", queue)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// 1146/AC9: the warrant the controller checked covers exactly the repository on the git seat and the module paths the
|
||||||
|
// merge holds; nothing else passes verbMayAsk on it.
|
||||||
|
func TestTheFirstBuildWarrantCoversOnlyItsRepositoryAndItsModulePaths(t *testing.T) {
|
||||||
|
ctx := withFirstBuildWarrant(t.Context(), "novox/mesh-delivery", []string{"", "view"}, "the Approve")
|
||||||
|
w := firstBuildWarrantIn(ctx)
|
||||||
|
for _, c := range []struct {
|
||||||
|
source buildSource
|
||||||
|
path string
|
||||||
|
covers bool
|
||||||
|
}{
|
||||||
|
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", true},
|
||||||
|
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "/view/", true},
|
||||||
|
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", false},
|
||||||
|
{buildSource{Repository: "agent/mesh-delivery", Seat: "git"}, "", false},
|
||||||
|
{buildSource{Repository: "novox/mesh-delivery"}, "", false},
|
||||||
|
{buildSource{Repository: "novox/mesh-delivery", Seat: "elsewhere"}, "", false},
|
||||||
|
} {
|
||||||
|
if got := w.covers(c.source, c.path); got != c.covers {
|
||||||
|
t.Errorf("1146/AC9: %v at %q covered = %v, want %v", c.source, c.path, got, c.covers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if firstBuildWarrantIn(t.Context()).covers(buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "") {
|
||||||
|
t.Error("1146/AC9: a context with no warrant covers an ask")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// firstBuilds asks the first build of every module a merge adds to a repository no registered module is built from, when
|
||||||
|
// the merge was announced as made on the operator's Approve of that head (novox/hq ADR 0432, issue 1146): the modules'
|
||||||
|
// directories are the ones the announcement says hold a module.json at the merge commit, as touchedBy reads them. The
|
||||||
|
// merge is kept, so the build's outcome is judged by the rule that registers it (mayRegisterFrom, approvedMove).
|
||||||
|
//
|
||||||
|
// **Complete in either order, with no timer** (ADR 0426): when mesh-delivery's word is kept first, it is found here; when
|
||||||
|
// the forge's merge is heard first, the merge is kept as a first merge waiting for the word, and keepingDeliveryMerges
|
||||||
|
// runs this again when the word is kept (resumeFirstBuilds). A word that says no (a Decline, another head) asks nothing and
|
||||||
|
// says why. Answers whether the merge was dealt with here.
|
||||||
|
func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries []inventory.Entry,
|
||||||
|
read map[string][]inventory.ReadRepository, now time.Time) (bool, error) {
|
||||||
|
inv := f.open.inventory
|
||||||
|
repository := m.Owner + "/" + m.Repo
|
||||||
|
added, why, err := firstBuildsOf(ctx, inv, m, entries, read)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, errFirstNotApproved):
|
||||||
|
fmt.Printf("%s merged into %s (%.8s) adds %s, and no build is asked: %v\n", repository, m.Base, m.Commit,
|
||||||
|
strings.Join(addedDirs(m, entries, read), ", "), err)
|
||||||
|
return true, nil
|
||||||
|
case errors.Is(err, errNotAnnounced):
|
||||||
|
// Kept, waiting for mesh-delivery's word: a restart keeps it, and the word's arrival asks the builds.
|
||||||
|
if kerr := keepFirstMerge(ctx, inv, m, now, true); kerr != nil {
|
||||||
|
return true, notNow(kerr)
|
||||||
|
}
|
||||||
|
fmt.Printf("%s merged into %s (%.8s) adds %s; it is kept, and the first build is asked when mesh-delivery "+
|
||||||
|
"announces the merge on the operator's Approve\n", repository, m.Base, m.Commit,
|
||||||
|
strings.Join(addedDirs(m, entries, read), ", "))
|
||||||
|
return true, nil
|
||||||
|
case err != nil:
|
||||||
|
return true, notNow(err)
|
||||||
|
case len(added) == 0:
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err := askFirstBuilds(ctx, inv, m, now, added, why); err != nil {
|
||||||
|
return true, notNow(err)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepFirstMerge keeps a first merge as the controller heard it, waiting (no plan answers it).
|
||||||
|
func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, waitingForWord bool) error {
|
||||||
|
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||||
|
if err != nil {
|
||||||
|
merged = now
|
||||||
|
}
|
||||||
|
event, err := json.Marshal(keptMerge{SourceMoved: m, FirstBuild: waitingForWord})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = inv.AddMerge(ctx, inventory.BatchedMerge{Repository: m.Owner + "/" + m.Repo, Branch: m.Base, Commit: m.Commit,
|
||||||
|
Merged: merged.UTC(), Heard: now, Event: event})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// askFirstBuilds keeps the merge, asks the builds, and marks the merge as no longer waiting for them.
|
||||||
|
func askFirstBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, added []string,
|
||||||
|
why string) error {
|
||||||
|
repository := m.Owner + "/" + m.Repo
|
||||||
|
if err := keepFirstMerge(ctx, inv, m, now, false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so the "+
|
||||||
|
"first build of %s is asked\n", repository, m.Base, m.Commit, why, strings.Join(added, ", "))
|
||||||
|
// The ask carries the warrant just checked, for this repository and these module paths alone: the guard of a build
|
||||||
|
// through a verb (verbMayAsk) lets exactly that through.
|
||||||
|
paths := make([]string, 0, len(added))
|
||||||
|
for _, dir := range added {
|
||||||
|
if dir == "." {
|
||||||
|
dir = ""
|
||||||
|
}
|
||||||
|
paths = append(paths, dir)
|
||||||
|
}
|
||||||
|
askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, m, buildSource{Repository: repository, Seat: gitSeat}, added)
|
||||||
|
// Answered: it no longer waits, so the word's arrival again asks nothing twice.
|
||||||
|
return inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull
|
||||||
|
// request's head: the other order of the same two events.
|
||||||
|
func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error {
|
||||||
|
actingOnMerges.Lock()
|
||||||
|
defer actingOnMerges.Unlock()
|
||||||
|
inv := f.open.inventory
|
||||||
|
waiting, err := inv.WaitingMerges(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
for _, w := range waiting {
|
||||||
|
k := announcedOf(w)
|
||||||
|
if !k.FirstBuild || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||||
|
k.HeadSHA != d.HeadSHA {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
read, err := readForPlanning(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, errFirstNotApproved):
|
||||||
|
fmt.Printf("%s/%s merged into %s (%.8s) was waiting for mesh-delivery's word, and no build is asked: %v\n",
|
||||||
|
k.Owner, k.Repo, k.Base, k.Commit, err)
|
||||||
|
if aerr := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); aerr != nil {
|
||||||
|
return notNow(aerr)
|
||||||
|
}
|
||||||
|
case err != nil:
|
||||||
|
return notNow(err)
|
||||||
|
case len(added) > 0:
|
||||||
|
if err := askFirstBuilds(ctx, inv, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// errFirstNotApproved says a merge into a repository the mesh builds nothing from is not one the operator approved.
|
||||||
|
var errFirstNotApproved = errors.New("it is no merge made on the operator's Approve of its head, so a hand `build` " +
|
||||||
|
"registers a module from a repository the mesh does not build yet")
|
||||||
|
|
||||||
|
// addedDirs are the module directories the merge adds to a repository no registered module is built from, none where
|
||||||
|
// the merge is not into the trunk or some module is built from the repository already.
|
||||||
|
func addedDirs(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []string {
|
||||||
|
if m.Base != "main" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.Provided && sameRepository(e.Source.Repository, m) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, added, _ := touchedBy(nil, entries, m, read)
|
||||||
|
return added
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstBuildsOf is the directories whose first build a merge asks, and the warrant it was made on; none when the merge
|
||||||
|
// is no first merge into a repository the mesh builds nothing from. errFirstNotApproved when it is and the word on record
|
||||||
|
// is no Approve of its head; errNotAnnounced when mesh-delivery's word is not kept yet.
|
||||||
|
func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, entries []inventory.Entry,
|
||||||
|
read map[string][]inventory.ReadRepository) ([]string, string, error) {
|
||||||
|
added := addedDirs(m, entries, read)
|
||||||
|
if len(added) == 0 {
|
||||||
|
return nil, "", nil
|
||||||
|
}
|
||||||
|
why, err := approvedAnnounced(ctx, inv, m.Owner, m.Repo, keptMerge{SourceMoved: m})
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
return added, why, nil
|
||||||
|
case errors.Is(err, errWordNotAnApprove):
|
||||||
|
return nil, "", fmt.Errorf("%w (%v)", errFirstNotApproved, err)
|
||||||
|
}
|
||||||
|
// Not announced yet, or the bus or the store could not be asked: the caller waits or tries again, and asks nothing.
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
|
||||||
|
const probeFirstBuildsID = "D18"
|
||||||
|
|
||||||
|
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
|
||||||
|
const kindFirstBuildUnasked = "first-build-unasked"
|
||||||
|
|
||||||
|
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
|
||||||
|
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
|
||||||
|
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
|
||||||
|
const firstBuildWaitBound = 10 * time.Minute
|
||||||
|
|
||||||
|
// firstBuildObservations are the conditions of the first merges waiting past the bound, one for each; a merge answered is
|
||||||
|
// no longer listed as waiting, so its condition clears with the next look.
|
||||||
|
func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, w := range waiting {
|
||||||
|
k := announcedOf(w)
|
||||||
|
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
repository := k.Owner + "/" + k.Repo
|
||||||
|
for_ := now.Sub(w.Heard).Round(time.Minute)
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge,
|
||||||
|
ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), Kind: kindFirstBuildUnasked, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the merge %.8s of %s adds modules the mesh builds nothing from yet, and has waited %s for "+
|
||||||
|
"mesh-delivery's announcement of the operator's Approve of its head: no build is asked", k.Commit, repository, for_),
|
||||||
|
Said: fmt.Sprintf("%s at %s waits for mesh-delivery's word on pull request %d (head %.8s)", repository, k.Commit,
|
||||||
|
k.Number, k.HeadSHA),
|
||||||
|
Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository),
|
||||||
|
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+
|
||||||
|
"only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+
|
||||||
|
"announcement has come in %s. The merge is kept, and the builds are asked the moment the announcement is kept.",
|
||||||
|
k.Commit, repository, for_),
|
||||||
|
Resolved: "mesh-delivery announces the merge, or a hand `build` registers the modules",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
|
||||||
|
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
waiting, err := d.open.inventory.WaitingMerges(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return firstBuildObservations(waiting, time.Now().UTC()), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"slices"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1146/AC7, whole against a real bus and the store, in both orders of the two events (novox/hq ADR 0432, issue 1146):
|
||||||
|
// mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's merge,
|
||||||
|
// heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the controller
|
||||||
|
// consumes with the router's record of the ask in its bucket, come in either order; each way the first build of the
|
||||||
|
// module the merge commit holds is asked exactly once, its outcome registers the module, and it is in the catalogue.
|
||||||
|
// Without the Approve in the record nothing is asked. Store-backed: skips without one.
|
||||||
|
func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule(t *testing.T) {
|
||||||
|
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
wordFirst bool
|
||||||
|
approved bool // the router's record holds the operator's Approve of the merged head
|
||||||
|
wantRegiter bool
|
||||||
|
}{
|
||||||
|
{what: "the forge's merge before mesh-delivery's word", approved: true, wantRegiter: true},
|
||||||
|
{what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegiter: true},
|
||||||
|
{what: "a word whose record holds no Approve", wordFirst: true},
|
||||||
|
} {
|
||||||
|
t.Run(c.what, func(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
// The real ask, guard and queue included: only its answers are kept, so the build's outcome can be matched.
|
||||||
|
aForgeAndABuildSeat(t, open)
|
||||||
|
// Written by the controller's merge path on its own goroutine, read by the test: under one lock.
|
||||||
|
var mu sync.Mutex
|
||||||
|
var asks [][3]string
|
||||||
|
var ids []string
|
||||||
|
asked := func() [][3]string { mu.Lock(); defer mu.Unlock(); return slices.Clone(asks) }
|
||||||
|
realAsk := askABuild
|
||||||
|
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
id, err := realAsk(c, source, path, ref)
|
||||||
|
if err == nil {
|
||||||
|
mu.Lock()
|
||||||
|
asks = append(asks, [3]string{source.Repository, path, ref})
|
||||||
|
ids = append(ids, id)
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = realAsk })
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||||
|
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||||
|
if c.approved {
|
||||||
|
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||||
|
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
following := following{open: open}
|
||||||
|
server := link.ConnectNats(js, nil, noListener{})
|
||||||
|
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims, kept: following.resumeFirstBuilds}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := broker.AssertServingConsumers(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
serving, stop := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||||
|
t.Cleanup(func() { stop(); <-done })
|
||||||
|
servingBus.Store(js)
|
||||||
|
t.Cleanup(func() { servingBus.Store(nil) })
|
||||||
|
|
||||||
|
word := func() {
|
||||||
|
t.Helper()
|
||||||
|
body, _ := json.Marshal(a.merge)
|
||||||
|
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
becomes(t, 15*time.Second, "the word being kept", func() bool {
|
||||||
|
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, head)
|
||||||
|
return err == nil && kept
|
||||||
|
})
|
||||||
|
time.Sleep(500 * time.Millisecond) // the keeper's hook runs as the word is kept
|
||||||
|
}
|
||||||
|
forge := func() {
|
||||||
|
t.Helper()
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||||
|
MergedAt: time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||||
|
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||||
|
if err := following.SourceMoved(ctx, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.wordFirst {
|
||||||
|
word()
|
||||||
|
forge()
|
||||||
|
} else {
|
||||||
|
forge()
|
||||||
|
if len(asked()) != 0 {
|
||||||
|
t.Fatalf("1146/AC7: the merge asked %v before mesh-delivery's word was kept", asked())
|
||||||
|
}
|
||||||
|
word()
|
||||||
|
}
|
||||||
|
if !c.wantRegiter {
|
||||||
|
if len(asked()) != 0 {
|
||||||
|
t.Fatalf("1146/AC7: a merge whose record holds no Approve asked %v", asked())
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
want := [3]string{"novox/mesh-delivery", "", "main"}
|
||||||
|
becomes(t, 10*time.Second, "the first build asked", func() bool { return len(asked()) >= 1 })
|
||||||
|
if got := asked(); len(got) != 1 || got[0] != want {
|
||||||
|
t.Fatalf("1146/AC7: the merge asked %v, want %v once", got, want)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
first := ids[0]
|
||||||
|
mu.Unlock()
|
||||||
|
build := onTrunk(first, "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-view", "version": "1"})
|
||||||
|
build.Commit = commit
|
||||||
|
if _, _, err := takeIn(ctx, inv, build); err != nil {
|
||||||
|
t.Fatalf("1146/AC7: the outcome of the first build was refused: %v", err)
|
||||||
|
}
|
||||||
|
src, err := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||||
|
if err != nil || src.Repository != "novox/mesh-delivery" {
|
||||||
|
t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err)
|
||||||
|
}
|
||||||
|
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); !known {
|
||||||
|
t.Fatal("1146/AC7: the module is not in the catalogue")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) {
|
||||||
|
t.Helper()
|
||||||
|
entries, err := inv.Catalogued(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.Module == module {
|
||||||
|
return e, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.Entry{}, false
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1146/AC6: the first merge into a repository no registered module is built from asks the first build of every module
|
||||||
|
// the merge commit holds when it was made on the operator's Approve of that head, and asks nothing without it.
|
||||||
|
func TestAFirstMergeIntoANewRepositoryAsksItsModulesBuildsOnlyOnTheApprove(t *testing.T) {
|
||||||
|
const head = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||||
|
now := time.Date(2026, 10, 19, 0, 6, 0, 0, time.UTC)
|
||||||
|
mergedAt := now.Add(-time.Hour).Format(time.RFC3339Nano)
|
||||||
|
heard := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: "0177ea7a",
|
||||||
|
Number: 1, MergedAt: mergedAt, ModuleDirsSaid: true, ModuleDirs: []string{"watchman", "view"},
|
||||||
|
Paths: []string{"view/module.json", "watchman/module.json", "README.md"}}
|
||||||
|
built := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git"}}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
change func(*anApprovedMerge, *link.SourceMoved, *[]inventory.Entry)
|
||||||
|
unkept bool
|
||||||
|
dirs []string
|
||||||
|
refused bool
|
||||||
|
waits bool
|
||||||
|
}{
|
||||||
|
{what: "the Approve of the merged head", dirs: []string{"view", "watchman"}},
|
||||||
|
{what: "a Decline", refused: true, change: func(a *anApprovedMerge, _ *link.SourceMoved, _ *[]inventory.Entry) {
|
||||||
|
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
||||||
|
}},
|
||||||
|
{what: "no word of mesh-delivery yet, however long ago the merge was made", unkept: true, waits: true},
|
||||||
|
{what: "a module already built from the repository", change: func(_ *anApprovedMerge, _ *link.SourceMoved, e *[]inventory.Entry) {
|
||||||
|
*e = append(*e, inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}})
|
||||||
|
}},
|
||||||
|
{what: "a merge into another branch", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { m.Base = "feat" }},
|
||||||
|
{what: "a merge that holds no module.json", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) {
|
||||||
|
m.ModuleDirs, m.Paths = nil, []string{"README.md"}
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(c.what, func(t *testing.T) {
|
||||||
|
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||||
|
a.merge.Number = 1
|
||||||
|
m, entries := heard, []inventory.Entry{built}
|
||||||
|
if c.change != nil {
|
||||||
|
c.change(&a, &m, &entries)
|
||||||
|
}
|
||||||
|
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||||
|
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||||
|
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||||
|
return a.merge, !c.unkept, nil
|
||||||
|
}
|
||||||
|
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
|
return &a.ask, &a.warrant, nil
|
||||||
|
}
|
||||||
|
dirs, _, err := firstBuildsOf(t.Context(), nil, m, entries, nil)
|
||||||
|
switch {
|
||||||
|
case c.waits && !errors.Is(err, errNotAnnounced):
|
||||||
|
t.Fatalf("1146/AC6: %s: want a wait for the word, got %v", c.what, err)
|
||||||
|
case c.refused && !errors.Is(err, errFirstNotApproved):
|
||||||
|
t.Fatalf("1146/AC6: %s: want the Approve missing, got %v (dirs %v)", c.what, err, dirs)
|
||||||
|
case !c.refused && !c.waits && err != nil:
|
||||||
|
t.Fatalf("1146/AC6: %s: %v", c.what, err)
|
||||||
|
case !slices.Equal(dirs, c.dirs):
|
||||||
|
t.Fatalf("1146/AC6: %s: first builds asked of %v, want %v", c.what, dirs, c.dirs)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1146/AC8: a first merge waiting past the bound for mesh-delivery's word is said, naming the repository, the commit and
|
||||||
|
// the missing word; one inside the bound, one that is no first merge, and one answered (no longer waiting) are not.
|
||||||
|
func TestAFirstMergeWaitingPastItsBoundIsSaidAndClearsWhenAnswered(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC)
|
||||||
|
waiting := func(commit string, first bool, heard time.Time) inventory.BatchedMerge {
|
||||||
|
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
|
||||||
|
Commit: commit, Number: 1, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: first})
|
||||||
|
return inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, Heard: heard, Event: event}
|
||||||
|
}
|
||||||
|
got := firstBuildObservations([]inventory.BatchedMerge{
|
||||||
|
waiting("0177ea7a00", true, now.Add(-firstBuildWaitBound-time.Minute)),
|
||||||
|
waiting("1111111100", true, now.Add(-time.Minute)),
|
||||||
|
waiting("2222222200", false, now.Add(-time.Hour)),
|
||||||
|
}, now)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("1146/AC8: %d conditions raised, want the one merge past the bound: %+v", len(got), got)
|
||||||
|
}
|
||||||
|
o := got[0]
|
||||||
|
if o.Kind != kindFirstBuildUnasked || o.Scope != conditions.ScopeMerge || o.Severity != conditions.Warning ||
|
||||||
|
!strings.Contains(o.Summary, "novox/mesh-delivery") || !strings.Contains(o.Summary, "0177ea7a") ||
|
||||||
|
!strings.Contains(o.Summary, "announcement of the operator's Approve") {
|
||||||
|
t.Fatalf("1146/AC8: the condition is %+v", o)
|
||||||
|
}
|
||||||
|
// Answered, the merge is no longer listed as waiting: nothing is observed, and the condition clears.
|
||||||
|
if cleared := firstBuildObservations(nil, now); len(cleared) != 0 {
|
||||||
|
t.Fatalf("1146/AC8: an answered merge still raises %+v", cleared)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,6 +23,7 @@ func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
|||||||
resolver := entry("dnsmasq", "modules/dnsmasq")
|
resolver := entry("dnsmasq", "modules/dnsmasq")
|
||||||
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
||||||
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
||||||
|
delivery.Manifest.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||||
webApp := entry("photos", "modules/photos")
|
webApp := entry("photos", "modules/photos")
|
||||||
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
||||||
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
||||||
@@ -44,7 +45,7 @@ func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
|||||||
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||||
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
||||||
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
||||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-delivery/module.json"}, inventory.ClassCore},
|
{"a new module on the controller's own path, by its own class", []string{"next/mesh-host/module.json"}, inventory.ClassCore},
|
||||||
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||||
} {
|
} {
|
||||||
if got := plan(c.paths...).Class; got != c.want {
|
if got := plan(c.paths...).Class; got != c.want {
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ var (
|
|||||||
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
|
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
|
||||||
// delivery budget; how many readings its gate takes is gateRuleOf's.
|
// delivery budget; how many readings its gate takes is gateRuleOf's.
|
||||||
func coreModule(module string, m catalogue.Manifest) bool {
|
func coreModule(module string, m catalogue.Manifest) bool {
|
||||||
if _, own := onTheControllersPath[module]; own {
|
if onTheOwnPath(module, m) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
for _, c := range m.Claims {
|
for _, c := range m.Claims {
|
||||||
@@ -106,7 +106,7 @@ func gateRuleOf(module string, m catalogue.Manifest) gateRule {
|
|||||||
case coreComponent(module) == lease.ComponentNodeTools:
|
case coreComponent(module) == lease.ComponentNodeTools:
|
||||||
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
|
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
|
||||||
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||||
case module == catalogue.DeliverySeat:
|
case m.ClaimsSeat(catalogue.DeliverySeat):
|
||||||
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
|
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
|
||||||
case coreModule(module, m):
|
case coreModule(module, m):
|
||||||
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
|
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
||||||
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
||||||
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
|
{catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"},
|
||||||
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
||||||
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||||
@@ -56,7 +56,12 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
|
|||||||
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
||||||
func TestTheCoreStaysTheCore(t *testing.T) {
|
func TestTheCoreStaysTheCore(t *testing.T) {
|
||||||
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
||||||
if !coreModule(m, catalogue.Manifest{Module: m}) {
|
// The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 §3, issue 1210).
|
||||||
|
claims := []catalogue.Claim(nil)
|
||||||
|
if m == catalogue.DeliverySeat {
|
||||||
|
claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||||
|
}
|
||||||
|
if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) {
|
||||||
t.Errorf("%s is no longer judged as the core for its class", m)
|
t.Errorf("%s is no longer judged as the core for its class", m)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -78,7 +83,11 @@ func aGateOn(module, node string, since time.Time) *gateOn {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (o *gateOn) read(at time.Time, h health, why string) {
|
func (o *gateOn) read(at time.Time, h health, why string) {
|
||||||
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
|
m := catalogue.Manifest{Module: o.module}
|
||||||
|
if o.module == catalogue.DeliverySeat { // today's module, which claims the seat
|
||||||
|
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||||
|
}
|
||||||
|
r := gateRuleOf(o.module, m)
|
||||||
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
||||||
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
||||||
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
||||||
|
|||||||
@@ -72,6 +72,10 @@ var handActVerbs = []handActVerb{
|
|||||||
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
|
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
|
||||||
// channel that proved who answered, performed by the controller as itself.
|
// channel that proved who answered, performed by the controller as itself.
|
||||||
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
|
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
|
||||||
|
// A seat passed by the controller to its only claimant when its holder on record left (novox/hq ADR 0449
|
||||||
|
// §4): the mesh working as decided. A person's own `seat --to` is not recorded under this cause.
|
||||||
|
{Verb: "seat", Decision: "the controller passes a seat held once to its only claimant when its recorded " +
|
||||||
|
"holder is assigned nowhere (ADR 0449 §4)", DecidedFor: []string{"seat-passed"}},
|
||||||
{Verb: "broker consumer-reset"},
|
{Verb: "broker consumer-reset"},
|
||||||
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
// Silencing the same condition twice says the condition, or what it watches, wants mending — unless
|
||||||
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
// it is the operator's answer on a notification: a decision to live with it (novox/hq ADR 0258).
|
||||||
|
|||||||
@@ -77,15 +77,11 @@ func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
|||||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||||
Took: time.Duration(i) * time.Second})
|
Took: time.Duration(i) * time.Second})
|
||||||
}
|
}
|
||||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
|
||||||
got := summarise(ds)
|
got := summarise(ds)
|
||||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||||
t.Fatalf("%+v", got)
|
t.Fatalf("%+v", got)
|
||||||
}
|
}
|
||||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
|
||||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Last heard (novox/hq ADR 0433 §3, issue 1183): one answer to when the controller last heard a machine, read from the
|
||||||
|
// watchman's presence. The alive message that used to keep last_seen fresh is gone; a report, which is rare, is the
|
||||||
|
// answer only where presence has no key for the machine.
|
||||||
|
|
||||||
|
// presenceReader reads presence.<node>'s entry for each node; a node without an entry is left out.
|
||||||
|
type presenceReader func(ctx context.Context, nodes []string) (map[string]presenceEntry, error)
|
||||||
|
|
||||||
|
// lastHeard is when a machine was last heard: now when presence says it is online, the entry's time when it says offline
|
||||||
|
// (the time it went), and the machine's last report only when presence has no entry for it. Zero is never heard.
|
||||||
|
func lastHeard(report time.Time, entry *presenceEntry, now time.Time) time.Time {
|
||||||
|
switch {
|
||||||
|
case entry == nil:
|
||||||
|
return report
|
||||||
|
case entry.Online:
|
||||||
|
return now
|
||||||
|
default:
|
||||||
|
return entry.At
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodesHeard returns the nodes with their last-heard time (LastSeen) as lastHeard says it. Unreadable presence is
|
||||||
|
// returned as an error with the nodes unchanged, so a caller says it and shows the reports.
|
||||||
|
func nodesHeard(ctx context.Context, nodes []inventory.Node, read presenceReader, now time.Time) ([]inventory.Node, error) {
|
||||||
|
names := make([]string, len(nodes))
|
||||||
|
for i, n := range nodes {
|
||||||
|
names[i] = n.Name
|
||||||
|
}
|
||||||
|
entries, err := read(ctx, names)
|
||||||
|
if err != nil {
|
||||||
|
return nodes, fmt.Errorf("what the watchman says of the machines' presence cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
out := make([]inventory.Node, len(nodes))
|
||||||
|
for i, n := range nodes {
|
||||||
|
var entry *presenceEntry
|
||||||
|
if e, ok := entries[n.Name]; ok {
|
||||||
|
entry = &e
|
||||||
|
}
|
||||||
|
n.LastSeen = lastHeard(n.LastSeen, entry, now)
|
||||||
|
out[i] = n
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardNodes is every node, read with its last-heard time from the watchman's presence over a connection of its own. A bus
|
||||||
|
// that cannot be reached is said on standard error and the reports stand, which reads a machine as quiet rather than well.
|
||||||
|
func (h *stores) heardNodes(ctx context.Context) ([]inventory.Node, error) {
|
||||||
|
nodes, err := h.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return h.withPresence(ctx, nodes), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardNode is one node, read the same way.
|
||||||
|
func (h *stores) heardNode(ctx context.Context, name string) (inventory.Node, error) {
|
||||||
|
n, err := h.inventory.NodeByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
return h.withPresence(ctx, []inventory.Node{n})[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *stores) withPresence(ctx context.Context, nodes []inventory.Node) []inventory.Node {
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
var js *broker.JetStream
|
||||||
|
if err == nil {
|
||||||
|
js, err = broker.Dial(address)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "what the watchman says of the machines' presence cannot be read: %v\n", err)
|
||||||
|
return nodes
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
|
||||||
|
return readPresence(ctx, js.Conn(), names)
|
||||||
|
}
|
||||||
|
heard, err := nodesHeard(ctx, nodes, read, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
}
|
||||||
|
return heard
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardFromAt says when a node was last heard from, in a form somebody can act on.
|
||||||
|
//
|
||||||
|
// "never" and "an hour ago" are different answers and are kept different. A node that has never spoken did not finish
|
||||||
|
// joining; a node last heard from an hour ago is running an hour-old picture of the mesh.
|
||||||
|
func heardFromAt(n inventory.Node, now time.Time) string {
|
||||||
|
if n.LastSeen.IsZero() {
|
||||||
|
return "never spoken"
|
||||||
|
}
|
||||||
|
if silent := now.Sub(n.LastSeen); silent > SilentFor {
|
||||||
|
return "out of touch " + roughly(silent)
|
||||||
|
}
|
||||||
|
return "here"
|
||||||
|
}
|
||||||
|
|
||||||
|
// notHeardLately is the nodes never heard from or not within the hour: different from failing, for a machine that says
|
||||||
|
// nothing may be new, switched off or unreachable, and none of those is a machine that tried and could not.
|
||||||
|
func notHeardLately(nodes []inventory.Node, now time.Time) []inventory.Node {
|
||||||
|
var out []inventory.Node
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.LastSeen.IsZero() || now.Sub(n.LastSeen) > time.Hour {
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1183/AC5: with the watchman's presence bucket holding a node online and no alive message ever sent, the node reads here
|
||||||
|
// in `nodes` four minutes later by the controller's clock, however old its last report; and an alive message still sent
|
||||||
|
// by a node-engine is published to the same bus without error and changes nothing.
|
||||||
|
func TestAnOnlineNodeReadsHereWithNoAliveMessage(t *testing.T) { // 1183/AC5
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if _, err := broker.RaiseBuckets(js, []broker.Bucket{
|
||||||
|
{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
put := func(node string, online bool) {
|
||||||
|
t.Helper()
|
||||||
|
key, err := sdkdelivery.PresenceNodeKey(node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := sdkdelivery.EncodePresence(sdkdelivery.Presence{Online: online})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(ctx, key, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
put("ace", true)
|
||||||
|
put("laptop", false)
|
||||||
|
// A node-engine still sending its alive message: harmless.
|
||||||
|
if err := js.Conn().Publish("mesh.control.ace.alive", []byte(`{"node":"ace"}`)); err != nil {
|
||||||
|
t.Fatalf("an alive message could not be published: %v", err)
|
||||||
|
}
|
||||||
|
if err := js.Conn().Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
later := time.Now().Add(4 * time.Minute)
|
||||||
|
nodes := []inventory.Node{{Name: "ace", LastSeen: time.Now().Add(-time.Hour)},
|
||||||
|
{Name: "laptop", LastSeen: time.Now().Add(-time.Hour)}}
|
||||||
|
read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
|
||||||
|
return readPresence(ctx, js.Conn(), names)
|
||||||
|
}
|
||||||
|
heard, err := nodesHeard(ctx, nodes, read, later)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := heardFromAt(heard[0], later); got != "here" {
|
||||||
|
t.Fatalf("a node online by presence reads %q four minutes later", got)
|
||||||
|
}
|
||||||
|
if got := heardFromAt(heard[1], later); got == "here" {
|
||||||
|
t.Fatalf("a node offline by presence reads %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// aPresenceFake is the watchman's presence bucket as a reader sees it.
|
||||||
|
func aPresenceFake(entries map[string]presenceEntry, err error) presenceReader {
|
||||||
|
return func(_ context.Context, _ []string) (map[string]presenceEntry, error) { return entries, err }
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1183/AC1: one function says when a machine was last heard: online is now, offline is the entry's time, and only where
|
||||||
|
// presence has no key, the last report.
|
||||||
|
func TestLastHeardIsPresenceThenTheReport(t *testing.T) { // 1183/AC1
|
||||||
|
now := time.Now()
|
||||||
|
report := now.Add(-time.Hour)
|
||||||
|
if got := lastHeard(report, nil, now); !got.Equal(report) {
|
||||||
|
t.Fatalf("no presence key: last heard is %v, want the report %v", got, report)
|
||||||
|
}
|
||||||
|
if got := lastHeard(report, &presenceEntry{Online: true, At: now.Add(-9 * time.Hour)}, now); !got.Equal(now) {
|
||||||
|
t.Fatalf("online: last heard is %v, want now", got)
|
||||||
|
}
|
||||||
|
offlineAt := now.Add(-4 * time.Minute)
|
||||||
|
if got := lastHeard(now.Add(-time.Minute), &presenceEntry{At: offlineAt}, now); !got.Equal(offlineAt) {
|
||||||
|
t.Fatalf("offline: last heard is %v, want the entry's time %v, not a report's", got, offlineAt)
|
||||||
|
}
|
||||||
|
if got := lastHeard(time.Time{}, nil, now); !got.IsZero() {
|
||||||
|
t.Fatalf("never heard and no key: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The machines facts, the nodes list, node and status all take that one function.
|
||||||
|
func TestEveryReaderOfLastHeardTakesPresence(t *testing.T) { // 1183/AC1
|
||||||
|
now := time.Now()
|
||||||
|
m := machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
|
||||||
|
applyPresence(&m, nil, &presenceEntry{At: now.Add(-4 * time.Minute)}, now)
|
||||||
|
if !m.lastHeard.Equal(now.Add(-4 * time.Minute)) {
|
||||||
|
t.Fatalf("machine facts take a report over the offline entry: %v", m.lastHeard)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func twoNodes(now time.Time) []inventory.Node {
|
||||||
|
return []inventory.Node{
|
||||||
|
{Name: "anchor", LastSeen: now.Add(-time.Hour)},
|
||||||
|
{Name: "laptop", LastSeen: now.Add(-30 * time.Second)},
|
||||||
|
{Name: "fresh"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1183/AC2: a machine online by presence whose last report is an hour old reads here, is not quiet and is not silent; one
|
||||||
|
// offline for 4 minutes reads out of touch.
|
||||||
|
func TestOnlineByPresenceReadsHereWhateverTheReport(t *testing.T) { // 1183/AC2
|
||||||
|
now := time.Now()
|
||||||
|
read := aPresenceFake(map[string]presenceEntry{
|
||||||
|
"anchor": {Online: true, At: now.Add(-6 * time.Hour)},
|
||||||
|
"laptop": {Online: false, At: now.Add(-4 * time.Minute)},
|
||||||
|
}, nil)
|
||||||
|
nodes, err := nodesHeard(context.Background(), twoNodes(now), read, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := heardFromAt(nodes[0], now); got != "here" {
|
||||||
|
t.Errorf("anchor online with an hour-old report reads %q", got)
|
||||||
|
}
|
||||||
|
if got := heardFromAt(nodes[1], now); got != "out of touch 4m" {
|
||||||
|
t.Errorf("laptop offline 4 minutes reads %q", got)
|
||||||
|
}
|
||||||
|
if got := heardFromAt(nodes[2], now); got != "never spoken" {
|
||||||
|
t.Errorf("a machine with no key and no report reads %q", got)
|
||||||
|
}
|
||||||
|
quiet := notHeardLately(nodes, now)
|
||||||
|
if len(quiet) != 1 || quiet[0].Name != "fresh" {
|
||||||
|
t.Errorf("quiet is %+v: an online machine is not quiet, and a machine offline 4 minutes is not quiet yet", quiet)
|
||||||
|
}
|
||||||
|
f := &machineFacts{name: "anchor", lastHeard: nodes[0].LastSeen}
|
||||||
|
if f.silent(now) {
|
||||||
|
t.Errorf("a machine online by presence is silent")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unreadable presence is said, and the report stands as it was.
|
||||||
|
func TestUnreadablePresenceLeavesTheReport(t *testing.T) { // 1183/AC1
|
||||||
|
now := time.Now()
|
||||||
|
nodes, err := nodesHeard(context.Background(), twoNodes(now), aPresenceFake(nil, errors.New("no bus")), now)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no bus") {
|
||||||
|
t.Fatalf("an unreadable presence is not said: %v", err)
|
||||||
|
}
|
||||||
|
if !nodes[0].LastSeen.Equal(now.Add(-time.Hour)) {
|
||||||
|
t.Fatalf("the report was replaced: %v", nodes[0].LastSeen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1183/AC4: where the watchman gives no verdict a machine is silent after a fixed three minutes, whatever interval a
|
||||||
|
// node-engine once said.
|
||||||
|
func TestTheSilentBoundIsFixed(t *testing.T) { // 1183/AC4
|
||||||
|
now := time.Now()
|
||||||
|
for _, c := range []struct {
|
||||||
|
ago time.Duration
|
||||||
|
want bool
|
||||||
|
}{{179 * time.Second, false}, {181 * time.Second, true}} {
|
||||||
|
m := machineFacts{name: "laptop", lastHeard: now.Add(-c.ago)}
|
||||||
|
if got := m.silent(now); got != c.want {
|
||||||
|
t.Errorf("heard %v ago: silent %v, want %v", c.ago, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// aLongGroup is members as JSON over 200 KB: 1666 paths of the shape of mesh-delivery #5's change (novox/hq issue 1272).
|
||||||
|
func aLongGroup(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
var paths []string
|
||||||
|
for i := 0; i < 1666; i++ {
|
||||||
|
paths = append(paths, fmt.Sprintf("modules/mesh-delivery-forge-bridge/vendor/github.com/nats-io/nats-server/v2/server/certstore/certstore_windows_file_%04d.go", i))
|
||||||
|
}
|
||||||
|
body, err := json.Marshal([]map[string]any{{"id": "novox/mesh-delivery#5", "paths": paths}, {"id": "novox/mesh-catalog#374"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(body) < 200<<10 {
|
||||||
|
t.Fatalf("the members are only %d bytes", len(body))
|
||||||
|
}
|
||||||
|
return string(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readBackFromTheCommand(t *testing.T, argv []string, members string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, a := range argv {
|
||||||
|
if len(a) > 64<<10 {
|
||||||
|
t.Errorf("an argument of %d bytes is handed to the command", len(a))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i := slices.Index(argv, "--members-file")
|
||||||
|
if i < 0 {
|
||||||
|
t.Fatalf("the command does not name a members file: %.200v", argv)
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(argv[i+1])
|
||||||
|
if err != nil || string(raw) != members {
|
||||||
|
t.Errorf("the file does not hold the whole members list (%d bytes, %v)", len(raw), err)
|
||||||
|
}
|
||||||
|
removeMembersFile(argv)
|
||||||
|
if _, err := os.Stat(argv[i+1]); !os.IsNotExist(err) {
|
||||||
|
t.Errorf("the file is still there after the command: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeliveryCheckHandsALongMembersListInAFile(t *testing.T) { // 1272/AC1
|
||||||
|
members := aLongGroup(t)
|
||||||
|
argv, err := argvFor("delivery-check", map[string]any{"group": "feat/x", "members": members})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if argv[0] != "delivery" || argv[1] != "check" || !slices.Contains(argv, "--group") {
|
||||||
|
t.Errorf("the command is %.100v", strings.Join(argv, " "))
|
||||||
|
}
|
||||||
|
readBackFromTheCommand(t, argv, members)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeliveryOrderHandsALongMembersListInAFile(t *testing.T) { // 1272/AC2
|
||||||
|
members := aLongGroup(t)
|
||||||
|
argv, err := argvFor("delivery-order", map[string]any{"members": members})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if argv[0] != "delivery" || argv[1] != "order" {
|
||||||
|
t.Errorf("the command is %.100v", strings.Join(argv, " "))
|
||||||
|
}
|
||||||
|
readBackFromTheCommand(t, argv, members)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAShortMembersListIsStillTheArgumentItWas(t *testing.T) { // 1272/AC3
|
||||||
|
members := `[{"id":"novox/a#1"},{"id":"novox/b#2"}]`
|
||||||
|
argv, err := argvFor("delivery-check", map[string]any{"group": "feat/x", "members": members, "ready": "true"})
|
||||||
|
want := []string{"delivery", "check", "--group", "feat/x", "--members", members, "--ready"}
|
||||||
|
if err != nil || !slices.Equal(argv, want) {
|
||||||
|
t.Errorf("a short list gives %v, %v, want %v", argv, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCommandReadsTheMembersFromTheFileAsFromTheArgument(t *testing.T) { // 1272/AC1
|
||||||
|
members := `[{"id":"novox/a#1"}]`
|
||||||
|
argv, _ := membersFlags(strings.Repeat(" ", membersInArgument) + members)
|
||||||
|
defer removeMembersFile(argv)
|
||||||
|
if len(argv) != 2 || argv[0] != "--members-file" {
|
||||||
|
t.Fatalf("a long list gives %v", argv)
|
||||||
|
}
|
||||||
|
if st, err := os.Stat(argv[1]); err != nil || st.Mode().Perm() != 0o600 {
|
||||||
|
t.Errorf("the file is %v, %v", st, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -154,6 +154,24 @@ type mergeWidthOf struct {
|
|||||||
// wideRebuild is how many modules a rebuild may take before the gate says so as a warning.
|
// wideRebuild is how many modules a rebuild may take before the gate says so as a warning.
|
||||||
var wideRebuild = 12
|
var wideRebuild = 12
|
||||||
|
|
||||||
|
// changedPaths are the paths --changed (comma-separated) and --changed-file (one per line) name together.
|
||||||
|
func changedPaths(list, file string) ([]string, error) {
|
||||||
|
out := splitList(list)
|
||||||
|
if file == "" {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(file)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("--changed-file: %w", err)
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(string(raw), "\n") {
|
||||||
|
if line = strings.TrimSpace(line); line != "" {
|
||||||
|
out = append(out, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// mergeGateCommand is `gate`:
|
// mergeGateCommand is `gate`:
|
||||||
//
|
//
|
||||||
// merge-gate --facts <file> --store <postgres> [--repository owner/repo --tree <dir>] [--changed a,b] [--json]
|
// merge-gate --facts <file> --store <postgres> [--repository owner/repo --tree <dir>] [--changed a,b] [--json]
|
||||||
@@ -167,6 +185,8 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
|||||||
repository := set.String("repository", "", "owner/repository of the change")
|
repository := set.String("repository", "", "owner/repository of the change")
|
||||||
tree := set.String("tree", "", "the change's checkout: every module.json in it replaces the mesh's")
|
tree := set.String("tree", "", "the change's checkout: every module.json in it replaces the mesh's")
|
||||||
changed := set.String("changed", "", "the paths the change touches, comma-separated, for its rebuild's width")
|
changed := set.String("changed", "", "the paths the change touches, comma-separated, for its rebuild's width")
|
||||||
|
changedFile := set.String("changed-file", "", "a file of the paths the change touches, one per line; the same as "+
|
||||||
|
"--changed, for a list too long for an argument (novox/hq issue 1247)")
|
||||||
group := set.String("group", "", "a delivery group's other heads, as JSON [{repository, tree, changed}], composed "+
|
group := set.String("group", "", "a delivery group's other heads, as JSON [{repository, tree, changed}], composed "+
|
||||||
"with this one as one future state (novox/hq ADR 0239)")
|
"with this one as one future state (novox/hq ADR 0239)")
|
||||||
asJSON := set.Bool("json", false, "print the verdict as JSON")
|
asJSON := set.Bool("json", false, "print the verdict as JSON")
|
||||||
@@ -198,7 +218,11 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
|||||||
// verdict at all. What is said goes beside the report.
|
// verdict at all. What is said goes beside the report.
|
||||||
os.Stdout = os.Stderr
|
os.Stdout = os.Stderr
|
||||||
}
|
}
|
||||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: splitList(*changed),
|
changedList, err := changedPaths(*changed, *changedFile)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: changedList,
|
||||||
group: others, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
group: others, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||||
os.Stdout = stdout
|
os.Stdout = stdout
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1120,6 +1144,9 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error)
|
|||||||
drop()
|
drop()
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
// The chain lives on the bus, and the gate has none: this mesh is raised for a moment, with a chain of its own in memory
|
||||||
|
// that no other reads (issue 1121).
|
||||||
|
open.inventory.UseThrowawayChain()
|
||||||
for _, m := range provided {
|
for _, m := range provided {
|
||||||
if err := open.inventory.Provide(ctx, m); err != nil {
|
if err := open.inventory.Provide(ctx, m); err != nil {
|
||||||
open.Close()
|
open.Close()
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTheGateJudgesAFileOfPathsAsItDoesTheSameListOfThem(t *testing.T) { // 1247/AC3
|
||||||
|
paths := []string{"modules/a/x.go", "modules/a/y.go", "modules/b/z.go", "README.md"}
|
||||||
|
file := filepath.Join(t.TempDir(), "changed.txt")
|
||||||
|
if err := os.WriteFile(file, []byte(strings.Join(paths, "\n")+"\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fromFile, err := changedPaths("", file)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fromList, err := changedPaths(strings.Join(paths, ","), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(fromFile, fromList) || len(fromFile) != len(paths) {
|
||||||
|
t.Errorf("the file gives %v, the list %v", fromFile, fromList)
|
||||||
|
}
|
||||||
|
if _, err := changedPaths("", filepath.Join(t.TempDir(), "none")); err == nil {
|
||||||
|
t.Error("a file that is not there is read as no paths")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,9 +50,17 @@ func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) {
|
|||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
stop()
|
stop()
|
||||||
<-done
|
<-done
|
||||||
|
lookingAtBatches.Wait()
|
||||||
setWindowClock(nil)
|
setWindowClock(nil)
|
||||||
windowLook.Store(nil)
|
windowLook.Store(nil)
|
||||||
})
|
})
|
||||||
|
// The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists
|
||||||
|
// is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the
|
||||||
|
// publisher waits for the consumer.
|
||||||
|
becomes(t, 10*time.Second, "the merges' consumer being made", func() bool {
|
||||||
|
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
|
||||||
|
return err == nil
|
||||||
|
})
|
||||||
return func(m link.SourceMoved) {
|
return func(m link.SourceMoved) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
body, err := json.Marshal(m)
|
body, err := json.Marshal(m)
|
||||||
|
|||||||
@@ -145,9 +145,15 @@ func (w *busWindow) Forget(ctx context.Context, batch string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the
|
||||||
|
// next test replaces what they read.
|
||||||
|
var lookingAtBatches sync.WaitGroup
|
||||||
|
|
||||||
// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches
|
// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches
|
||||||
// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once.
|
// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once.
|
||||||
func lookAtBatches(ctx context.Context, open *stores) {
|
func lookAtBatches(ctx context.Context, open *stores) {
|
||||||
|
lookingAtBatches.Add(1)
|
||||||
|
defer lookingAtBatches.Done()
|
||||||
release, err := open.inventory.HoldPlans(ctx, true)
|
release, err := open.inventory.HoldPlans(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err)
|
fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err)
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1211/AC1 to AC3: the controller keeps a merge announced by a module that claims the delivery seat, whatever its
|
||||||
|
// name, and takes no word from one that does not (novox/hq ADR 0437 §3). With no bus held, a word that passes the
|
||||||
|
// claim is held for the bus (ErrTryAgain); a word that does not is refused before the bus is asked.
|
||||||
|
func TestOnlyAModuleClaimingTheSeatIsTheDeliverysWordOnAMerge(t *testing.T) {
|
||||||
|
entries := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("gitea"), namedOnly("mesh-delivery")}
|
||||||
|
keeper := keepingDeliveryMerges{
|
||||||
|
js: func() *broker.JetStream { return nil },
|
||||||
|
catalog: func(context.Context) ([]inventory.Entry, error) { return entries, nil },
|
||||||
|
}
|
||||||
|
body := []byte(`{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve"}`)
|
||||||
|
for emitter, passes := range map[string]bool{
|
||||||
|
"mesh-delivery-gate": true, // the claimant, under another name
|
||||||
|
"gitea": false, // claims no seat
|
||||||
|
"mesh-delivery": false, // named so, claiming no seat here
|
||||||
|
} {
|
||||||
|
err := keeper.KeepDeliveryMerge(context.Background(), emitter, body)
|
||||||
|
if passes != errors.Is(err, link.ErrTryAgain) || (!passes && !errors.Is(err, link.ErrNotTheDeliverysWord)) {
|
||||||
|
t.Errorf("%s: %v (passes the claim: %v)", emitter, err, passes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries = []inventory.Entry{claimingTheSeat("mesh-delivery")}
|
||||||
|
if err := keeper.KeepDeliveryMerge(context.Background(), "mesh-delivery", body); !errors.Is(err, link.ErrTryAgain) {
|
||||||
|
t.Errorf("today's module, claiming the seat: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -926,13 +926,45 @@ func brokerReachableAt(ctx context.Context, inv *inventory.Inventory, known brok
|
|||||||
// A module that runs its machine's provisioner client and declares no broker is issued the client's account
|
// A module that runs its machine's provisioner client and declares no broker is issued the client's account
|
||||||
// alone (issue 917): the client speaks on the bus only as itself.
|
// alone (issue 917): the client speaks on the bus only as itself.
|
||||||
func mayIssue(m catalogue.Manifest) error {
|
func mayIssue(m catalogue.Manifest) error {
|
||||||
if _, reads := m.OwnSecrets["broker"]; !reads && !runsTheProvisionerClient(m) {
|
processes := m.ProcessBuses()
|
||||||
|
// A module whose own tools run in the machine's tool runner — mesh-mail's real shape — declares no
|
||||||
|
// `broker` of its own at all (novox/hq ADR 0436 §2): its process's account is a different account,
|
||||||
|
// delivered as its own secret below, so a process with a bus is reason enough to pass this check on
|
||||||
|
// its own, before the module's own `broker` is asked for.
|
||||||
|
if _, reads := m.OwnSecrets["broker"]; !reads && !runsTheProvisionerClient(m) && len(processes) == 0 {
|
||||||
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
||||||
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
||||||
}
|
}
|
||||||
|
// And every process with a bus declaration (novox/hq ADR 0436 §2): its own account is delivered as the
|
||||||
|
// module's own secret broker-<process id>, never the module's `broker`, so a process with a bus and no
|
||||||
|
// such secret has nowhere to read an account either.
|
||||||
|
for id := range processes {
|
||||||
|
name := processSecretName(id)
|
||||||
|
if _, reads := m.OwnSecrets[name]; !reads {
|
||||||
|
return fmt.Errorf("%s's process %s declares bus, and declares no own secret named %s, so there "+
|
||||||
|
"is nowhere to deliver its account; a process with a bus declares \"own-secrets\": "+
|
||||||
|
"{%q: <path>} (novox/hq ADR 0436 §2)", m.Module, id, name, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// processSecretName is the own secret a process's bus account is delivered in (novox/hq ADR 0436 §2):
|
||||||
|
// broker-<process id>, never the module's own `broker`.
|
||||||
|
func processSecretName(processID string) string { return "broker-" + processID }
|
||||||
|
|
||||||
|
// processPrincipalFor is the one process of m's bus declarations named processID, as `rollout mint` finds it
|
||||||
|
// again from a KindProcess principal's bare id (novox/hq ADR 0436 §1, §2): false when the manifest no longer
|
||||||
|
// declares it.
|
||||||
|
func processPrincipalFor(m catalogue.Manifest, processID string) (broker.ProcessPrincipal, bool) {
|
||||||
|
for _, pr := range inventory.ProcessesOf(m) {
|
||||||
|
if pr.Process == processID {
|
||||||
|
return pr, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return broker.ProcessPrincipal{}, false
|
||||||
|
}
|
||||||
|
|
||||||
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||||
//
|
//
|
||||||
// **Three things differ from a management call, and each is the point of the move.** The credential
|
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||||
@@ -968,12 +1000,82 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Every process with a bus declaration gets an account of its own (novox/hq ADR 0436 §1, §2), minted and
|
||||||
|
// delivered whether or not the module itself speaks on the bus — its own secret is broker-<process id>,
|
||||||
|
// never the module's `broker`.
|
||||||
|
for _, pr := range inventory.ProcessesOf(m) {
|
||||||
|
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
if !runsTheProvisionerClient(m) {
|
if !runsTheProvisionerClient(m) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return issueProvisionerClient(ctx, inv, m, node, known, reachable)
|
return issueProvisionerClient(ctx, inv, m, node, known, reachable)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// issueProcess gives one process its account: minted under its own name <node>.<module>.<process id>,
|
||||||
|
// delivered as the module's own secret broker-<process id> in the provisioner client's form (novox/hq ADR
|
||||||
|
// 0436 §2), handed to the vault as any covered account is, and its work-queue consumer or consumers
|
||||||
|
// created where the bus can be reached — never the module's claims, since a process holds no seat.
|
||||||
|
func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
|
||||||
|
node, busAddress string, known broker.Broker, reachable string) error {
|
||||||
|
user := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}.Username()
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
name := processSecretName(pr.Process)
|
||||||
|
held, err := heldProcessAccount(known, reachable, node, m.Module, user, password)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := inv.AcceptSecretForModule(ctx, node, m.Module, name, string(held)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := handIssuedToVault(ctx, inv, m, node, name, string(held)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(pr.Queues) > 0 {
|
||||||
|
if busAddress == "" {
|
||||||
|
fmt.Printf(" %s's process %s consumes its work queue; its consumer is created when the bus is "+
|
||||||
|
"reachable (`push --behind`, then `rollout mint` again is harmless)\n", m.Module, pr.Process)
|
||||||
|
} else {
|
||||||
|
js, err := broker.Dial(busAddress)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
||||||
|
"%s's process %s's work-queue consumer: %w", m.Module, pr.Process, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
who := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}
|
||||||
|
for _, q := range pr.Queues {
|
||||||
|
if err := js.EnsureConsumer(broker.QueueConsumer(who, q)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("bus user %s minted for %s's process %s, scoped to its bus declaration\n", user, m.Module, pr.Process)
|
||||||
|
fmt.Printf(" sealed to %s as %s. It arrives with the next push — `push %s` to send it\n", node, name, node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldProcessAccount is a process's account, in the provisioner client's form (novox/hq ADR 0436 §2):
|
||||||
|
// {url, fingerprint, node, module, user, password} — never claims, since a process holds no seat (§3).
|
||||||
|
func heldProcessAccount(known broker.Broker, reachable, node, module, user, password string) ([]byte, error) {
|
||||||
|
return json.Marshal(struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
User string `json:"user"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
}{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, Module: module, User: user,
|
||||||
|
Password: password})
|
||||||
|
}
|
||||||
|
|
||||||
// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an
|
// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an
|
||||||
// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret
|
// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret
|
||||||
// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor).
|
// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor).
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1145/AC1 on a store: declarationWith itself sets the module of each resource and the build of each module for a
|
||||||
|
// node recorded as reading them, and for no other. Skips without a store.
|
||||||
|
func TestAComposedDeclarationCarriesModulesAndBuildsOnlyForANodeThatReadsThem(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
start := time.Now().Add(-time.Hour)
|
||||||
|
manifest := resticFixture(false)
|
||||||
|
manifest["module"], manifest["version"] = "restic", "1"
|
||||||
|
raw, _ := json.Marshal(manifest)
|
||||||
|
b := link.BuildResult{ID: link.NewBuildID(start), Repository: "novox/mesh-catalog", Path: "modules/restic",
|
||||||
|
On: "anchor", Module: "restic", Commit: "c1", Manifest: raw,
|
||||||
|
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||||
|
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, n, "restic"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordReadsModules(ctx, anchor.ID, true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantBuild, err := inv.BuildOf(ctx, "restic", "c1")
|
||||||
|
if err != nil || wantBuild == "" {
|
||||||
|
t.Fatalf("the build of restic at c1 is %q (%v)", wantBuild, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
declare := func(node string) sendable {
|
||||||
|
t.Helper()
|
||||||
|
plan, settings, err := planFor(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gens, err := generators(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
d, err := declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
reads := bodyKeys(t, declare("anchor"))
|
||||||
|
builds, _ := reads["builds"].(map[string]any)
|
||||||
|
if builds["restic"] != wantBuild {
|
||||||
|
t.Errorf("1145/AC1: anchor reads modules and was sent builds %v, want restic at %s", reads["builds"], wantBuild)
|
||||||
|
}
|
||||||
|
owned := 0
|
||||||
|
for _, r := range reads["resources"].([]any) {
|
||||||
|
r := r.(map[string]any)
|
||||||
|
if strings.HasPrefix(r["id"].(string), "restic.") {
|
||||||
|
owned++
|
||||||
|
if r["module"] != "restic" {
|
||||||
|
t.Errorf("1145/AC1: resource %v carries module %v, want restic", r["id"], r["module"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if owned == 0 {
|
||||||
|
t.Fatal("1145/AC1: no resource of restic was composed")
|
||||||
|
}
|
||||||
|
|
||||||
|
plain := bodyKeys(t, declare("laptop"))
|
||||||
|
if _, there := plain["builds"]; there {
|
||||||
|
t.Errorf("1145/AC1: laptop never said it reads modules and was sent builds %v", plain["builds"])
|
||||||
|
}
|
||||||
|
for _, r := range plain["resources"].([]any) {
|
||||||
|
if _, there := r.(map[string]any)["module"]; there {
|
||||||
|
t.Fatalf("1145/AC1: laptop was sent a module on %v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/chain"
|
||||||
|
"github.com/novox/mesh-host/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The module of each resource and the build of each module (novox/hq issue 1145, design 47 release 1-2). The
|
||||||
|
// composer's owner of an id and the module's build go on the wire only to a machine whose node-engine said it reads
|
||||||
|
// them: an older one decodes strictly and refuses a key it does not know, whole.
|
||||||
|
|
||||||
|
func modulesDeclaration(reads bool) sendable {
|
||||||
|
return sendable{
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "web.app.conf", "type": "file", "path": "/etc/app.conf", "content": "x\n"},
|
||||||
|
{"id": "web.app.data", "type": "file", "path": "/etc/data", "content": "d\n"},
|
||||||
|
{"id": "mesh.guard", "type": "file", "path": "/etc/guard", "content": "g\n"}},
|
||||||
|
moduleOf: map[string]string{"web.app.conf": "web.app", "web.app.data": "web.app"},
|
||||||
|
buildIDs: map[string]string{"web.app": "build-7"},
|
||||||
|
ReadsModules: reads,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1145/AC1
|
||||||
|
func TestAMachineThatReadsModulesIsSentEachResourcesModuleAndEachModulesBuild(t *testing.T) {
|
||||||
|
keys := bodyKeys(t, modulesDeclaration(true))
|
||||||
|
builds, _ := keys["builds"].(map[string]any)
|
||||||
|
if len(builds) != 1 || builds["web.app"] != "build-7" {
|
||||||
|
t.Fatalf("1145/AC1: the body carries builds %v", keys["builds"])
|
||||||
|
}
|
||||||
|
want := map[string]any{"web.app.conf": "web.app", "web.app.data": "web.app", "mesh.guard": nil}
|
||||||
|
for _, r := range keys["resources"].([]any) {
|
||||||
|
r := r.(map[string]any)
|
||||||
|
if r["module"] != want[r["id"].(string)] {
|
||||||
|
t.Errorf("1145/AC1: resource %v carries module %v, want %v", r["id"], r["module"], want[r["id"].(string)])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1145/AC2: the body is byte for byte what it is today.
|
||||||
|
func TestAMachineThatDoesNotReadModulesIsSentTheBodyItWasSentBefore(t *testing.T) {
|
||||||
|
d := modulesDeclaration(false)
|
||||||
|
got, err := d.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before := sendable{Resources: d.Resources}
|
||||||
|
want, _ := before.Body()
|
||||||
|
if string(got) != string(want) {
|
||||||
|
t.Fatalf("1145/AC2: the body changed:\n%s\nwas\n%s", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1145/AC4: the node-engine's own validator, as the gate runs it, takes the two words.
|
||||||
|
func TestTheNodeEnginesValidatorTakesAModulePerResourceAndABuildPerModule(t *testing.T) {
|
||||||
|
raw, err := modulesDeclaration(true).Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var generic map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &generic); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if problems := validate.Declaration(raw); len(problems) != 0 {
|
||||||
|
t.Fatalf("1145/AC4: refused: %q\n%s", problems, fmt.Sprint(generic))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The build a module carries is the one its assignment names when that is the commit carried, else the one built from
|
||||||
|
// the commit carried; none when neither is known.
|
||||||
|
func TestTheBuildOfACarriedCommitIsTheAssignmentsOrTheOneBuiltFromIt(t *testing.T) {
|
||||||
|
runs := map[string]chain.Ref{"a": {Build: "b-a", Commit: "c1"}, "b": {Build: "b-old", Commit: "c0"}}
|
||||||
|
looked := map[string]string{"b/c2": "b-new"}
|
||||||
|
got, err := wireBuilds(map[string]string{"a": "c1", "b": "c2", "c": "c3", "d": ""}, runs,
|
||||||
|
func(module, commit string) (string, error) { return looked[module+"/"+commit], nil })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := map[string]string{"a": "b-a", "b": "b-new"}
|
||||||
|
if len(got) != len(want) || got["a"] != want["a"] || got["b"] != want["b"] {
|
||||||
|
t.Fatalf("1145: builds %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -12,8 +13,10 @@ import (
|
|||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
@@ -44,6 +47,28 @@ type deliveryMerge struct {
|
|||||||
AskID string `json:"ask_id"`
|
AskID string `json:"ask_id"`
|
||||||
Option string `json:"option"`
|
Option string `json:"option"`
|
||||||
Heads map[string]string `json:"heads"`
|
Heads map[string]string `json:"heads"`
|
||||||
|
// Asker is the module that asked the ask, whose warrant is read from the router under that name (novox/hq ADR
|
||||||
|
// 0437 §6): empty is today's module, mesh-delivery.
|
||||||
|
Asker string `json:"asker,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// askerOf is the asker whose warrant the merge is read under: the one the announcement names, mesh-delivery when it
|
||||||
|
// names none.
|
||||||
|
func (d deliveryMerge) askerOf() string {
|
||||||
|
if d.Asker == "" {
|
||||||
|
return theDelivery
|
||||||
|
}
|
||||||
|
return d.Asker
|
||||||
|
}
|
||||||
|
|
||||||
|
// askerAccepted is whether the controller takes a warrant from this asker for a merge: the teller, and mesh-delivery
|
||||||
|
// until the gate takes the seat over (novox/hq ADR 0437 §6). Any other registers no move.
|
||||||
|
func askerAccepted(asker string) error {
|
||||||
|
if asker == theDelivery || asker == sdkdelivery.TellerModule {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("the merge names the asker %q, and the controller takes a warrant only from %s and %s",
|
||||||
|
asker, sdkdelivery.TellerModule, theDelivery)
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseDeliveryMerge reads an announcement, refusing one that names no pull request, head or ask.
|
// parseDeliveryMerge reads an announcement, refusing one that names no pull request, head or ask.
|
||||||
@@ -79,6 +104,10 @@ func mergeAct(ask, option string, heads map[string]string) asks.Act {
|
|||||||
// proved by the channel it came through, of this very ask as it was published (digest), bound to the heads the
|
// proved by the channel it came through, of this very ask as it was published (digest), bound to the heads the
|
||||||
// announcement names, and those heads hold the head merged. Answers why it holds, or why not.
|
// announcement names, and those heads hold the head merged. Answers why it holds, or why not.
|
||||||
func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
||||||
|
asker := d.askerOf()
|
||||||
|
if err := askerAccepted(asker); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case ask == nil || w == nil:
|
case ask == nil || w == nil:
|
||||||
return "", errors.New("the router holds no closed record of the ask the merge was made on")
|
return "", errors.New("the router holds no closed record of the ask the merge was made on")
|
||||||
@@ -87,7 +116,7 @@ func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, er
|
|||||||
case d.Option != optionApprove:
|
case d.Option != optionApprove:
|
||||||
return "", fmt.Errorf("the merge was made on the option %q, not an Approve", d.Option)
|
return "", fmt.Errorf("the merge was made on the option %q, not an Approve", d.Option)
|
||||||
}
|
}
|
||||||
option, err := w.For(theDelivery, *ask)
|
option, err := w.For(asker, *ask)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -115,9 +144,26 @@ func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, er
|
|||||||
// keepingDeliveryMerges keeps each merge mesh-delivery announces in the controller's bucket for them.
|
// keepingDeliveryMerges keeps each merge mesh-delivery announces in the controller's bucket for them.
|
||||||
type keepingDeliveryMerges struct {
|
type keepingDeliveryMerges struct {
|
||||||
js func() *broker.JetStream
|
js func() *broker.JetStream
|
||||||
|
// catalog is the catalogue, whose modules claiming the delivery seat are the only ones whose word on a merge is
|
||||||
|
// kept (novox/hq ADR 0437 §3).
|
||||||
|
catalog func(ctx context.Context) ([]inventory.Entry, error)
|
||||||
|
// kept, when set, is told each merge once it is kept: a first merge waiting for the word asks its builds then
|
||||||
|
// (novox/hq ADR 0432, issue 1146).
|
||||||
|
kept func(ctx context.Context, d deliveryMerge) error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, body []byte) error {
|
func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, emitter string, body []byte) error {
|
||||||
|
if k.catalog == nil {
|
||||||
|
return fmt.Errorf("%w: the catalogue is not read", link.ErrTryAgain)
|
||||||
|
}
|
||||||
|
entries, err := k.catalog(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(claimantsOf(entries, catalogue.DeliverySeat), emitter) {
|
||||||
|
return fmt.Errorf("%w: %q is not among %v", link.ErrNotTheDeliverysWord, emitter,
|
||||||
|
claimantsOf(entries, catalogue.DeliverySeat))
|
||||||
|
}
|
||||||
d, err := parseDeliveryMerge(body)
|
d, err := parseDeliveryMerge(body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -137,6 +183,9 @@ func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, body []byt
|
|||||||
if _, err := kv.Put(ctx, deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA), body); err != nil {
|
if _, err := kv.Put(ctx, deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA), body); err != nil {
|
||||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||||
}
|
}
|
||||||
|
if k.kept != nil {
|
||||||
|
return k.kept(ctx, d)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -166,9 +215,9 @@ var keptDeliveryMerge = func(ctx context.Context, owner, repo string, number int
|
|||||||
return d, err == nil, err
|
return d, err == nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// routerAskOf is the router's record of one of mesh-delivery's asks: the ask as published and its warrant. A
|
// routerAskOf is the router's record of one of the delivery's asks, under its asker: the ask as published and its warrant. A
|
||||||
// variable so a test can hand it one.
|
// variable so a test can hand it one.
|
||||||
var routerAskOf = func(ctx context.Context, inv *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
var routerAskOf = func(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
serving := servingBus.Load()
|
serving := servingBus.Load()
|
||||||
if serving == nil {
|
if serving == nil {
|
||||||
return nil, nil, errors.New("only the serving controller reads the router's record")
|
return nil, nil, errors.New("only the serving controller reads the router's record")
|
||||||
@@ -180,7 +229,7 @@ var routerAskOf = func(ctx context.Context, inv *inventory.Inventory, id string)
|
|||||||
if bucket == "" {
|
if bucket == "" {
|
||||||
return nil, nil, errors.New("no module declares the operator channel's records")
|
return nil, nil, errors.New("no module declares the operator channel's records")
|
||||||
}
|
}
|
||||||
_, ask, w, err := readRouterEntry(ctx, serving.Conn(), bucket, theDelivery, id)
|
_, ask, w, err := readRouterEntry(ctx, serving.Conn(), bucket, asker, id)
|
||||||
return ask, w, err
|
return ask, w, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -216,7 +265,7 @@ var errNotAMerge = errors.New("the build is not of a merge")
|
|||||||
// (ADR 0420 §7): the build is of a merge, that merge is a pull request mesh-delivery announced it made, and the
|
// (ADR 0420 §7): the build is of a merge, that merge is a pull request mesh-delivery announced it made, and the
|
||||||
// ask it made it on holds an Approve by the operator of that head. Answers why it was accepted; an error says why
|
// ask it made it on holds an Approve by the operator of that head. Answers why it was accepted; an error says why
|
||||||
// it was not, and the refusal of ADR 0266 stands.
|
// it was not, and the refusal of ADR 0266 stands.
|
||||||
func approvedMove(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
||||||
built inventory.Source) (string, error) {
|
built inventory.Source) (string, error) {
|
||||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -236,24 +285,42 @@ func approvedMove(ctx context.Context, inv *inventory.Inventory, forms *sourceFo
|
|||||||
if !known {
|
if !known {
|
||||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, r.Commit)
|
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, r.Commit)
|
||||||
}
|
}
|
||||||
k := announcedOf(merged)
|
return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged))
|
||||||
|
}
|
||||||
|
|
||||||
|
// errWordNotAnApprove says the merge cannot be shown to be the operator's Approve of its head: its announcement names no
|
||||||
|
// pull request, or the word on record is not that Approve.
|
||||||
|
var errWordNotAnApprove = errors.New("the word on record is no Approve of the head merged")
|
||||||
|
|
||||||
|
// errNotAnnounced says mesh-delivery's word on a merge is not kept (yet): the merge may have been made by someone else,
|
||||||
|
// or its word may still be on its way.
|
||||||
|
var errNotAnnounced = errors.New("mesh-delivery announced no such merge")
|
||||||
|
|
||||||
|
// approvedAnnounced is the warrant of the merge the forge announced as k: mesh-delivery announced it, and the ask it made
|
||||||
|
// it on holds the operator's Approve of that head. Why it holds, or why not. Shared by the approved move of a known
|
||||||
|
// module and the first build of a new repository's modules (ADR 0420 §7, ADR 0432).
|
||||||
|
func approvedAnnounced(ctx context.Context, inv catalogueReader, owner, name string, k keptMerge) (string, error) {
|
||||||
if k.Number <= 0 || k.HeadSHA == "" {
|
if k.Number <= 0 || k.HeadSHA == "" {
|
||||||
return "", errors.New("the forge's announcement of the merge names no pull request or head")
|
return "", fmt.Errorf("%w: the forge's announcement of the merge names no pull request or head", errWordNotAnApprove)
|
||||||
}
|
}
|
||||||
d, kept, err := keptDeliveryMerge(ctx, owner, name, k.Number, k.HeadSHA)
|
d, kept, err := keptDeliveryMerge(ctx, owner, name, k.Number, k.HeadSHA)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if !kept {
|
if !kept {
|
||||||
return "", fmt.Errorf("mesh-delivery announced no merge of %s/%s#%d at %.8s", owner, name, k.Number, k.HeadSHA)
|
return "", fmt.Errorf("%w: %s/%s#%d at %.8s", errNotAnnounced, owner, name, k.Number, k.HeadSHA)
|
||||||
}
|
}
|
||||||
ask, w, err := routerAskOf(ctx, inv, d.AskID)
|
// The warrant is read under the asker the announcement names, and under none the controller does not take.
|
||||||
|
if err := askerAccepted(d.askerOf()); err != nil {
|
||||||
|
return "", fmt.Errorf("%w: %w", errWordNotAnApprove, err)
|
||||||
|
}
|
||||||
|
ask, w, err := routerAskOf(ctx, inv, d.askerOf(), d.AskID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
why, err := movedOnApprove(d, ask, w)
|
why, err := movedOnApprove(d, ask, w)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", fmt.Errorf("%w: %w", errWordNotAnApprove, err)
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%s/%s#%d merged at %.8s on %s", owner, name, k.Number, k.HeadSHA, why), nil
|
return fmt.Sprintf("%s/%s#%d merged at %.8s on %s", owner, name, k.Number, k.HeadSHA, why), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The controller reads a merge's warrant under the asker the announcement names, and accepts only the teller and
|
||||||
|
// today's mesh-delivery (novox/hq ADR 0437 §6, issue 1213). The teller's name is the SDK's.
|
||||||
|
|
||||||
|
// askedBy is an approved merge whose announcement names an asker ("" names none) and whose warrant is that asker's.
|
||||||
|
func askedBy(t *testing.T, named, warrantOf string) anApprovedMerge {
|
||||||
|
t.Helper()
|
||||||
|
a := approvedMerge(t)
|
||||||
|
a.merge.Asker = named
|
||||||
|
a.warrant.Asker = warrantOf
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1213/AC1: an announcement naming the teller, and the teller's warrant of the operator's verified Approve of the
|
||||||
|
// merge's heads: the move is registered (the warrant is read from the router under the teller).
|
||||||
|
func TestAMergeNamingTheTellerIsMovedOnTheTellersWarrant(t *testing.T) {
|
||||||
|
a := askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule)
|
||||||
|
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil {
|
||||||
|
t.Fatalf("the teller's warrant of the operator's Approve was refused: %v", err)
|
||||||
|
}
|
||||||
|
was := routerAskOf
|
||||||
|
t.Cleanup(func() { routerAskOf = was })
|
||||||
|
wasKept := keptDeliveryMerge
|
||||||
|
t.Cleanup(func() { keptDeliveryMerge = wasKept })
|
||||||
|
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||||
|
return a.merge, true, nil
|
||||||
|
}
|
||||||
|
var readUnder string
|
||||||
|
routerAskOf = func(_ context.Context, _ catalogueReader, asker, _ string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
|
readUnder = asker
|
||||||
|
return &a.ask, &a.warrant, nil
|
||||||
|
}
|
||||||
|
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil {
|
||||||
|
t.Fatalf("the merge was not moved on the teller's warrant: %v", err)
|
||||||
|
}
|
||||||
|
if readUnder != sdkdelivery.TellerModule {
|
||||||
|
t.Errorf("the warrant was read under %q", readUnder)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1213/AC2: an announcement naming mesh-delivery, or none (today's module): read under mesh-delivery, registered.
|
||||||
|
func TestAMergeNamingNoAskerIsMovedOnMeshDeliverysWarrant(t *testing.T) {
|
||||||
|
for _, named := range []string{"", theDelivery} {
|
||||||
|
a := askedBy(t, named, theDelivery)
|
||||||
|
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil {
|
||||||
|
t.Errorf("asker %q: refused: %v", named, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
was := routerAskOf
|
||||||
|
wasKept := keptDeliveryMerge
|
||||||
|
t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept })
|
||||||
|
a := askedBy(t, "", theDelivery)
|
||||||
|
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||||
|
return a.merge, true, nil
|
||||||
|
}
|
||||||
|
var readUnder string
|
||||||
|
routerAskOf = func(_ context.Context, _ catalogueReader, asker, _ string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
|
readUnder = asker
|
||||||
|
return &a.ask, &a.warrant, nil
|
||||||
|
}
|
||||||
|
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if readUnder != theDelivery {
|
||||||
|
t.Errorf("with no asker named the warrant was read under %q, not %q", readUnder, theDelivery)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1213/AC3: any other asker registers no move, and the reason names the asker refused; the router is not asked.
|
||||||
|
func TestAMergeNamingAnyOtherAskerMovesNothingAndNamesIt(t *testing.T) {
|
||||||
|
was := routerAskOf
|
||||||
|
wasKept := keptDeliveryMerge
|
||||||
|
t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept })
|
||||||
|
for _, other := range []string{sdkdelivery.GateModule, "mesh-controller", "unknown-module"} {
|
||||||
|
a := askedBy(t, other, other) // even a warrant of that very asker
|
||||||
|
_, err := movedOnApprove(a.merge, &a.ask, &a.warrant)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), other) {
|
||||||
|
t.Errorf("asker %q: %v", other, err)
|
||||||
|
}
|
||||||
|
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||||
|
return a.merge, true, nil
|
||||||
|
}
|
||||||
|
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
|
t.Errorf("the router was read under the refused asker %q", other)
|
||||||
|
return &a.ask, &a.warrant, nil
|
||||||
|
}
|
||||||
|
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), other) {
|
||||||
|
t.Errorf("asker %q, approved as announced: %v", other, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1213/AC4: a warrant whose own asker differs from the asker read, or whose heads differ from the head merged, moves nothing.
|
||||||
|
func TestAWarrantOfAnotherAskerOrOtherHeadsMovesNothing(t *testing.T) {
|
||||||
|
a := askedBy(t, sdkdelivery.TellerModule, theDelivery) // read under the teller, but the warrant says mesh-delivery
|
||||||
|
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||||
|
t.Errorf("a warrant of another asker was accepted: %s", why)
|
||||||
|
}
|
||||||
|
a = askedBy(t, "", sdkdelivery.TellerModule) // today's module named, the teller's warrant
|
||||||
|
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||||
|
t.Errorf("the teller's warrant under mesh-delivery was accepted: %s", why)
|
||||||
|
}
|
||||||
|
a = askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule)
|
||||||
|
a.merge.HeadSHA = "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4"
|
||||||
|
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||||
|
t.Errorf("a head the Approve did not cover was accepted: %s", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1213/AC5: the announcement's asker is read and kept with the rest of it, under the JSON name asker.
|
||||||
|
func TestTheAnnouncementsAskerIsReadAndKept(t *testing.T) {
|
||||||
|
body := `{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve","asker":"mesh-delivery-teller"}`
|
||||||
|
d, err := parseDeliveryMerge([]byte(body))
|
||||||
|
if err != nil || d.Asker != sdkdelivery.TellerModule {
|
||||||
|
t.Fatalf("read %+v, %v", d, err)
|
||||||
|
}
|
||||||
|
none, err := parseDeliveryMerge([]byte(strings.Replace(body, `,"asker":"mesh-delivery-teller"`, "", 1)))
|
||||||
|
if err != nil || none.Asker != "" {
|
||||||
|
t.Fatalf("an announcement naming none read as %+v, %v", none, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// announcedPull is the forge's announcement of a merge of a pull request at a head.
|
||||||
|
func announcedPull(number int, head string) keptMerge {
|
||||||
|
var k keptMerge
|
||||||
|
k.Number, k.HeadSHA = number, head
|
||||||
|
return k
|
||||||
|
}
|
||||||
@@ -74,7 +74,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
|||||||
}
|
}
|
||||||
|
|
||||||
server := link.ConnectNats(js, nil, noListener{})
|
server := link.ConnectNats(js, nil, noListener{})
|
||||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||||
@@ -90,7 +90,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
|||||||
|
|
||||||
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||||
body, _ := json.Marshal(a.merge)
|
body, _ := json.Marshal(a.merge)
|
||||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubject, body); err != nil {
|
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||||
|
|||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The operator's Approve of a merge registers a new module from its repository** (novox/hq ADR 0432, issue 1146).
|
||||||
|
// The mesh as the decision reads it, with nothing registered from the repository, and the build request and merge the
|
||||||
|
// controller kept.
|
||||||
|
type anEmptyMesh struct {
|
||||||
|
request inventory.BuildRequest
|
||||||
|
asked bool
|
||||||
|
merge inventory.BatchedMerge
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *anEmptyMesh) Catalogue(context.Context) (map[string]catalogue.Manifest, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (m *anEmptyMesh) SourceOf(context.Context, string) (inventory.Source, error) {
|
||||||
|
return inventory.Source{}, inventory.ErrNoSuchModule
|
||||||
|
}
|
||||||
|
func (m *anEmptyMesh) SourceIdentity(context.Context, string) (int64, error) { return 0, nil }
|
||||||
|
func (m *anEmptyMesh) Catalogued(context.Context) ([]inventory.Entry, error) { return nil, nil }
|
||||||
|
func (m *anEmptyMesh) BuildRequestByID(_ context.Context, id string) (inventory.BuildRequest, bool, error) {
|
||||||
|
return m.request, m.asked && id == m.request.ID, nil
|
||||||
|
}
|
||||||
|
func (m *anEmptyMesh) MergeOf(context.Context, string, string) (inventory.BatchedMerge, bool, error) {
|
||||||
|
return m.merge, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// aDeliveryMergeOf is the Approved merge of a pull request of novox/mesh-delivery, its heads keyed by that repository,
|
||||||
|
// and the ask and warrant bound to exactly those heads (the digest the Approve left covers them).
|
||||||
|
func aDeliveryMergeOf(t *testing.T, head string, covered map[string]string) anApprovedMerge {
|
||||||
|
t.Helper()
|
||||||
|
a := approvedMerge(t)
|
||||||
|
a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-delivery", 7, head
|
||||||
|
a.merge.Heads = covered
|
||||||
|
for n := range a.ask.Options {
|
||||||
|
d, err := asks.ActDigest(mergeAct(a.merge.AskID, a.ask.Options[n].ID, covered))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a.ask.Options[n].Binds = d
|
||||||
|
}
|
||||||
|
a.warrant.AskDigest = a.ask.Digest()
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwise(t *testing.T) {
|
||||||
|
const commit, head = "c0ffee0123456789", "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||||
|
built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
change func(*anApprovedMerge)
|
||||||
|
request string // "merge", or "build" through a verb
|
||||||
|
unkept bool
|
||||||
|
forge any // what the forge says of the repository; nil is protected
|
||||||
|
built *inventory.Source
|
||||||
|
accepted bool
|
||||||
|
ac string
|
||||||
|
}{
|
||||||
|
{what: "the Approve of the merged head, a protected trunk on the forge", request: "merge", accepted: true, ac: "1146/AC1"},
|
||||||
|
{what: "a build asked through a verb", request: "build", ac: "1146/AC2"},
|
||||||
|
{what: "a merge with no warrant", request: "merge", unkept: true, ac: "1146/AC2"},
|
||||||
|
{what: "a merged head the Approve did not cover", request: "merge", ac: "1146/AC2", change: func(a *anApprovedMerge) {
|
||||||
|
*a = aDeliveryMergeOf(t, head, map[string]string{
|
||||||
|
"novox/mesh-delivery@eeeeeeeeeeee": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"})
|
||||||
|
}},
|
||||||
|
{what: "heads other than the ones the Approve was bound to", request: "merge", ac: "1146/AC2", change: func(a *anApprovedMerge) {
|
||||||
|
a.merge.Heads = map[string]string{"novox/mesh-delivery@eeeeeeeeeeee": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"}
|
||||||
|
}},
|
||||||
|
{what: "a repository not on the mesh's forge", request: "merge", ac: "1146/AC3",
|
||||||
|
built: &inventory.Source{Repository: "http://elsewhere.example/x/y.git"}},
|
||||||
|
{what: "a forge that cannot be asked", request: "merge", ac: "1146/AC3", forge: errors.New("connection refused")},
|
||||||
|
{what: "an unprotected trunk", request: "merge", ac: "1146/AC3", forge: forgeFacts{ID: 9, Guarded: false, Why: "direct push"}},
|
||||||
|
} {
|
||||||
|
t.Run(c.ac+" "+c.what, func(t *testing.T) {
|
||||||
|
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||||
|
if c.change != nil {
|
||||||
|
c.change(&a)
|
||||||
|
}
|
||||||
|
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||||
|
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||||
|
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||||
|
return a.merge, !c.unkept, nil
|
||||||
|
}
|
||||||
|
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
|
return &a.ask, &a.warrant, nil
|
||||||
|
}
|
||||||
|
if c.forge != nil {
|
||||||
|
theForge.Store("novox/mesh-delivery", c.forge)
|
||||||
|
t.Cleanup(func() { theForge.Delete("novox/mesh-delivery") })
|
||||||
|
}
|
||||||
|
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||||
|
"head_sha": head, "merge_commit_sha": commit})
|
||||||
|
mesh := &anEmptyMesh{asked: true, merge: inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main",
|
||||||
|
Commit: commit, Merged: time.Now(), Event: event},
|
||||||
|
request: inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "",
|
||||||
|
Ref: "main", Commit: commit, For: c.request}}
|
||||||
|
forms := &sourceForms{bases: map[string]string{}, base: func(seat string) string {
|
||||||
|
if seat == "git" {
|
||||||
|
return "http://forge.internal:20000"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}}
|
||||||
|
src := built
|
||||||
|
if c.built != nil {
|
||||||
|
src = *c.built
|
||||||
|
}
|
||||||
|
id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", src, "build-1", "", "main")
|
||||||
|
switch {
|
||||||
|
case c.accepted && err != nil:
|
||||||
|
t.Fatalf("%s: the Approved merge's new module was refused: %v", c.ac, err)
|
||||||
|
case c.accepted && id == 0:
|
||||||
|
t.Fatalf("%s: registered with no forge identity to record", c.ac)
|
||||||
|
case !c.accepted && !errors.Is(err, errNotItsSource):
|
||||||
|
t.Fatalf("%s: %s was not refused: %v", c.ac, c.what, err)
|
||||||
|
case !c.accepted && c.ac == "1146/AC2" &&
|
||||||
|
(!strings.Contains(err.Error(), "controller's terminal") || !strings.Contains(err.Error(), "Approve")):
|
||||||
|
t.Fatalf("%s: the refusal does not name both ways in: %v", c.ac, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second new module from a repository whose first was registered is the case ADR 0266 §44 already has —
|
||||||
|
// "alongside": registered through a verb with no warrant asked of the router (1146/AC5).
|
||||||
|
type aMeshWithOneAlready struct{ anEmptyMesh }
|
||||||
|
|
||||||
|
func (m *aMeshWithOneAlready) Catalogued(context.Context) ([]inventory.Entry, error) {
|
||||||
|
e := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}}
|
||||||
|
e.Manifest.Module = "mesh-delivery-view"
|
||||||
|
return []inventory.Entry{e}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASecondNewModuleFromARepositoryIsJudgedAlongsideAndAsksNoWarrant(t *testing.T) {
|
||||||
|
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||||
|
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||||
|
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||||
|
t.Error("1146/AC5: the announced merges were asked for a module judged alongside")
|
||||||
|
return deliveryMerge{}, false, nil
|
||||||
|
}
|
||||||
|
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||||
|
t.Error("1146/AC5: the router was asked for a module judged alongside")
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
mesh := &aMeshWithOneAlready{anEmptyMesh{asked: true,
|
||||||
|
request: inventory.BuildRequest{ID: "build-2", Repository: "novox/mesh-delivery", Seat: "git", For: "build"}}}
|
||||||
|
forms := &sourceForms{bases: map[string]string{}, base: func(string) string { return "http://forge.internal:20000" }}
|
||||||
|
built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}
|
||||||
|
if id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-watchman", built, "build-2", "", "main"); err != nil || id == 0 {
|
||||||
|
t.Fatalf("1146/AC5: a second module from a repository the catalogue builds from was refused (id %d): %v", id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1146/AC4: a new module from a repository nothing registered is built from, whole against a real bus and the store
|
||||||
|
// (novox/hq ADR 0432, issue 1146; the warrant is read as for a move, ADR 0420 §7, issue 1025): mesh-delivery's word that it
|
||||||
|
// merged a pull request is published on the subject the controller consumes and kept in its bucket; the router's
|
||||||
|
// record of the ask is read back from the router's bucket by a direct get; and the build of a module from the
|
||||||
|
// repository that merge was made in is registered from there only when that record holds the operator's Approve of
|
||||||
|
// the merged head. Store-backed: skips without one.
|
||||||
|
func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
answer bool // the router holds the operator's Approve of the merged heads
|
||||||
|
moved bool
|
||||||
|
}{
|
||||||
|
{what: "the router holds the operator's Approve", answer: true, moved: true},
|
||||||
|
{what: "the router holds no record of the ask"},
|
||||||
|
} {
|
||||||
|
t.Run(c.what, func(t *testing.T) {
|
||||||
|
open := theCatalogue(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
// The module declaring the router's records, as the controller finds them.
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||||
|
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||||
|
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a := aDeliveryMergeOf(t, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||||
|
map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"})
|
||||||
|
if c.answer {
|
||||||
|
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||||
|
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server := link.ConnectNats(js, nil, noListener{})
|
||||||
|
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||||
|
if err := broker.AssertServingConsumers(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
serving, stop := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||||
|
t.Cleanup(func() { stop(); <-done })
|
||||||
|
servingBus.Store(js)
|
||||||
|
t.Cleanup(func() { servingBus.Store(nil) })
|
||||||
|
|
||||||
|
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||||
|
body, _ := json.Marshal(a.merge)
|
||||||
|
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||||
|
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, a.merge.HeadSHA)
|
||||||
|
return err == nil && kept
|
||||||
|
})
|
||||||
|
|
||||||
|
// The forge's merge of that pull request, as the controller heard it, and the build it asked for.
|
||||||
|
const commit = "c0ffee0123456789"
|
||||||
|
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||||
|
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||||
|
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit,
|
||||||
|
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||||
|
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-delivery", Seat: "git",
|
||||||
|
Path: "modules/mesh-delivery-view", Ref: "main", Commit: commit, For: "merge"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
moved := onTrunk(id, "novox/mesh-delivery", "git", "modules/mesh-delivery-view", map[string]any{"module": "mesh-delivery-view", "version": "2"})
|
||||||
|
moved.Commit = commit
|
||||||
|
_, _, err = takeIn(ctx, inv, moved)
|
||||||
|
src, _ := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||||
|
if c.moved && (err != nil || src.Repository != "novox/mesh-delivery") {
|
||||||
|
t.Fatalf("the Approved move was not registered from its repository: %v (built from %q)", err, src.Repository)
|
||||||
|
}
|
||||||
|
if !c.moved && (!errors.Is(err, errNotItsSource) || src.Repository != "") {
|
||||||
|
t.Fatalf("a new module with no Approve was not refused as it is: %v (built from %q)", err, src.Repository)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,7 +50,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
return addNode(ctx, inv, args[1:])
|
return addNode(ctx, inv, args[1:])
|
||||||
|
|
||||||
case "list":
|
case "list":
|
||||||
nodes, err := inv.Nodes(ctx)
|
nodes, err := open.heardNodes(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -755,22 +755,8 @@ func consumerReset(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
// heardFrom is heardFromAt for now.
|
||||||
//
|
func heardFrom(n inventory.Node) string { return heardFromAt(n, time.Now()) }
|
||||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
|
||||||
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
|
|
||||||
// picture of the mesh.
|
|
||||||
func heardFrom(n inventory.Node) string {
|
|
||||||
silent, ever := n.Silent()
|
|
||||||
switch {
|
|
||||||
case !ever:
|
|
||||||
return "never spoken"
|
|
||||||
case silent > SilentFor:
|
|
||||||
return "out of touch " + roughly(silent)
|
|
||||||
default:
|
|
||||||
return "here"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// roughly is a duration a person reads rather than parses.
|
// roughly is a duration a person reads rather than parses.
|
||||||
func roughly(d time.Duration) string {
|
func roughly(d time.Duration) string {
|
||||||
@@ -795,7 +781,7 @@ func roughly(d time.Duration) string {
|
|||||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
func showNode(ctx context.Context, stored *stores, name string) error {
|
||||||
inv := stored.inventory
|
inv := stored.inventory
|
||||||
node, err := inv.NodeByName(ctx, name)
|
node, err := stored.heardNode(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/chain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1146: a merge's ask that failed is kept under an id the bus takes as a key, for a module at the repository's root
|
||||||
|
// (".") as for one in a folder.
|
||||||
|
func TestTheIdOfABuildNotAskedIsAKeyTheBusTakes(t *testing.T) {
|
||||||
|
for dir, want := range map[string]string{
|
||||||
|
".": "not-asked-0177ea7a",
|
||||||
|
"modules/mesh-delivery-view": "not-asked-0177ea7a-modules-mesh-delivery-view",
|
||||||
|
} {
|
||||||
|
id := notAskedID("0177ea7a00000000", dir)
|
||||||
|
if id != want {
|
||||||
|
t.Errorf("notAskedID(%q) = %q, want %q", dir, id, want)
|
||||||
|
}
|
||||||
|
if !chain.ValidKey("req." + id) {
|
||||||
|
t.Errorf("notAskedID(%q) = %q: the bus takes no key req.%s", dir, id, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The controller finds the delivery's owner by the claim of the seat mesh-delivery, never by the module's name
|
||||||
|
// (novox/hq ADR 0437 §3, issue 1210). The gate is a module named mesh-delivery-gate; this is only a fixture name.
|
||||||
|
|
||||||
|
func claimingTheSeat(module string) inventory.Entry {
|
||||||
|
return inventory.Entry{Manifest: catalogue.Manifest{Module: module,
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func namedOnly(module string) inventory.Entry {
|
||||||
|
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}, On: []string{"anchor"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1210/AC1: a module claiming the seat is on the controller's own path whatever its name, awaiting the delivery's
|
||||||
|
// word as today's module does; a module named mesh-delivery that claims no seat is not on it.
|
||||||
|
func TestTheModuleClaimingTheSeatIsOnTheControllersOwnPath(t *testing.T) {
|
||||||
|
gate := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")}
|
||||||
|
own := ownPathOf(gate)
|
||||||
|
if _, on := own["mesh-delivery-gate"]; !on {
|
||||||
|
t.Error("the module claiming the seat is not on the controller's own path")
|
||||||
|
}
|
||||||
|
if _, on := own["mesh-delivery"]; on {
|
||||||
|
t.Error("a module named mesh-delivery that claims no seat is on the controller's own path")
|
||||||
|
}
|
||||||
|
if d := awaitsFor(gate, []string{"gitea", "plex"}); d == nil || d.Awaits != catalogue.DeliverySeat {
|
||||||
|
t.Errorf("an ordinary walk awaits %+v with the gate holding the seat", d)
|
||||||
|
}
|
||||||
|
if d := awaitsFor(gate, []string{"mesh-delivery-gate", "gitea"}); d != nil {
|
||||||
|
t.Errorf("a walk moving the gate awaits %+v; the gate cannot gate itself", d)
|
||||||
|
}
|
||||||
|
if d := awaitsFor(gate, []string{"mesh-delivery", "gitea"}); d == nil {
|
||||||
|
t.Error("a walk moving a module named mesh-delivery that claims no seat starts at once")
|
||||||
|
}
|
||||||
|
if !ownWalk(walkOf("novox/one", "notes", "mesh-delivery-gate"), own) || ownWalk(walkOf("novox/one", "notes", "mesh-delivery"), own) {
|
||||||
|
t.Error("a walk is on the own path by the claim of the module it plans")
|
||||||
|
}
|
||||||
|
if ownPath([]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}) != true || ownPath([]inventory.Entry{namedOnly("mesh-delivery")}) {
|
||||||
|
t.Error("a merge touches the own path by the claim of the module it touches")
|
||||||
|
}
|
||||||
|
if classOfModules([]string{"mesh-delivery-gate"}, gate) != inventory.ClassCore {
|
||||||
|
t.Error("the module claiming the seat is not core")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1210/AC2: the merge gate's rule of the module claiming the seat is one reading; a module claiming no delivery
|
||||||
|
// seat gets its ordinary rule.
|
||||||
|
func TestTheModuleClaimingTheSeatGetsTheDeliverysGateRule(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
claimant := claimingTheSeat("mesh-delivery-gate").Manifest
|
||||||
|
r := gateRuleOf("mesh-delivery-gate", claimant)
|
||||||
|
if r.passes != 1 || r.within != 0 || !strings.HasPrefix(r.said, "mesh-delivery's rule: one reading (novox/hq ADR 0351)") {
|
||||||
|
t.Errorf("the gate's rule is %+v", r)
|
||||||
|
}
|
||||||
|
r = gateRuleOf("mesh-delivery", catalogue.Manifest{Module: "mesh-delivery"})
|
||||||
|
if !strings.HasPrefix(r.said, "a leaf module's rule") {
|
||||||
|
t.Errorf("a module named mesh-delivery claiming no seat gets %q", r.said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1210/AC3: with both today's module and the gate claiming the seat (R3d's group), both are on the own path and
|
||||||
|
// both get the delivery's rule.
|
||||||
|
func TestBothClaimantsAreOnTheOwnPathAndGetTheRule(t *testing.T) {
|
||||||
|
withTheGatesTimes(t)
|
||||||
|
both := []inventory.Entry{claimingTheSeat("mesh-delivery"), claimingTheSeat("mesh-delivery-gate")}
|
||||||
|
own := ownPathOf(both)
|
||||||
|
for _, m := range []string{"mesh-delivery", "mesh-delivery-gate"} {
|
||||||
|
if _, on := own[m]; !on {
|
||||||
|
t.Errorf("%s is not on the own path", m)
|
||||||
|
}
|
||||||
|
if r := gateRuleOf(m, claimingTheSeat(m).Manifest); r.passes != 1 || !strings.HasPrefix(r.said, "mesh-delivery's rule") {
|
||||||
|
t.Errorf("%s gets %+v", m, r)
|
||||||
|
}
|
||||||
|
if d := awaitsFor(both, []string{m, "gitea"}); d != nil {
|
||||||
|
t.Errorf("a walk moving %s awaits %+v", m, d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that
|
||||||
|
// announce a merge from it.
|
||||||
|
func theDeliveryClaims(context.Context) ([]inventory.Entry, error) {
|
||||||
|
return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil
|
||||||
|
}
|
||||||
@@ -491,8 +491,7 @@ func settlingPending(ctx context.Context, open *stores) {
|
|||||||
|
|
||||||
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
|
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
|
||||||
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
|
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
|
||||||
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
|
// ends its assignment; ended ones are raised as conditions and cleared once answered. Returns what it did.
|
||||||
// KeptFor are deleted. Returns what it did.
|
|
||||||
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
var said []string
|
var said []string
|
||||||
@@ -565,11 +564,6 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
said = append(said, raisePendingEnded(ctx, inv)...)
|
said = append(said, raisePendingEnded(ctx, inv)...)
|
||||||
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
|
|
||||||
say("ended pending assignments could not be deleted: %v", err)
|
|
||||||
} else if n > 0 {
|
|
||||||
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
|
|
||||||
}
|
|
||||||
return said
|
return said
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -585,7 +579,7 @@ func containsString(xs []string, x string) bool {
|
|||||||
// pendingObservation is the condition for a pending assignment that was not made: one per pending
|
// pendingObservation is the condition for a pending assignment that was not made: one per pending
|
||||||
// assignment, its row's id the last part of its id, so one row's condition is never another's.
|
// assignment, its row's id the last part of its id, so one row's condition is never another's.
|
||||||
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
|
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
|
||||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
|
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%s", p.Node, p.Module, p.ID),
|
||||||
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
|
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
|
||||||
Resolver: conditions.ResolverOperator, Source: "pending assignments",
|
Resolver: conditions.ResolverOperator, Source: "pending assignments",
|
||||||
Summary: p.Note}
|
Summary: p.Note}
|
||||||
@@ -671,8 +665,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
|
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
|
||||||
}
|
}
|
||||||
byID := map[int64]string{}
|
byID := map[string]string{}
|
||||||
var ids []int64
|
var ids []string
|
||||||
for _, c := range open {
|
for _, c := range open {
|
||||||
if c.Kind != kindPendingEnded {
|
if c.Kind != kindPendingEnded {
|
||||||
continue
|
continue
|
||||||
@@ -682,8 +676,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
|||||||
if len(parts) < 2 {
|
if len(parts) < 2 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var id int64
|
id := parts[len(parts)-2]
|
||||||
if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
|
if !strings.HasPrefix(id, "p-") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
byID[id] = c.Key
|
byID[id] = c.Key
|
||||||
@@ -701,7 +695,7 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
|||||||
if known[id] {
|
if known[id] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
|
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed, or the module was assigned and the wait taken back"); err != nil {
|
||||||
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
|
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -542,23 +542,26 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
|
|||||||
t.Fatalf("a module registered meanwhile: %+v", p)
|
t.Fatalf("a module registered meanwhile: %+v", p)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
|
// Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open
|
||||||
// still open (second review, bug B).
|
// ones never are left out, nor one whose condition is still open (second review, bug B).
|
||||||
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
|
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
later := time.Now().Add(inventory.KeptFor + time.Hour)
|
||||||
rows, err := inv.Pending(ctx, time.Unix(0, 0))
|
settlePending(ctx, open, later)
|
||||||
|
rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, r := range rows {
|
for _, r := range rows {
|
||||||
if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
|
if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
|
||||||
t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
|
t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
|
for _, r := range rows {
|
||||||
t.Fatalf("the applied row was not deleted: %v", said)
|
if r.Module == "heard" {
|
||||||
|
t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -637,8 +640,9 @@ func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
|
|||||||
}
|
}
|
||||||
settlePending(ctx, open, time.Now())
|
settlePending(ctx, open, time.Now())
|
||||||
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||||
if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
|
// No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared.
|
||||||
t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
|
if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil {
|
||||||
|
t.Fatalf("a cleared row is not kept, saying so: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
|
// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
|
|||||||
return found
|
return found
|
||||||
}
|
}
|
||||||
|
|
||||||
func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
func rowOf(t *testing.T, open *stores, id string) inventory.PendingAssignment {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
|
rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -34,7 +34,7 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
t.Fatalf("pending assignment %d is not on record", id)
|
t.Fatalf("pending assignment %s is not on record", id)
|
||||||
return inventory.PendingAssignment{}
|
return inventory.PendingAssignment{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -429,6 +429,11 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
"out) has not let it start yet.",
|
"out) has not let it start yet.",
|
||||||
Resolved: "Resolved: the delivery is no longer waiting"}
|
Resolved: "Resolved: the delivery is no longer waiting"}
|
||||||
}),
|
}),
|
||||||
|
kindPlanFailed: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "A delivery failed",
|
||||||
|
Explanation: "A delivery did not finish. Nothing is tried again until you say so.",
|
||||||
|
Resolved: "Resolved: the delivery is no longer failed"}
|
||||||
|
}),
|
||||||
"ask-lost": worded(func(o conditions.Observation) words {
|
"ask-lost": worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "A build has no result",
|
return words{Headline: "A build has no result",
|
||||||
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",
|
||||||
@@ -610,6 +615,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
"did not answer in ten minutes, so nothing of the delivery is built or sent.",
|
"did not answer in ten minutes, so nothing of the delivery is built or sent.",
|
||||||
Resolved: "Resolved: the backup is done and the delivery goes on"}
|
Resolved: "Resolved: the backup is done and the delivery goes on"}
|
||||||
}),
|
}),
|
||||||
|
kindStuck: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "Something in a delivery has stopped moving",
|
||||||
|
Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " +
|
||||||
|
"asked and not answered, a check that stopped, or a part that went offline. Nothing is lost, and it is " +
|
||||||
|
"said again until it moves.",
|
||||||
|
Resolved: "Resolved: it moves again"}
|
||||||
|
}),
|
||||||
|
kindSendKeyRefused: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "A delivery cannot watch a machine for its report",
|
||||||
|
Explanation: "The bus refused the note that makes the mesh notice a machine that never reports on a send. " +
|
||||||
|
"The delivery goes on, and the note is written again at its next look.",
|
||||||
|
Resolved: "Resolved: the delivery watches the machine for its report again"}
|
||||||
|
}),
|
||||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "Messages pile up for a listener",
|
return words{Headline: "Messages pile up for a listener",
|
||||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||||
|
|||||||
+93
-10
@@ -16,6 +16,7 @@ import (
|
|||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/chain"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/licences"
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -284,12 +285,28 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
assigned []string
|
assigned []string
|
||||||
}
|
}
|
||||||
var others []candidate
|
var others []candidate
|
||||||
|
// **Every machine's assignments, before the holdings are read by any resolution** (novox/hq ADR 0449 §4): a
|
||||||
|
// seat held once whose holder on record is assigned on no machine needs the whole mesh to be known, and it
|
||||||
|
// passes to its only claimant here, in the composition that finds it.
|
||||||
|
everyAssigned := map[string][]string{}
|
||||||
|
var assignedErr error
|
||||||
|
for _, n := range nodes {
|
||||||
|
theirs, err := inv.Assigned(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
assignedErr = errors.Join(assignedErr, fmt.Errorf("%s: %w", n.Name, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(theirs) > 0 {
|
||||||
|
everyAssigned[n.Name] = theirs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
holdings = theMeshPassesSeatsOn(ctx, inv, shelf, everyAssigned, assignedErr, holdings)
|
||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
if n.Name == exclude {
|
if n.Name == exclude {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
theirs, err := inv.Assigned(ctx, n.Name)
|
theirs := everyAssigned[n.Name]
|
||||||
if err != nil || len(theirs) == 0 {
|
if len(theirs) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
caps, _ := inv.ProfileOf(ctx, n.Name)
|
caps, _ := inv.ProfileOf(ctx, n.Name)
|
||||||
@@ -500,14 +517,55 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||||
if choosing == Allocating {
|
if choosing == Allocating {
|
||||||
current, err := open.inventory.CurrentBuilds(ctx)
|
carried, err := carriedOn(ctx, open, node, plan, composed.LeftOut, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, err
|
return sendable{}, err
|
||||||
}
|
}
|
||||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
out.Builds = carried
|
||||||
|
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||||
|
}
|
||||||
|
// And the module of each resource and the build of each module, to a machine whose node-engine said it reads them
|
||||||
|
// (novox/hq issue 1145) and to no other; the same whatever the composition is for, so what a question about the
|
||||||
|
// send digests is what the send digests.
|
||||||
|
reads, err := open.inventory.ReadsModules(ctx, record.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, err
|
return sendable{}, err
|
||||||
}
|
}
|
||||||
|
if reads {
|
||||||
|
carried := out.Builds
|
||||||
|
if choosing != Allocating {
|
||||||
|
if carried, err = carriedOn(ctx, open, node, plan, composed.LeftOut, false); err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runs, err := open.inventory.AssignmentRuns(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
builds, err := wireBuilds(carried, runs, func(module, commit string) (string, error) {
|
||||||
|
return open.inventory.BuildOf(ctx, module, commit)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
out.ReadsModules, out.moduleOf, out.buildIDs = true, composed.Owner, builds
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// carriedOn is the build of each module a declaration for node carries (see carriedBuilds), the module kept at the
|
||||||
|
// build a machine runs included (ADR 0245). Only a send (strict) fails for a kept module that cannot be read: a question
|
||||||
|
// about it is answered without.
|
||||||
|
func carriedOn(ctx context.Context, open *stores, node string, plan catalogue.Resolution, leftOut map[string]string,
|
||||||
|
strict bool) (map[string]string, error) {
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if !known {
|
if !known {
|
||||||
before = nil
|
before = nil
|
||||||
}
|
}
|
||||||
@@ -515,19 +573,20 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
names = append(names, m.Module)
|
names = append(names, m.Module)
|
||||||
}
|
}
|
||||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
out := carriedBuilds(names, leftOut, current, before)
|
||||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||||
kept, err := recordedKept(ctx, open, node)
|
kept, err := recordedKept(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sendable{}, err
|
if strict {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kept = nil
|
||||||
}
|
}
|
||||||
for m, was := range kept {
|
for m, was := range kept {
|
||||||
if _, carried := out.Builds[m]; carried {
|
if _, carried := out[m]; carried {
|
||||||
out.Builds[m] = was
|
out[m] = was
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
|
||||||
}
|
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2109,3 +2168,27 @@ func sortedNames[V any](m map[string]V) []string {
|
|||||||
sort.Strings(out)
|
sort.Strings(out)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// wireBuilds is the build of each carried module that goes on the wire (novox/hq issue 1145): the build its assignment
|
||||||
|
// names when that assignment is of the commit carried, else the build made from that commit; a module with neither
|
||||||
|
// is not named.
|
||||||
|
func wireBuilds(carried map[string]string, runs map[string]chain.Ref, lookup func(module, commit string) (string, error)) (map[string]string, error) {
|
||||||
|
out := map[string]string{}
|
||||||
|
for module, commit := range carried {
|
||||||
|
if commit == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ref, ok := runs[module]; ok && ref.Commit == commit && ref.Build != "" {
|
||||||
|
out[module] = ref.Build
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
build, err := lookup(module, commit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if build != "" {
|
||||||
|
out[module] = build
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// heldPublish is a bus that holds a plan-moved publish until it is let go, and keeps the body.
|
||||||
|
type heldPublish struct {
|
||||||
|
link.Bus
|
||||||
|
let chan struct{}
|
||||||
|
body chan []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *heldPublish) PublishSeatEvent(_ context.Context, _, _ string, body []byte) error {
|
||||||
|
<-h.let
|
||||||
|
h.body <- body
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func aWalkWithSteps() inventory.Plan {
|
||||||
|
built := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC)
|
||||||
|
return inventory.Plan{ID: "plan-1", Repository: "novox/one", Commit: "abc", State: "walking", Tier: 1,
|
||||||
|
Created: built, Updated: built, Revision: 4,
|
||||||
|
Tiers: [][]string{{"alpha"}, {"beta", "gamma"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"alpha": {State: "built", BuiltAt: &built, Commit: "abc", First: []string{"ace"}},
|
||||||
|
"beta": {State: "asked", Why: "waiting"},
|
||||||
|
"gamma": {},
|
||||||
|
},
|
||||||
|
Times: &inventory.PlanTimes{Started: &built}, WalkSaid: inventory.WalkStarted}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAnnouncementIsThePlanAsItWasWhenSaidWhatTheWalkDoesNext(t *testing.T) { // 1260/AC1
|
||||||
|
p := aWalkWithSteps()
|
||||||
|
bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)}
|
||||||
|
sayPlanMoved(t.Context(), bus, nil, p)
|
||||||
|
|
||||||
|
// The walk goes on, writing its steps while the announcement is being made.
|
||||||
|
for i := 0; i < 2000; i++ {
|
||||||
|
step := p.Modules["beta"]
|
||||||
|
step.State, step.Why = "built", "changed after the call"
|
||||||
|
step.First = append(step.First[:0], "novox")
|
||||||
|
p.Modules["delta"] = &inventory.PlanModule{State: "asked"}
|
||||||
|
p.Tiers[1][0] = "changed"
|
||||||
|
p.Times.Started = nil
|
||||||
|
delete(p.Modules, "delta")
|
||||||
|
}
|
||||||
|
close(bus.let)
|
||||||
|
|
||||||
|
var said inventory.Plan
|
||||||
|
select {
|
||||||
|
case body := <-bus.body:
|
||||||
|
if err := json.Unmarshal(body, &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("nothing was announced")
|
||||||
|
}
|
||||||
|
if said.Modules["beta"].State != "asked" || said.Modules["beta"].Why != "waiting" || said.Tiers[1][0] != "beta" ||
|
||||||
|
len(said.Modules) != 3 || said.Times == nil || said.Times.Started == nil {
|
||||||
|
t.Errorf("the announcement carries the walk as it went on, not as it was: %+v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAnnouncementOfAnUnchangedPlanIsTheBodyItWasBeforeTheCopy(t *testing.T) { // 1260/AC2
|
||||||
|
p := aWalkWithSteps()
|
||||||
|
bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)}
|
||||||
|
close(bus.let)
|
||||||
|
sayPlanMoved(t.Context(), bus, nil, p)
|
||||||
|
var got []byte
|
||||||
|
select {
|
||||||
|
case got = <-bus.body:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("nothing was announced")
|
||||||
|
}
|
||||||
|
// What it was: the plan whole, its phases worked out, and what the walk says of itself.
|
||||||
|
q := p
|
||||||
|
q.Phases = q.WalkPhases(time.Now(), nil)
|
||||||
|
want, err := json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), nil, q)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var a, b map[string]any
|
||||||
|
if json.Unmarshal(got, &a) != nil || json.Unmarshal(want, &b) != nil {
|
||||||
|
t.Fatal("a body is not JSON")
|
||||||
|
}
|
||||||
|
delete(a, "phases") // worked out from the time of the call
|
||||||
|
delete(b, "phases")
|
||||||
|
if !reflect.DeepEqual(a, b) {
|
||||||
|
t.Errorf("the body changed:\n got %s\nwant %s", got, want)
|
||||||
|
}
|
||||||
|
if a["walk"] != "plan-1" {
|
||||||
|
t.Errorf("the walk's own words are not in the body: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// failedModule is one module of a failed plan that failed to build, where it runs and what it said.
|
||||||
|
type failedModule struct {
|
||||||
|
module string
|
||||||
|
machines []string
|
||||||
|
why string
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1).
|
||||||
|
type failedPlanFacts struct {
|
||||||
|
id, repository, commit string
|
||||||
|
modules []string
|
||||||
|
failed []failedModule
|
||||||
|
// note is where the plan stopped when no module failed to build (a rollout stopped at its first machine).
|
||||||
|
note string
|
||||||
|
// since is the plan's last change: when it failed.
|
||||||
|
since time.Time
|
||||||
|
// refusal is why `plans retry` would not run it, empty when it would.
|
||||||
|
refusal string
|
||||||
|
// passed is set when a newer plan of its repository and branch is done and the retry is refused for that
|
||||||
|
// (novox/hq ADR 0451): the condition asks nothing.
|
||||||
|
passed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// machines are the machines the failed modules run on, sorted, each once.
|
||||||
|
func (p failedPlanFacts) machines() []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, m := range p.failed {
|
||||||
|
for _, n := range m.machines {
|
||||||
|
if !seen[n] {
|
||||||
|
seen[n] = true
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// failureSaid is what failed, in the mesh's own words, for the condition's evidence.
|
||||||
|
func (p failedPlanFacts) failureSaid() string {
|
||||||
|
var parts []string
|
||||||
|
for _, m := range p.failed {
|
||||||
|
line := m.module
|
||||||
|
if len(m.machines) > 0 {
|
||||||
|
line += " on " + strings.Join(m.machines, ", ")
|
||||||
|
}
|
||||||
|
if m.why != "" {
|
||||||
|
line += ": " + oneLine(m.why)
|
||||||
|
}
|
||||||
|
parts = append(parts, line)
|
||||||
|
}
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return "stopped: " + oneLine(p.note)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusalSaid is why a retry would not run, for the evidence.
|
||||||
|
func (p failedPlanFacts) refusalSaid() string {
|
||||||
|
if p.refusal == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "; a retry would not run: " + oneLine(p.refusal)
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan,
|
||||||
|
// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs).
|
||||||
|
func retryAction(plan string) conditions.Action {
|
||||||
|
return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
|
||||||
|
Arguments: map[string]string{"retry": plan}}
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`)
|
||||||
|
walkWd = regexp.MustCompile(`\bwalks\b`)
|
||||||
|
walkOne = regexp.MustCompile(`\bwalk\b`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a
|
||||||
|
// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the
|
||||||
|
// words say only that a retry would not run.
|
||||||
|
func plainRefusal(refusal string) string {
|
||||||
|
return plainRefusalOr(refusal, "a retry would not run now; the details say why")
|
||||||
|
}
|
||||||
|
|
||||||
|
// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly.
|
||||||
|
func plainRefusalOr(refusal, fallback string) string {
|
||||||
|
r := planIDWord.ReplaceAllString(refusal, "the plan")
|
||||||
|
r = walkWd.ReplaceAllString(r, "deliveries")
|
||||||
|
r = walkOne.ReplaceAllString(r, "delivery")
|
||||||
|
r = strings.TrimSpace(r)
|
||||||
|
if _, ok := conditions.Plain(r); !ok {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that
|
||||||
|
// is plain, and Retry when a retry would run, else the refusal's words.
|
||||||
|
func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) {
|
||||||
|
name := deliveryName(p.modules, p.repository)
|
||||||
|
headline = conditions.Capital(name) + " failed"
|
||||||
|
what := deliveryWhat(p.modules, p.repository)
|
||||||
|
var parts []string
|
||||||
|
for i, m := range p.failed {
|
||||||
|
if i == 2 {
|
||||||
|
parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
line := m.module
|
||||||
|
if len(m.machines) > 0 {
|
||||||
|
line += " for " + readableList(m.machines)
|
||||||
|
}
|
||||||
|
if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" {
|
||||||
|
if _, ok := conditions.Plain(w); ok {
|
||||||
|
line += " (" + w + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts = append(parts, line)
|
||||||
|
}
|
||||||
|
if len(parts) > 0 {
|
||||||
|
explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts))
|
||||||
|
} else {
|
||||||
|
explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case p.passed:
|
||||||
|
explanation += " A newer delivery has passed it, so a retry would send older code. Nothing is needed from you."
|
||||||
|
case p.refusal == "":
|
||||||
|
explanation += " Nothing is tried again until you say so."
|
||||||
|
needs = "try the failed builds again " + FromMeshMCPServer
|
||||||
|
actions = []conditions.Action{retryAction(p.id)}
|
||||||
|
default:
|
||||||
|
explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "."
|
||||||
|
needs = "look at why it failed; a retry would not run now."
|
||||||
|
}
|
||||||
|
resolved = conditions.Capital(name) + " is no longer failed"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// cutWords is s cut to at most n bytes at a word.
|
||||||
|
func cutWords(s string, n int) string {
|
||||||
|
if len(s) <= n {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
t := s[:n]
|
||||||
|
if i := strings.LastIndexByte(t, ' '); i > n/2 {
|
||||||
|
t = t[:i]
|
||||||
|
}
|
||||||
|
return strings.TrimRight(t, " ,;:") + "…"
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator
|
||||||
|
// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not
|
||||||
|
// stopped by a person. Each carries why `plans retry` would not run it, when it would not.
|
||||||
|
func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) {
|
||||||
|
plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the plans cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var failed []inventory.Plan
|
||||||
|
for _, p := range plans {
|
||||||
|
if failedAndAsked(p) {
|
||||||
|
failed = append(failed, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(failed) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
onPath := ownPathOf(entries)
|
||||||
|
runsOn := map[string][]string{}
|
||||||
|
for _, e := range entries {
|
||||||
|
runsOn[e.Manifest.Module] = e.On
|
||||||
|
}
|
||||||
|
var out []failedPlanFacts
|
||||||
|
for _, p := range failed {
|
||||||
|
out = append(out, failedFactsOf(p, plans, onPath, runsOn))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not
|
||||||
|
// stopped by a person.
|
||||||
|
func failedAndAsked(p inventory.Plan) bool {
|
||||||
|
return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each
|
||||||
|
// module runs.
|
||||||
|
func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string,
|
||||||
|
runsOn map[string][]string) failedPlanFacts {
|
||||||
|
f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p),
|
||||||
|
note: p.Note, since: p.Updated}
|
||||||
|
for _, m := range f.modules {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||||
|
machines := append([]string(nil), runsOn[m]...)
|
||||||
|
sort.Strings(machines)
|
||||||
|
f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := retryRefusal(p, plans, onPath); err != nil {
|
||||||
|
f.refusal = err.Error()
|
||||||
|
var passed passedBy
|
||||||
|
f.passed = errors.As(err, &passed)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
@@ -0,0 +1,338 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314).
|
||||||
|
|
||||||
|
func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan {
|
||||||
|
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed,
|
||||||
|
Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC),
|
||||||
|
Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}},
|
||||||
|
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
||||||
|
if mutate != nil {
|
||||||
|
mutate(&p)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts {
|
||||||
|
return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{},
|
||||||
|
map[string][]string{"openrazer": {"g14", "shanks"}})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1314/AC1
|
||||||
|
func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||||
|
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||||
|
f := calm(now)
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||||
|
got := watchPlansFailed(f)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
o := got[0]
|
||||||
|
if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator ||
|
||||||
|
o.Scope != conditions.ScopePlan || o.ID != p.ID {
|
||||||
|
t.Fatalf("kind, severity, resolver or subject: %+v", o)
|
||||||
|
}
|
||||||
|
if len(o.Actions) != 1 {
|
||||||
|
t.Fatalf("actions: %+v", o.Actions)
|
||||||
|
}
|
||||||
|
a := o.Actions[0]
|
||||||
|
if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove ||
|
||||||
|
len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID {
|
||||||
|
t.Fatalf("the action is not Retry with only the plan: %+v", a)
|
||||||
|
}
|
||||||
|
if o.Needs == "" {
|
||||||
|
t.Fatal("no Needs: the asker wants an action with a need")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} {
|
||||||
|
if !strings.Contains(o.Headline+o.Explanation+o.Said, want) {
|
||||||
|
t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") {
|
||||||
|
t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation)
|
||||||
|
}
|
||||||
|
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions}
|
||||||
|
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||||
|
t.Errorf("the words are not plain: %s\n%+v", why, words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1314/AC2
|
||||||
|
func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) {
|
||||||
|
for name, p := range map[string]inventory.Plan{
|
||||||
|
"held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }),
|
||||||
|
"stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }),
|
||||||
|
"no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }),
|
||||||
|
"building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }),
|
||||||
|
} {
|
||||||
|
if failedAndAsked(p) {
|
||||||
|
t.Errorf("%s: a plan not asked of the operator is asked", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !failedAndAsked(aFailedPlan("plan-5", nil)) {
|
||||||
|
t.Error("a failed delivery plan is not asked")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it.
|
||||||
|
func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||||
|
older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) })
|
||||||
|
newer := aFailedPlan("plan-2", func(p *inventory.Plan) {
|
||||||
|
p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute)
|
||||||
|
})
|
||||||
|
superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" })
|
||||||
|
done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone })
|
||||||
|
first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC)
|
||||||
|
gate := aFailedPlan("plan-5", func(p *inventory.Plan) {
|
||||||
|
p.Note = "openrazer stopped at its first machine"
|
||||||
|
p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"},
|
||||||
|
Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"},
|
||||||
|
Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}}
|
||||||
|
})
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
p inventory.Plan
|
||||||
|
others []inventory.Plan
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
"a newer plan": {older, []inventory.Plan{newer}, "supersedes it"},
|
||||||
|
"superseded": {superseded, nil, "superseded"},
|
||||||
|
"done": {done, nil, "nothing to retry"},
|
||||||
|
"gate failed": {gate, nil, "failed its gate"},
|
||||||
|
} {
|
||||||
|
f := calm(now)
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)}
|
||||||
|
got := watchPlansFailed(f)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("%s: %+v", name, got)
|
||||||
|
}
|
||||||
|
o := got[0]
|
||||||
|
if len(o.Actions) != 0 {
|
||||||
|
t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions)
|
||||||
|
}
|
||||||
|
if !strings.Contains(o.Said, c.says) {
|
||||||
|
t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) {
|
||||||
|
t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation)
|
||||||
|
}
|
||||||
|
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||||
|
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||||
|
t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23.
|
||||||
|
func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
defer k.Close(t.Context())
|
||||||
|
p := aFailedPlan("plan-1792433168676870614", nil)
|
||||||
|
f := calm(now)
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||||
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
open, _ := k.Open(t.Context())
|
||||||
|
if len(open) != 1 || open[0].Kind != "plan-failed" {
|
||||||
|
t.Fatalf("not raised: %+v", open)
|
||||||
|
}
|
||||||
|
// Retried: the plan is building, so it is no longer among the failed.
|
||||||
|
f = calm(now.Add(time.Minute))
|
||||||
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if open, _ = k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("the condition outlived the failure: %+v", open)
|
||||||
|
}
|
||||||
|
// Failed again: asked again, with Retry, and nothing retried it.
|
||||||
|
f = calm(now.Add(2 * time.Minute))
|
||||||
|
again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) })
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(again)}
|
||||||
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
open, _ = k.Open(t.Context())
|
||||||
|
if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" {
|
||||||
|
t.Fatalf("a second failure was not asked again with Retry: %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing
|
||||||
|
// S23 produces is an observation that offers an answer.
|
||||||
|
func TestNoClockRetriesAFailedPlan(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||||
|
f := calm(now)
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))}
|
||||||
|
for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} {
|
||||||
|
f.now = now.Add(later)
|
||||||
|
got := watchPlansFailed(f)
|
||||||
|
if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" {
|
||||||
|
t.Fatalf("after %s: %+v", later, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A plan failed longer ago than the week it is kept is left to `plans`.
|
||||||
|
f.now = now.Add(planFailedWithin + time.Second)
|
||||||
|
if got := watchPlansFailed(f); len(got) != 0 {
|
||||||
|
t.Fatalf("a plan failed beyond the window is still raised: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A later plan of the same repository and branch, in a state, made after the failed one.
|
||||||
|
func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan {
|
||||||
|
q := aFailedPlan(id, func(p *inventory.Plan) {
|
||||||
|
p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour)
|
||||||
|
})
|
||||||
|
return q
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan.
|
||||||
|
func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) {
|
||||||
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
||||||
|
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
||||||
|
err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") {
|
||||||
|
t.Fatalf("the retry was not refused, naming the later plan: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing.
|
||||||
|
func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) {
|
||||||
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) })
|
||||||
|
open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed)
|
||||||
|
err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "supersedes it") {
|
||||||
|
t.Fatalf("with the later plan open the refusal is not today's: %v", err)
|
||||||
|
}
|
||||||
|
other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed)
|
||||||
|
if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil {
|
||||||
|
t.Fatalf("a later done plan of another branch refused the retry: %v", err)
|
||||||
|
}
|
||||||
|
earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed)
|
||||||
|
earlier.Created = failed.Created.Add(-time.Hour)
|
||||||
|
if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil {
|
||||||
|
t.Fatalf("an earlier done plan refused the retry: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery.
|
||||||
|
func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||||
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) })
|
||||||
|
later := laterPlan("plan-2", "main", inventory.PlanDone, failed)
|
||||||
|
f := calm(now)
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(failed, later)}
|
||||||
|
got := watchPlansFailed(f)
|
||||||
|
if len(got) != 1 || len(got[0].Actions) != 0 {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
o := got[0]
|
||||||
|
if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") {
|
||||||
|
t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation)
|
||||||
|
}
|
||||||
|
if strings.Contains(o.Explanation, "details say why") {
|
||||||
|
t.Errorf("the generic fallback was said: %q", o.Explanation)
|
||||||
|
}
|
||||||
|
words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
||||||
|
if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok {
|
||||||
|
t.Errorf("not plain: %s\n%+v", why, words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since.
|
||||||
|
type plansBehind []inventory.Plan
|
||||||
|
|
||||||
|
func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) {
|
||||||
|
var out []inventory.Plan
|
||||||
|
for _, p := range s {
|
||||||
|
if p.Open() {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) {
|
||||||
|
all := append([]inventory.Plan(nil), s...)
|
||||||
|
sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) })
|
||||||
|
if len(all) > limit {
|
||||||
|
all = all[:limit]
|
||||||
|
}
|
||||||
|
return all, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) {
|
||||||
|
var out []inventory.Plan
|
||||||
|
for _, p := range s {
|
||||||
|
if p.Created.After(since) || p.Open() {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it.
|
||||||
|
func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) {
|
||||||
|
failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) })
|
||||||
|
all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)}
|
||||||
|
for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50
|
||||||
|
other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed)
|
||||||
|
other.Repository = "novox/other"
|
||||||
|
other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour)
|
||||||
|
all = append(all, other)
|
||||||
|
}
|
||||||
|
plans, err := plansForRetry(t.Context(), all, failed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") {
|
||||||
|
t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its
|
||||||
|
// plan's last change passes 7 days.
|
||||||
|
func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)
|
||||||
|
old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) })
|
||||||
|
f := calm(now)
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(old)}
|
||||||
|
if got := watchPlansFailed(f); len(got) != 0 {
|
||||||
|
t.Fatalf("a plan last changed 8 days ago raised %+v", got)
|
||||||
|
}
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||||
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
||||||
|
defer k.Close(t.Context())
|
||||||
|
fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) })
|
||||||
|
f.failed = []failedPlanFacts{failedFactsFor(fresh)}
|
||||||
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
||||||
|
t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open)
|
||||||
|
}
|
||||||
|
f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old
|
||||||
|
if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||||
|
t.Fatalf("the condition outlived the 7 days: %+v", open)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -138,7 +138,9 @@ func askToLook() {
|
|||||||
|
|
||||||
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
|
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
|
||||||
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
|
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
|
||||||
func startPlanLooks(ctx context.Context, open *stores) {
|
// It answers a channel that is closed once the look has returned after the context ended, for a test that restores what
|
||||||
|
// the look reads (issue 1338).
|
||||||
|
func startPlanLooks(ctx context.Context, open *stores) <-chan struct{} {
|
||||||
l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) })
|
l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) })
|
||||||
l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) }
|
l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) }
|
||||||
lookerMu.Lock()
|
lookerMu.Lock()
|
||||||
@@ -152,8 +154,13 @@ func startPlanLooks(ctx context.Context, open *stores) {
|
|||||||
}
|
}
|
||||||
lookerMu.Unlock()
|
lookerMu.Unlock()
|
||||||
}()
|
}()
|
||||||
go l.run(ctx)
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
l.run(ctx)
|
||||||
|
}()
|
||||||
l.ask()
|
l.ask()
|
||||||
|
return done
|
||||||
}
|
}
|
||||||
|
|
||||||
// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's
|
// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's
|
||||||
@@ -163,8 +170,10 @@ var applySilentAfter = inventory.ApplySilentAfter
|
|||||||
// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974):
|
// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974):
|
||||||
// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own
|
// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own
|
||||||
// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left
|
// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left
|
||||||
// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one.
|
// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one. It answers
|
||||||
func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) {
|
// a function that waits until the look at the plans has returned, which only a test that ends the context needs (issue
|
||||||
|
// 1338): the serving controller ignores it.
|
||||||
|
func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) (wait func()) {
|
||||||
if err := startRolloutSilent(ctx, js); err != nil {
|
if err := startRolloutSilent(ctx, js); err != nil {
|
||||||
fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err)
|
fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err)
|
||||||
}
|
}
|
||||||
@@ -174,7 +183,8 @@ func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream)
|
|||||||
if err := startPlanMovedLooks(ctx, js); err != nil {
|
if err := startPlanMovedLooks(ctx, js); err != nil {
|
||||||
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
|
fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err)
|
||||||
}
|
}
|
||||||
startPlanLooks(ctx, open)
|
done := startPlanLooks(ctx, open)
|
||||||
|
return func() { <-done }
|
||||||
}
|
}
|
||||||
|
|
||||||
// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a
|
// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a
|
||||||
@@ -400,33 +410,39 @@ func syncSilentKeys(ctx context.Context, open *stores) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
|
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
|
||||||
// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
|
// cannot be read, one cannot be put — the refusal is raised as a condition naming the plan and the machine, never
|
||||||
// the look is retried.
|
// stood in for by a timer, and the error is given back so the look is retried and writes the key again (novox/hq ADR
|
||||||
|
// 0433 §4). The set of refusals is what this look found: a key written at the next look is no longer in it.
|
||||||
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
|
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
|
||||||
byTimer := func() {
|
refused := func(err error) map[silentWait]error {
|
||||||
for w, sent := range want {
|
out := map[silentWait]error{}
|
||||||
silentByTimer(w, sent, silentTTL(sent, now))
|
for w := range want {
|
||||||
|
out[w] = err
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
if c == nil {
|
if c == nil {
|
||||||
byTimer()
|
noteRefusedSendKeys(refused(errors.New("the bus's clock for the waits is not there")))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
c = boundedClock{c}
|
||||||
held, err := c.Held(ctx)
|
held, err := c.Held(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
byTimer()
|
noteRefusedSendKeys(refused(err))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
arm, drop := silentDiff(want, held)
|
arm, drop := silentDiff(want, held)
|
||||||
var first error
|
var first error
|
||||||
|
notTaken := map[silentWait]error{}
|
||||||
for _, w := range arm {
|
for _, w := range arm {
|
||||||
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
|
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
|
||||||
silentByTimer(w, want[w], silentTTL(want[w], now))
|
notTaken[w] = err
|
||||||
if first == nil {
|
if first == nil {
|
||||||
first = err
|
first = err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
noteRefusedSendKeys(notTaken)
|
||||||
for _, w := range drop {
|
for _, w := range drop {
|
||||||
if err := c.Reported(ctx, w); err != nil && first == nil {
|
if err := c.Reported(ctx, w); err != nil && first == nil {
|
||||||
first = err
|
first = err
|
||||||
@@ -435,29 +451,49 @@ func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time
|
|||||||
return first
|
return first
|
||||||
}
|
}
|
||||||
|
|
||||||
var silentTimers = struct {
|
// silentCallBound is how long one call to the silent clock may take while the look holds the plans (novox/hq issue 1121).
|
||||||
mu sync.Mutex
|
// The vendored key listing reads its watcher without the context, so a consumer lost when the broker reloaded its user
|
||||||
timers map[silentWait]time.Time
|
// list left the look waiting for good, with the plans and every node held.
|
||||||
}{timers: map[silentWait]time.Time{}}
|
var silentCallBound = 20 * time.Second
|
||||||
|
|
||||||
// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
|
// boundedClock is a clock whose every call is answered within silentCallBound, or fails: a call that never returns is
|
||||||
// (DegradableGrants says why that can be): never silently never. Once for each send.
|
// left behind on its own goroutine.
|
||||||
func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
|
type boundedClock struct{ silentClock }
|
||||||
silentTimers.mu.Lock()
|
|
||||||
defer silentTimers.mu.Unlock()
|
func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) {
|
||||||
if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
|
ctx, cancel := context.WithTimeout(ctx, silentCallBound)
|
||||||
return
|
defer cancel()
|
||||||
|
type answer struct {
|
||||||
|
v T
|
||||||
|
err error
|
||||||
}
|
}
|
||||||
silentTimers.timers[w] = sent
|
done := make(chan answer, 1)
|
||||||
// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
|
go func() { v, err := call(ctx); done <- answer{v, err} }()
|
||||||
time.AfterFunc(ttl, func() {
|
select {
|
||||||
silentTimers.mu.Lock()
|
case a := <-done:
|
||||||
if silentTimers.timers[w].Equal(sent) {
|
return a.v, a.err
|
||||||
delete(silentTimers.timers, w)
|
case <-ctx.Done():
|
||||||
|
var zero T
|
||||||
|
return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err())
|
||||||
}
|
}
|
||||||
silentTimers.mu.Unlock()
|
}
|
||||||
askToLook()
|
|
||||||
|
func (b boundedClock) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
|
||||||
|
_, err := boundedCall(ctx, "arming a wait", func(ctx context.Context) (struct{}, error) {
|
||||||
|
return struct{}{}, b.silentClock.Arm(ctx, w, ttl, sent)
|
||||||
})
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b boundedClock) Held(ctx context.Context) (map[silentWait]time.Time, error) {
|
||||||
|
return boundedCall(ctx, "listing the waits", b.silentClock.Held)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b boundedClock) Reported(ctx context.Context, w silentWait) error {
|
||||||
|
_, err := boundedCall(ctx, "dropping a wait", func(ctx context.Context) (struct{}, error) {
|
||||||
|
return struct{}{}, b.silentClock.Reported(ctx, w)
|
||||||
|
})
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
|
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -159,23 +161,105 @@ func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
|||||||
return nil, errors.New("the bus is down")
|
return nil, errors.New("the bus is down")
|
||||||
}
|
}
|
||||||
|
|
||||||
// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
|
// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by
|
||||||
// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
|
// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error
|
||||||
func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
|
// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4).
|
||||||
|
func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1
|
||||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||||
sent := time.Now().UTC()
|
sent := time.Now().UTC()
|
||||||
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
|
for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{},
|
||||||
err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
|
"no clock": nil} {
|
||||||
if err == nil {
|
t.Run(name, func(t *testing.T) {
|
||||||
|
noteRefusedSendKeys(nil)
|
||||||
|
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||||
|
err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent)
|
||||||
|
if clock != nil && err == nil {
|
||||||
t.Fatal("no error given back")
|
t.Fatal("no error given back")
|
||||||
}
|
}
|
||||||
silentTimers.mu.Lock()
|
got := observeRefusedSendKeys()
|
||||||
armed := silentTimers.timers[w].Equal(sent)
|
if len(got) != 1 {
|
||||||
silentTimers.mu.Unlock()
|
t.Fatalf("observations %+v", got)
|
||||||
if !armed {
|
|
||||||
t.Error("no timer of its own watches the wait")
|
|
||||||
}
|
}
|
||||||
|
o := got[0]
|
||||||
|
if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused ||
|
||||||
|
!strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") {
|
||||||
|
t.Fatalf("the condition does not name the plan and the machine: %+v", o)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is.
|
||||||
|
func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2
|
||||||
|
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||||
|
sent := time.Now().UTC()
|
||||||
|
noteRefusedSendKeys(nil)
|
||||||
|
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||||
|
want := map[silentWait]time.Time{w: sent}
|
||||||
|
if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 {
|
||||||
|
t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys())
|
||||||
|
}
|
||||||
|
clock := &keptClock{}
|
||||||
|
if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !clock.armed[w].Equal(sent) {
|
||||||
|
t.Fatalf("the next look did not write the key again: %v", clock.armed)
|
||||||
|
}
|
||||||
|
if got := observeRefusedSendKeys(); len(got) != 0 {
|
||||||
|
t.Fatalf("the condition stays after the key was written: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type armRefusingClock struct{ silentClock }
|
||||||
|
|
||||||
|
func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||||
|
func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error {
|
||||||
|
return errors.New("the bus refuses the key")
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptClock is a clock that takes every key.
|
||||||
|
type keptClock struct {
|
||||||
|
silentClock
|
||||||
|
armed map[silentWait]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||||
|
func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error {
|
||||||
|
if c.armed == nil {
|
||||||
|
c.armed = map[silentWait]time.Time{}
|
||||||
|
}
|
||||||
|
c.armed[w] = sent
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
|
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
|
||||||
// looked at: the controller follows its own plan-moved (issue 974, round 1).
|
// looked at: the controller follows its own plan-moved (issue 974, round 1).
|
||||||
|
|
||||||
|
// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that
|
||||||
|
// never reaches its end.
|
||||||
|
type hungClock struct{ silentClock }
|
||||||
|
|
||||||
|
func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||||
|
select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it)
|
||||||
|
}
|
||||||
|
|
||||||
|
// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
|
||||||
|
// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition.
|
||||||
|
func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
|
||||||
|
was := silentCallBound
|
||||||
|
silentCallBound = 50 * time.Millisecond
|
||||||
|
t.Cleanup(func() { silentCallBound = was })
|
||||||
|
w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
|
||||||
|
sent := time.Now().UTC()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a clock that never answered was reported as fine")
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the look is still waiting for a clock that never answers")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
||||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
|
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the
|
||||||
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
|
// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts
|
||||||
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
|
// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed.
|
||||||
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||||
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
||||||
var found []string
|
var found []string
|
||||||
@@ -31,11 +31,13 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
text := string(raw)
|
text := string(raw)
|
||||||
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
if write.MatchString(text) {
|
||||||
found = append(found, path+" writes the plan table")
|
found = append(found, path+" writes the plan table")
|
||||||
}
|
}
|
||||||
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
|
||||||
found = append(found, path+" calls writePlan")
|
if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" &&
|
||||||
|
strings.Contains(text, "chain.RolloutsOf") {
|
||||||
|
found = append(found, path+" reaches the rollouts bucket")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
@@ -51,17 +53,26 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
text := string(raw)
|
text := string(raw)
|
||||||
if n := len(write.FindAllString(text, -1)); n != 1 {
|
|
||||||
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
|
|
||||||
}
|
|
||||||
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
|
|
||||||
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
||||||
next := strings.Index(text[save:], "\nfunc ")
|
next := strings.Index(text[save:], "\nfunc ")
|
||||||
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
|
if next < 0 {
|
||||||
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
|
next = len(text) - save
|
||||||
}
|
}
|
||||||
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
|
if !strings.Contains(text[save:save+next], "chainSavePlans(") {
|
||||||
t.Error("SavePlansMoving does not say each plan it wrote")
|
t.Error("SavePlansMoving does not write through chainSavePlans")
|
||||||
|
}
|
||||||
|
chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
chainText := string(chainRaw)
|
||||||
|
at := strings.Index(chainText, "func (i *Inventory) chainSavePlans")
|
||||||
|
end := strings.Index(chainText[at:], "\nfunc ")
|
||||||
|
if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") {
|
||||||
|
t.Error("chainSavePlans does not say each plan it wrote")
|
||||||
|
}
|
||||||
|
if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 {
|
||||||
|
t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -154,8 +154,33 @@ func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bo
|
|||||||
return inventory.Plan{}, false
|
return inventory.Plan{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newerDonePlan is a done plan of the same repository and branch made after this one: the newer commit it delivered
|
||||||
|
// runs now (novox/hq ADR 0451).
|
||||||
|
func newerDonePlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||||
|
var found inventory.Plan
|
||||||
|
var ok bool
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == p.ID || q.State != inventory.PlanDone || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||||
|
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !ok || q.Created.After(found.Created) {
|
||||||
|
found, ok = q, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// passedBy is the refusal of a retry of a plan that a newer done plan of its repository and branch has passed.
|
||||||
|
type passedBy struct{ plan, by, repository, commit string }
|
||||||
|
|
||||||
|
func (e passedBy) Error() string {
|
||||||
|
return fmt.Sprintf("%s was passed by %s, a newer plan of %s (at %s) that is done: retrying it would build and "+
|
||||||
|
"send the older commit over the newer one that runs", e.plan, e.by, e.repository, short(e.commit))
|
||||||
|
}
|
||||||
|
|
||||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error {
|
||||||
switch {
|
switch {
|
||||||
case p.State == inventory.PlanDone:
|
case p.State == inventory.PlanDone:
|
||||||
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||||
@@ -165,12 +190,17 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
|||||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A newer delivery of the branch has passed it** (novox/hq ADR 0451): retrying would build and send the older
|
||||||
|
// commit over the newer one that runs.
|
||||||
|
if q, found := newerDonePlan(p, plans); found {
|
||||||
|
return passedBy{plan: p.ID, by: q.ID, repository: q.Repository, commit: q.Commit}
|
||||||
|
}
|
||||||
if len(failedIn(p)) > 0 {
|
if len(failedIn(p)) > 0 {
|
||||||
if q, found := newerOpenPlan(p, plans); found {
|
if q, found := newerOpenPlan(p, plans); found {
|
||||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||||
}
|
}
|
||||||
return oneWalkAtATime(p, plans)
|
return oneWalkAtATime(p, plans, onPath)
|
||||||
}
|
}
|
||||||
stopped := stoppedRollouts(p)
|
stopped := stoppedRollouts(p)
|
||||||
if len(stopped) == 0 {
|
if len(stopped) == 0 {
|
||||||
@@ -194,20 +224,20 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
|||||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return oneWalkAtATime(p, plans)
|
return oneWalkAtATime(p, plans, onPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq
|
// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq
|
||||||
// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same
|
// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same
|
||||||
// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it
|
// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it
|
||||||
// plans, which already hold their dependents, and the repositories it carries.
|
// plans, which already hold their dependents, and the repositories it carries.
|
||||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error {
|
||||||
mine := footprintOfWalk(p, nil)
|
mine := footprintOfWalk(p, nil)
|
||||||
for _, q := range plans {
|
for _, q := range plans {
|
||||||
if q.ID == p.ID || !q.Open() || q.Release != nil {
|
if q.ID == p.ID || !q.Open() || q.Release != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if ownWalk(p) || ownWalk(q) {
|
if ownWalk(p, onPath) || ownWalk(q, onPath) {
|
||||||
return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+
|
return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+
|
||||||
"retry %s once it ended", q.ID, q.Named(), p.ID)
|
"retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||||
}
|
}
|
||||||
@@ -275,6 +305,32 @@ func goesOnAgain(p *inventory.Plan) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// planReader is what a retry reads the mesh's plans through.
|
||||||
|
type planReader interface {
|
||||||
|
OpenPlans(ctx context.Context) ([]inventory.Plan, error)
|
||||||
|
RecentPlans(ctx context.Context, limit int) ([]inventory.Plan, error)
|
||||||
|
PlansSince(ctx context.Context, since time.Time) ([]inventory.Plan, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// plansForRetry are the plans a retry of p judges itself against.
|
||||||
|
func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inventory.Plan, error) {
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
recent, err := inv.RecentPlans(ctx, 50)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// **Every plan made after this one** (novox/hq ADR 0451 §1): a plan that passed it is seen however many plans
|
||||||
|
// were made since, not only when it is among the newest.
|
||||||
|
later, err := inv.PlansSince(ctx, p.Created)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return append(append(plans, recent...), later...), nil
|
||||||
|
}
|
||||||
|
|
||||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||||
// that tier: what follows is the plan going on as if they had built the first time.
|
// that tier: what follows is the plan going on as if they had built the first time.
|
||||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||||
@@ -288,15 +344,10 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
plans, err := inv.OpenPlans(ctx)
|
plans, err := plansForRetry(ctx, inv, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
recent, err := inv.RecentPlans(ctx, 50)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
plans = append(plans, recent...)
|
|
||||||
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
// **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for
|
||||||
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
// the merge that brought the failure answers them now, and a retried walk would name them twice.
|
||||||
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
// **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped,
|
||||||
@@ -327,7 +378,11 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
|||||||
}
|
}
|
||||||
failed = toRetry(&p, recorded, byID)
|
failed = toRetry(&p, recorded, byID)
|
||||||
}
|
}
|
||||||
if err := retryRefusal(p, plans); err != nil {
|
onPath, err := ownPathNow(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := retryRefusal(p, plans, onPath); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// flagsOf reads a command line's flags (and its first positional after --what-if) into a map.
|
||||||
|
func flagsOf(argv []string) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
for i := 0; i < len(argv); i++ {
|
||||||
|
if !strings.HasPrefix(argv[i], "--") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := strings.TrimPrefix(argv[i], "--")
|
||||||
|
if i+1 < len(argv) && !strings.HasPrefix(argv[i+1], "--") {
|
||||||
|
out[name] = argv[i+1]
|
||||||
|
i++
|
||||||
|
} else {
|
||||||
|
out[name] = "true"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The plan of a change as data from `plans`** (novox/hq ADR 0450 §2, issue 1324 AC1): asked for one change
|
||||||
|
// both ways, `plans` with data and `delivery-plan` answer the same plan in every field.
|
||||||
|
func TestPlansWhatIfAsDataAnswersWhatDeliveryPlanAnswers(t *testing.T) {
|
||||||
|
change := map[string]any{"repository": "novox/mesh-catalog", "paths": "modules/gitea/index.ts,modules/gitea/old.txt",
|
||||||
|
"head": "abc123", "base": "main", "module-dirs": "modules/gitea", "removed": "modules/gitea/old.txt"}
|
||||||
|
with := func(extra map[string]any) map[string]any {
|
||||||
|
out := map[string]any{}
|
||||||
|
for k, v := range change {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
for k, v := range extra {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
plansArgv, err := argvFor("plans", with(map[string]any{"data": "true"}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deliveryArgv, err := argvFor("delivery-plan", change)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if plansArgv[0] != "plans" || !slicesHas(plansArgv, "--data") || !slicesHas(plansArgv, "--what-if") {
|
||||||
|
t.Fatalf("plans with data became %q", strings.Join(plansArgv, " "))
|
||||||
|
}
|
||||||
|
p, d := flagsOf(plansArgv), flagsOf(deliveryArgv)
|
||||||
|
for _, name := range []string{"paths", "head", "base", "module-dirs", "removed"} {
|
||||||
|
if p[name] == "" || p[name] != d[name] {
|
||||||
|
t.Errorf("--%s: plans says %q, delivery-plan %q", name, p[name], d[name])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if p["what-if"] != d["repository"] {
|
||||||
|
t.Errorf("repository: plans says %q, delivery-plan %q", p["what-if"], d["repository"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both commands build their document from the one function, from the same member.
|
||||||
|
const src = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||||
|
gitea := fromRepo("gitea", src, "modules/gitea")
|
||||||
|
gitea.Source.BuiltFrom = "old"
|
||||||
|
gitea.On = []string{"anchor", "laptop"}
|
||||||
|
entries := []inventory.Entry{gitea}
|
||||||
|
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||||
|
member := func(f map[string]string, repository string) orderMember {
|
||||||
|
return changeMember(repository, f["base"], f["head"], splitList(f["paths"]), splitList(f["removed"]),
|
||||||
|
moduleDirsOf(f["module-dirs"]))
|
||||||
|
}
|
||||||
|
a := deliveryPlanDocument(member(p, p["what-if"]), entries, nil, nil, policy)
|
||||||
|
b := deliveryPlanDocument(member(d, d["repository"]), entries, nil, nil, policy)
|
||||||
|
ja, _ := json.Marshal(a)
|
||||||
|
jb, _ := json.Marshal(b)
|
||||||
|
if !reflect.DeepEqual(ja, jb) {
|
||||||
|
t.Errorf("the two plans differ:\n%s\n%s", ja, jb)
|
||||||
|
}
|
||||||
|
plan := a["plan"].(interface{})
|
||||||
|
raw, _ := json.Marshal(plan)
|
||||||
|
var fields map[string]any
|
||||||
|
_ = json.Unmarshal(raw, &fields)
|
||||||
|
for _, want := range []string{"moved", "tiers", "machines", "summary", "class", "head", "base", "repository"} {
|
||||||
|
if _, ok := fields[want]; !ok {
|
||||||
|
t.Errorf("the plan has no %q: %s", want, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fields["head"] != "abc123" || fields["base"] != "main" {
|
||||||
|
t.Errorf("head and base are %v and %v", fields["head"], fields["base"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without the data form the text answer is unchanged (AC2): no data flag, and the data-only arguments are not passed.
|
||||||
|
func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) {
|
||||||
|
argv, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a,b"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a,b" {
|
||||||
|
t.Errorf("the text what-if reads %q", got)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"}); err == nil {
|
||||||
|
t.Error("a head without data was passed over, not refused")
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "data": "false"})
|
||||||
|
if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a" {
|
||||||
|
t.Errorf("data false reads %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// delivery-plan stays and answers as before (AC3, ADR 0450 §8).
|
||||||
|
func TestDeliveryPlanStaysAsItWas(t *testing.T) {
|
||||||
|
argv, err := argvFor("delivery-plan", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := strings.Join(argv, " "); got != "delivery plan --repository novox/x --paths a --head h" {
|
||||||
|
t.Errorf("delivery-plan reads %q", got)
|
||||||
|
}
|
||||||
|
if _, ok := catalogueVerb("delivery-plan"); !ok {
|
||||||
|
t.Error("delivery-plan is gone")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func catalogueVerb(name string) (catalogue.Verb, bool) { return controllerVerb(name) }
|
||||||
|
|
||||||
|
func slicesHas(s []string, v string) bool {
|
||||||
|
for _, x := range s {
|
||||||
|
if x == v {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -629,7 +629,7 @@ func heardMachines(d *doctor) map[string]bool {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
for _, m := range f.machines {
|
for _, m := range f.machines {
|
||||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
if !m.lastHeard.IsZero() && !m.silent(f.now) && !m.asleep() {
|
||||||
out[m.name] = true
|
out[m.name] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// review round 3, fault B (1181): the test asserted a literal url, while issueOnTheNewBus works out the
|
||||||
|
// machine's reachable address from the catalogue's network records (brokerReachableAt), not from this
|
||||||
|
// process's environment — the real try found "nats://anchor.internal:5671", not the literal guessed here.
|
||||||
|
// Fixed by calling issueProcess directly, with the known broker and reachable address given to it, exactly
|
||||||
|
// as issued_to_vault_test.go's issue/issueWith and issueProvisionerClient are called directly; the url
|
||||||
|
// asserted below is one of those two inputs, never a separate guess.
|
||||||
|
//
|
||||||
|
// review round 1, findings 3 and 4 (1181/AC1, AC2): module issue mints and delivers a process's own
|
||||||
|
// account against the store — no bus to dial, busAddress "" — for mesh-mail's real manifest shape (review
|
||||||
|
// round 1 finding 1: no own secret `broker` at all, its tools served by the tool runner), and issuing it
|
||||||
|
// again is the path every bus account takes (`rollout mint`'s KindProcess case, and a re-run of `module
|
||||||
|
// issue`): it rotates the password, and the old hash no longer matches.
|
||||||
|
func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
theVaultIsInstalled(t, open)
|
||||||
|
|
||||||
|
m := mailSenderWithProcess()
|
||||||
|
m.Moved = true // so handIssuedToVault's accountIsCovered is true, and the hand-off is exercised whole
|
||||||
|
register(t, open, m)
|
||||||
|
|
||||||
|
vault := newStandInVault(t)
|
||||||
|
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||||
|
t.Cleanup(func() { vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } })
|
||||||
|
|
||||||
|
pr, found := processPrincipalFor(m, "send")
|
||||||
|
if !found {
|
||||||
|
t.Fatal("mesh-mail declares no process send")
|
||||||
|
}
|
||||||
|
known := broker.Broker{Fingerprint: "ab"}
|
||||||
|
const reachable = "10.0.0.1:4222"
|
||||||
|
issue := func() {
|
||||||
|
t.Helper()
|
||||||
|
if err := issueProcess(ctx, inv, m, pr, "laptop", "", known, reachable); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
issue()
|
||||||
|
user := broker.Principal{Kind: broker.KindProcess, Node: "laptop", Module: "mesh-mail", Process: "send"}.Username()
|
||||||
|
if user != "laptop.mesh-mail.send" {
|
||||||
|
t.Fatalf("the process's username: %q", user)
|
||||||
|
}
|
||||||
|
hash1, held, err := inv.BusUserHash(ctx, user)
|
||||||
|
if err != nil || !held || hash1 == "" {
|
||||||
|
t.Fatalf("the process's account was not minted: held %v, %v", held, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1181/AC1: accepted with the provisioner client's JSON, and handed to the vault whole — the url and
|
||||||
|
// the fingerprint being exactly what issueProcess was given above, not a guess at what it computes.
|
||||||
|
taken := vault.taken["accept"]
|
||||||
|
for _, want := range []string{`"url":"nats://` + reachable + `"`, `"fingerprint":"ab"`, `"node":"laptop"`,
|
||||||
|
`"module":"mesh-mail"`, `"user":"laptop.mesh-mail.send"`, `"password"`} {
|
||||||
|
if !strings.Contains(taken, want) {
|
||||||
|
t.Errorf("the vault was handed %q, without %s", taken, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(vault.said(), `"name":"broker-send"`) {
|
||||||
|
t.Errorf("the vault was not told the secret's name broker-send: %s", vault.said())
|
||||||
|
}
|
||||||
|
// The file's own owner (mesh-mail-send, review round 1 finding 7's area) is 1170's mechanism, proven
|
||||||
|
// generically there and for this fixture in TestAProcessesOwnSecretIsOwnedByItsOwnAccount.
|
||||||
|
|
||||||
|
// 1181/AC2: issued again — the path `rollout mint`'s KindProcess case and a re-run of `module issue`
|
||||||
|
// both take — rotates the password; the old hash no longer matches what is recorded.
|
||||||
|
issue()
|
||||||
|
hash2, held, err := inv.BusUserHash(ctx, user)
|
||||||
|
if err != nil || !held {
|
||||||
|
t.Fatalf("the process's account is gone after a re-issue: held %v, %v", held, err)
|
||||||
|
}
|
||||||
|
if hash1 == hash2 {
|
||||||
|
t.Fatal("issuing the process's account again did not rotate its password")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mailSenderWithProcess is mesh-mail's real manifest shape (novox/hq ADR 0436 §7, §8, the lead's fixture):
|
||||||
|
// no own secret named `broker` at all — its tools run in the machine's tool runner — and the process "send"
|
||||||
|
// declares bus, whose own secret broker-send is where module issue delivers its account (ADR 0436 §2),
|
||||||
|
// owned by mesh-mail-send, the user resource the process declares, at mode 0600 as every own secret is.
|
||||||
|
func mailSenderWithProcess() catalogue.Manifest {
|
||||||
|
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1","emits":["mail.send.sent"],
|
||||||
|
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||||
|
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
||||||
|
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
||||||
|
"bus":{"emits":["mail.send.sent"]}}]}`))
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// review round 1, finding 1 (1181): a module declaring no `broker` of its own at all — mesh-mail's real
|
||||||
|
// shape, tools served by the tool runner — still passes mayIssue on its process's bus declaration alone.
|
||||||
|
func TestAModuleWithNoBrokerAndOnlyAProcessMayBeIssued(t *testing.T) {
|
||||||
|
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
||||||
|
t.Fatalf("a module with no broker of its own, whose process declares bus, was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1181/AC1: a process with a bus declaration and no own secret broker-<process id> cannot be issued — the
|
||||||
|
// same refusal mayIssue already gives a module with no `broker` (novox/hq ADR 0436 §2).
|
||||||
|
func TestAProcessWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
||||||
|
m := mailSenderWithProcess()
|
||||||
|
delete(m.OwnSecrets, "broker-send")
|
||||||
|
err := mayIssue(m)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a process with no broker-<id> own secret was issued an account")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"mesh-mail", "send", "broker-send", "own-secrets"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
||||||
|
t.Errorf("a process declaring its own broker-<id> secret was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1181/AC1: the account held for a process is the provisioner client's form — {url, fingerprint, node,
|
||||||
|
// module, user, password} — named for the process, never the module's own account.
|
||||||
|
func TestAProcesssHeldAccountIsTheProvisionersForm(t *testing.T) {
|
||||||
|
user := broker.Principal{Kind: broker.KindProcess, Node: "novox", Module: "mesh-mail", Process: "send"}.Username()
|
||||||
|
if user != "novox.mesh-mail.send" {
|
||||||
|
t.Fatalf("the process's username: %q", user)
|
||||||
|
}
|
||||||
|
held, err := heldProcessAccount(broker.Broker{Fingerprint: "abc"}, "novox.example:4222", "novox", "mesh-mail",
|
||||||
|
user, "a-password")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(held, &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := map[string]any{"url": "nats://novox.example:4222", "fingerprint": "abc", "node": "novox",
|
||||||
|
"module": "mesh-mail", "user": "novox.mesh-mail.send", "password": "a-password"}
|
||||||
|
for k, v := range want {
|
||||||
|
if got[k] != v {
|
||||||
|
t.Errorf("%s: %v, want %v", k, got[k], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Errorf("the held account carries %v; want exactly %v (never claims — a process holds no seat)", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1181/AC1: composed, the process's own secret belongs to its own account at 0600 (novox/hq ADR 0436 §1, §2;
|
||||||
|
// the owner mechanism is issue 1170's, proven generically there — this is mesh-mail's own shape).
|
||||||
|
func TestAProcessesOwnSecretIsOwnedByItsOwnAccount(t *testing.T) {
|
||||||
|
m := mailSenderWithProcess()
|
||||||
|
r := catalogue.Resolution{Node: "novox", Modules: []catalogue.Manifest{m}}
|
||||||
|
out, err := r.Declaration(catalogue.Rendering{
|
||||||
|
Needed: map[string]map[string]string{"mesh-mail": {"broker-send": "sealed-account"}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("composing the declaration: %v", err)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, res := range out {
|
||||||
|
if res["type"] == "file" && res["path"] == "/var/lib/mesh-mail/broker-send" {
|
||||||
|
found = true
|
||||||
|
if got := res["owner"]; got != "mesh-mail-send" {
|
||||||
|
t.Fatalf("broker-send's owner: %v; want mesh-mail-send", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("broker-send was not composed at all")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats-server/v2/server"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mailSenderLiveManifest is mesh-mail's future manifest shape whole (novox/hq ADR 0436 §7, §8): the
|
||||||
|
// process "send" declares every clause of bus (emits, its one work-queue stream, its state, asks, and its
|
||||||
|
// invoked seat verb), so the account module issue delivers for it can be put through every grant at once.
|
||||||
|
func mailSenderLiveManifest(t *testing.T) catalogue.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
|
||||||
|
"emits":["mail.send.sent","mail.send.failed"],"state":["sends"],
|
||||||
|
"streams":[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":8,
|
||||||
|
"retention":"work-queue","max-deliveries":5}],
|
||||||
|
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||||
|
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
||||||
|
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
||||||
|
"bus":{"emits":["mail.send.sent","mail.send.failed"],"queues":["mail-sends"],"state":["sends"],
|
||||||
|
"asks":true,"invokes":["seat:operator-channel.tell"]}}]}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// operatorChannelWithTell is operator-channel as messenger declares it (broker.operatorChannel, novox/hq
|
||||||
|
// ADR 0259 §3), with the one verb this release adds: `tell` (novox/hq ADR 0436 §1's own fixture), so the
|
||||||
|
// process's invoked seat verb has a holder to answer it.
|
||||||
|
func operatorChannelWithTell() broker.Seat {
|
||||||
|
return broker.Seat{Name: "operator-channel", Scope: "mesh", Accepts: []string{"ask", "cancel"},
|
||||||
|
Emits: []string{"decided"}, Serves: []string{"open", "history", "notify", "tell"},
|
||||||
|
ByCaller: []string{"ask", "cancel", "decided"}, Records: []string{"messenger_asks"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// review round 1, finding 5 (1181/AC3), the release's acceptance test, on a real bus and the store: the
|
||||||
|
// account module issue delivers for mesh-mail's process "send" — composed from the manifest the real way
|
||||||
|
// (inventory.ProcessesOf -> broker.Users -> broker.ComposeAccounts), minted with the password module issue
|
||||||
|
// mints (aMesh) — does exactly what its bus declares and nothing else.
|
||||||
|
func TestAProcessAccountOnARealBusAndStore(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
m := mailSenderLiveManifest(t)
|
||||||
|
operatorChannel := operatorChannelWithTell()
|
||||||
|
mailSends := broker.ModuleStream{Module: "mesh-mail", Name: "MAIL_SENDS", Events: []string{"mail.send.approved"},
|
||||||
|
MaxAgeDays: 1, MaxMiB: 8, WorkQueue: true, Duplicates: 1}
|
||||||
|
records := broker.Records{Nodes: []string{"laptop"}, Assigned: map[string][]broker.Declared{"laptop": {
|
||||||
|
{Module: broker.RuntimeModule},
|
||||||
|
{Module: "messenger", Holds: []broker.Seat{operatorChannel}, RunsAs: "messenger"},
|
||||||
|
{Module: "mesh-mail", NoAccount: true, Uses: []broker.Seat{operatorChannel}, Streams: []broker.ModuleStream{mailSends},
|
||||||
|
Processes: inventory.ProcessesOf(m)},
|
||||||
|
}}}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The four principals this test drives, each minted the password module issue mints — the same
|
||||||
|
// inv.MintBusPassword call issueProcess makes for the process account. mesh-mail publishes its own
|
||||||
|
// stream's events as the tool runner carrying it (review round 2's finding): the controller's own
|
||||||
|
// grants do not cover a module's stream, only the runtime that carries the module does.
|
||||||
|
var admin, runtime, messenger, sender broker.Principal
|
||||||
|
for _, p := range users {
|
||||||
|
switch {
|
||||||
|
case p.Kind == broker.KindController:
|
||||||
|
admin = p
|
||||||
|
case p.Kind == broker.KindNodeTools:
|
||||||
|
runtime = p
|
||||||
|
case p.Module == "messenger":
|
||||||
|
messenger = p
|
||||||
|
case p.Kind == broker.KindProcess:
|
||||||
|
sender = p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if admin.Kind == "" || runtime.Kind == "" || messenger.Kind == "" || sender.Kind == "" {
|
||||||
|
t.Fatalf("not every principal was composed: admin=%+v runtime=%+v messenger=%+v sender=%+v",
|
||||||
|
admin, runtime, messenger, sender)
|
||||||
|
}
|
||||||
|
if sender.Username() != "laptop.mesh-mail.send" {
|
||||||
|
t.Fatalf("the process's username: %q", sender.Username())
|
||||||
|
}
|
||||||
|
|
||||||
|
passwords := map[string]string{}
|
||||||
|
hashes := map[string]string{}
|
||||||
|
for _, p := range []struct {
|
||||||
|
principal broker.Principal
|
||||||
|
kind string
|
||||||
|
}{{admin, inventory.BusController}, {runtime, inventory.BusNodeTools}, {messenger, inventory.BusModule},
|
||||||
|
{sender, inventory.BusProcess}} {
|
||||||
|
pw, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.principal.Username(), Kind: p.kind,
|
||||||
|
Node: p.principal.Node, Module: p.principal.Module})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
passwords[p.principal.Username()] = pw
|
||||||
|
hash, held, err := inv.BusUserHash(ctx, p.principal.Username())
|
||||||
|
if err != nil || !held {
|
||||||
|
t.Fatalf("the minted password of %s was not recorded: %v, %v", p.principal.Username(), held, err)
|
||||||
|
}
|
||||||
|
hashes[p.principal.Username()] = hash
|
||||||
|
}
|
||||||
|
filled, missing := broker.WithPasswords([]broker.Principal{admin, runtime, messenger, sender}, hashes)
|
||||||
|
if len(missing) > 0 {
|
||||||
|
t.Fatalf("no password for %v", missing)
|
||||||
|
}
|
||||||
|
|
||||||
|
accounts, err := broker.ComposeAccounts(filled)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||||
|
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||||
|
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
opts, err := server.ProcessConfigFile(conf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||||
|
}
|
||||||
|
opts.NoLog, opts.NoSigs = true, true
|
||||||
|
s, err := server.NewServer(opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go s.Start()
|
||||||
|
if !s.ReadyForConnections(10 * time.Second) {
|
||||||
|
t.Fatal("the bus did not come up")
|
||||||
|
}
|
||||||
|
defer s.Shutdown()
|
||||||
|
|
||||||
|
// Every connection names its own inbox prefix: the grant is `_INBOX.<username>.>`, and the client
|
||||||
|
// library's own default random inbox matches nothing granted, so replies are refused without it.
|
||||||
|
adminConn, err := broker.Dial(s.ClientURL(), nats.UserInfo(admin.Username(), passwords[admin.Username()]),
|
||||||
|
nats.CustomInboxPrefix("_INBOX."+admin.Username()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer adminConn.Close()
|
||||||
|
if err := adminConn.EnsureStream(mailSends.Stream()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := adminConn.EnsureConsumer(broker.QueueConsumer(sender, sender.Queues[0])); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := adminConn.EnsureBucket(broker.Bucket{Module: "mesh-mail", Name: "sends"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
js, err := jetstream.New(adminConn.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "messenger_asks"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The machine's tool runner, which carries mesh-mail (it has no account of its own) and so is the
|
||||||
|
// principal that publishes its own stream's events on its behalf.
|
||||||
|
runtimeConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(runtime.Username(), passwords[runtime.Username()]),
|
||||||
|
nats.CustomInboxPrefix("_INBOX."+runtime.Username()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer runtimeConn.Close()
|
||||||
|
|
||||||
|
// messenger, the router, answers exactly its granted verbs — never a wildcard the grant does not give it.
|
||||||
|
holder, err := nats.Connect(s.ClientURL(), nats.UserInfo(messenger.Username(), passwords[messenger.Username()]),
|
||||||
|
nats.CustomInboxPrefix("_INBOX."+messenger.Username()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer holder.Close()
|
||||||
|
heard := make(chan string, 1)
|
||||||
|
for _, verb := range []string{"open", "history", "notify", "tell"} {
|
||||||
|
if _, err := holder.Subscribe("mesh.seat.operator-channel.tool."+verb, func(m *nats.Msg) {
|
||||||
|
heard <- m.Subject
|
||||||
|
_ = m.Respond([]byte(`{"result":{}}`))
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = holder.Flush()
|
||||||
|
|
||||||
|
refusals := make(chan error, 8)
|
||||||
|
sender2, err := nats.Connect(s.ClientURL(), nats.UserInfo(sender.Username(), passwords[sender.Username()]),
|
||||||
|
nats.CustomInboxPrefix("_INBOX."+sender.Username()),
|
||||||
|
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer sender2.Close()
|
||||||
|
|
||||||
|
// 1. Its event.
|
||||||
|
if err := sender2.Publish("mesh.mod.mesh-mail.event.mail.send.sent", []byte("{}")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Its state: writes and reads its own bucket.
|
||||||
|
senderKV, err := jetstream.New(sender2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bucket, err := senderKV.KeyValue(ctx, "mesh-mail_sends")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := bucket.Put(ctx, "key", []byte("value")); err != nil {
|
||||||
|
t.Fatalf("the process may not write its own state: %v", err)
|
||||||
|
}
|
||||||
|
if got, err := bucket.Get(ctx, "key"); err != nil || string(got.Value()) != "value" {
|
||||||
|
t.Fatalf("the process may not read its own state back: %v, %v", got, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Its queue: pulls and acknowledges MAIL_SENDS through its own consumer. Published as the tool
|
||||||
|
// runner carrying mesh-mail — the live path a module's own stream's event takes — never as the
|
||||||
|
// controller, whose own grants do not cover a module's stream at all.
|
||||||
|
runtimeJS, err := jetstream.New(runtimeConn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := runtimeJS.Publish(ctx, broker.ModuleStreamSubject("mesh-mail", "mail.send.approved"), []byte("{}")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reading, err := jetstream.New(sender2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
consumer, err := reading.Consumer(ctx, "MAIL_SENDS", broker.QueueConsumer(sender, sender.Queues[0]).Name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the process may not bind its own queue consumer: %v", err)
|
||||||
|
}
|
||||||
|
batch, err := consumer.Fetch(1, jetstream.FetchMaxWait(3*time.Second))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := false
|
||||||
|
for msg := range batch.Messages() {
|
||||||
|
got = true
|
||||||
|
if err := msg.Ack(); err != nil {
|
||||||
|
t.Fatalf("the process may not acknowledge its own queue: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !got {
|
||||||
|
t.Fatal("the process did not pull the message its own queue holds")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Its invoked seat verb, in its own name.
|
||||||
|
if _, err := sender2.Request("mesh.seat.operator-channel.tool.tell", []byte(`{}`), 3*time.Second); err != nil {
|
||||||
|
t.Fatalf("the process may not call operator-channel.tell: %v", err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case s := <-heard:
|
||||||
|
if s != "mesh.seat.operator-channel.tool.tell" {
|
||||||
|
t.Fatalf("messenger heard %s", s)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("messenger never heard the call")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Refused, one at a time, so each refusal is counted against the attempt that caused it rather than
|
||||||
|
// any other: answering a tool call it holds no seat and serves no tool for.
|
||||||
|
if _, err := sender2.Subscribe("mesh.mod.mesh-mail.tool.>", func(*nats.Msg) {}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = sender2.Flush()
|
||||||
|
select {
|
||||||
|
case err := <-refusals:
|
||||||
|
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||||
|
t.Errorf("subscribing a tool call was not refused as a permissions violation: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Error("subscribing a tool call the process does not serve was not refused")
|
||||||
|
}
|
||||||
|
// 6. Refused: publishing a warrant — only the holder, messenger, may say operator-channel decided.
|
||||||
|
if err := sender2.Publish("mesh.seat.operator-channel.event.decided.mesh-mail", []byte("{}")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = sender2.Flush()
|
||||||
|
select {
|
||||||
|
case err := <-refusals:
|
||||||
|
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||||
|
t.Errorf("publishing a warrant was not refused as a permissions violation: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Error("publishing a warrant the process does not hold the seat for was not refused")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -209,7 +209,8 @@ func serve(ctx context.Context) (err error) {
|
|||||||
}
|
}
|
||||||
// And the merges mesh-delivery made, kept a week with the ask each was made on: what registers a module that
|
// And the merges mesh-delivery made, kept a week with the ask each was made on: what registers a module that
|
||||||
// moved between repositories on the operator's Approve (novox/hq ADR 0420 §7, issue 1025).
|
// moved between repositories on the operator's Approve (novox/hq ADR 0420 §7, issue 1025).
|
||||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: open.inventory.Catalogued,
|
||||||
|
kept: following{open: open}.resumeFirstBuilds}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable
|
// No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
|||||||
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||||
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||||
} {
|
} {
|
||||||
err := retryRefusal(c.p, c.others)
|
err := retryRefusal(c.p, c.others, ownPathOf(nil))
|
||||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||||
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||||
}
|
}
|
||||||
@@ -76,20 +76,23 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
|||||||
stopped := failed
|
stopped := failed
|
||||||
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||||
stopped.Note = "a stopped at its first machine"
|
stopped.Note = "a stopped at its first machine"
|
||||||
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
if err := retryRefusal(stopped, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||||
}
|
}
|
||||||
// A failed or done plan does not supersede it.
|
// A later failed plan does not supersede it; a later done one passes it (novox/hq ADR 0451).
|
||||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour)),
|
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}, ownPathOf(nil)); err != nil {
|
||||||
plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}); err != nil {
|
|
||||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := retryRefusal(failed, []inventory.Plan{plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}, ownPathOf(nil)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "plan-5") {
|
||||||
|
t.Errorf("a plan a later done plan passed was retried: %v", err)
|
||||||
|
}
|
||||||
// Another branch's open walk, or an older one, does not supersede it, and is one walk open: one at a time
|
// Another branch's open walk, or an older one, does not supersede it, and is one walk open: one at a time
|
||||||
// (novox/hq ADR 0276).
|
// (novox/hq ADR 0276).
|
||||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||||
other.Branch = "release"
|
other.Branch = "release"
|
||||||
for _, open := range []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour))} {
|
for _, open := range []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour))} {
|
||||||
if err := retryRefusal(failed, []inventory.Plan{open}); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
if err := retryRefusal(failed, []inventory.Plan{open}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
||||||
t.Errorf("retried beside the open walk %s: %v", open.ID, err)
|
t.Errorf("retried beside the open walk %s: %v", open.ID, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,7 +44,6 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
|||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||||
recordmirror.Assignments.Name: inv.AssignmentsMirrorToWrite,
|
|
||||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||||
}
|
}
|
||||||
@@ -55,15 +54,11 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
|||||||
events := &assignmentEvents{}
|
events := &assignmentEvents{}
|
||||||
for {
|
for {
|
||||||
if holds() {
|
if holds() {
|
||||||
// The assignments, once they are keys of the bus, are written by the verbs that change them (novox/hq ADR
|
// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
|
||||||
// 0408): this tick keeps the provisioner's plan in their values and says what changed; the mirror of the store
|
// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
|
||||||
// writes them only until the move.
|
keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
||||||
onTheBus := keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
|
||||||
for _, m := range recordmirror.Mirrors() {
|
for _, m := range recordmirror.Mirrors() {
|
||||||
if m.Name == recordmirror.Assignments.Name && onTheBus {
|
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||||
continue
|
|
||||||
}
|
|
||||||
why := mirrorOnce(ctx, inv, api, m, writers, sources[m.Name], announce, epoch, holds)
|
|
||||||
if why != failing[m.Name] {
|
if why != failing[m.Name] {
|
||||||
if why != "" {
|
if why != "" {
|
||||||
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
|
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
|
||||||
@@ -83,7 +78,7 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
|||||||
}
|
}
|
||||||
|
|
||||||
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
|
// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could.
|
||||||
func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer,
|
||||||
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
|
source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64,
|
||||||
holds func() bool) string {
|
holds func() bool) string {
|
||||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
@@ -95,9 +90,6 @@ func mirrorOnce(ctx context.Context, inv *inventory.Inventory, api jetstream.Jet
|
|||||||
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
|
return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err)
|
||||||
}
|
}
|
||||||
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
|
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
|
||||||
if m.Name == recordmirror.Assignments.Name {
|
|
||||||
w.Holds = assignmentsHolds(inv, holds)
|
|
||||||
}
|
|
||||||
writers[m.Name] = w
|
writers[m.Name] = w
|
||||||
}
|
}
|
||||||
want, err := source(reading)
|
want, err := source(reading)
|
||||||
@@ -128,21 +120,10 @@ type assignmentEvents struct {
|
|||||||
// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
|
// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
|
||||||
// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
|
// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
|
||||||
func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
|
func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
|
||||||
events *assignmentEvents, failing map[string]string) bool {
|
events *assignmentEvents, failing map[string]string) {
|
||||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
const name = "assignments"
|
const name = "assignments"
|
||||||
if _, on, err := inv.KeptAssignments(reading); err != nil || !on {
|
|
||||||
if err != nil {
|
|
||||||
// Not known to be off the bus: the store's mirror is not written over what the bus may hold.
|
|
||||||
if failing[name] == "" {
|
|
||||||
failing[name] = err.Error()
|
|
||||||
fmt.Printf("the assignments cannot be read from the bus: %v\n", err)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
|
if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
|
||||||
if failing[name] != err.Error() {
|
if failing[name] != err.Error() {
|
||||||
failing[name] = err.Error()
|
failing[name] = err.Error()
|
||||||
@@ -152,9 +133,9 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
|
|||||||
fmt.Printf("the provisioner's plans are kept in the assignments again\n")
|
fmt.Printf("the provisioner's plans are kept in the assignments again\n")
|
||||||
failing[name] = ""
|
failing[name] = ""
|
||||||
}
|
}
|
||||||
now, _, err := inv.KeptAssignments(reading)
|
now, err := inv.KeptAssignments(reading)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return true
|
return
|
||||||
}
|
}
|
||||||
if events.known != nil {
|
if events.known != nil {
|
||||||
keys := make([]string, 0, len(now)+len(events.known))
|
keys := make([]string, 0, len(now)+len(events.known))
|
||||||
@@ -194,12 +175,4 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
events.known = now
|
events.known = now
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// assignmentsHolds is the assignments mirror's leave to write: the lease's, and the assignments neither being switched to
|
|
||||||
// the bus nor on it. Asked by the writer before every write and purge, so a pass that read its source before the switch
|
|
||||||
// writes nothing after it (ADR 0408 §4).
|
|
||||||
func assignmentsHolds(inv *inventory.Inventory, holds func() bool) func() bool {
|
|
||||||
return func() bool { return holds() && inv.AssignmentsMirrorMayWrite() }
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
|
||||||
"github.com/novox/mesh-controller/internal/recordmirror"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A writer of the store's mirror of the assignments, made before the assignments are switched to the bus, writes and
|
|
||||||
// purges nothing after** (ADR 0408 §4, issue 1093): its leave to write is asked before every write, and false once they are
|
|
||||||
// switching or moved.
|
|
||||||
func TestAMirrorWriterMadeBeforeTheSwitchWritesAndPurgesNothingAfter(t *testing.T) {
|
|
||||||
inv := inventory.NewForTest()
|
|
||||||
w := &recordmirror.Writer{Mirror: recordmirror.Assignments, Holds: assignmentsHolds(inv, func() bool { return true })}
|
|
||||||
if !w.Holds() {
|
|
||||||
t.Fatal("the writer may not write before the switch")
|
|
||||||
}
|
|
||||||
inv.BeginSwitchForTest(inventory.KindAssignments)
|
|
||||||
if w.Holds() {
|
|
||||||
t.Fatal("the writer may write while the assignments are switching")
|
|
||||||
}
|
|
||||||
// Refused before it reads the bucket (it has none here) or lays or ends a key.
|
|
||||||
r, err := w.Sync(t.Context(), map[string][]byte{"anchor.notes": []byte(`{}`)})
|
|
||||||
if !errors.Is(err, recordmirror.ErrNotHeld) || r.Written != 0 || r.Removed != 0 {
|
|
||||||
t.Fatalf("the writer wrote after the switch: %+v %v", r, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -144,12 +144,12 @@ func relatedOpen(cand footprint, candOwn bool, open []openWalk, edges []inventor
|
|||||||
|
|
||||||
// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path
|
// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path
|
||||||
// (issue 973).
|
// (issue 973).
|
||||||
func ownWalk(p inventory.Plan) bool {
|
func ownWalk(p inventory.Plan, onPath map[string]string) bool {
|
||||||
if p.OwnPath() {
|
if p.OwnPath() {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
for name := range p.Modules {
|
for name := range p.Modules {
|
||||||
if _, own := onTheControllersPath[name]; own {
|
if _, own := onPath[name]; own {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -173,19 +173,19 @@ func startedWalks(plans []inventory.Plan, id string, created time.Time) []invent
|
|||||||
|
|
||||||
// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the
|
// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the
|
||||||
// merges the record holds for it.
|
// merges the record holds for it.
|
||||||
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (openWalk, error) {
|
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan, onPath map[string]string) (openWalk, error) {
|
||||||
kept, err := inv.MergesOf(ctx, p.ID)
|
kept, err := inv.MergesOf(ctx, p.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return openWalk{}, err
|
return openWalk{}, err
|
||||||
}
|
}
|
||||||
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p)}, nil
|
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p, onPath)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// openWalksOf are openWalkOf for each of plans.
|
// openWalksOf are openWalkOf for each of plans.
|
||||||
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) ([]openWalk, error) {
|
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan, onPath map[string]string) ([]openWalk, error) {
|
||||||
var out []openWalk
|
var out []openWalk
|
||||||
for _, p := range plans {
|
for _, p := range plans {
|
||||||
w, err := openWalkOf(ctx, inv, p)
|
w, err := openWalkOf(ctx, inv, p, onPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -119,19 +119,19 @@ func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) {
|
|||||||
|
|
||||||
// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973).
|
// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973).
|
||||||
func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) {
|
func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) {
|
||||||
if ownWalk(walkOf("novox/one", "notes")) {
|
if ownWalk(walkOf("novox/one", "notes"), ownPathOf(nil)) {
|
||||||
t.Error("an ordinary walk read as on the controller's own path")
|
t.Error("an ordinary walk read as on the controller's own path")
|
||||||
}
|
}
|
||||||
flagged := walkOf("novox/one", "notes")
|
flagged := walkOf("novox/one", "notes")
|
||||||
flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||||
if !ownWalk(flagged) {
|
if !ownWalk(flagged, ownPathOf(nil)) {
|
||||||
t.Error("a walk whose batch is own-path read as ordinary")
|
t.Error("a walk whose batch is own-path read as ordinary")
|
||||||
}
|
}
|
||||||
var module string
|
var module string
|
||||||
for m := range onTheControllersPath {
|
for m := range onTheControllersPath {
|
||||||
module = m
|
module = m
|
||||||
}
|
}
|
||||||
if !ownWalk(walkOf("novox/one", "notes", module)) {
|
if !ownWalk(walkOf("novox/one", "notes", module), ownPathOf(nil)) {
|
||||||
t.Errorf("a walk planning %s read as ordinary", module)
|
t.Errorf("a walk planning %s read as ordinary", module)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -168,15 +168,15 @@ func TestARetryWaitsOnlyForAnOpenWalkItIsRelatedTo(t *testing.T) {
|
|||||||
p.ID, p.State = id, inventory.PlanBuilding
|
p.ID, p.State = id, inventory.PlanBuilding
|
||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err != nil {
|
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err != nil {
|
||||||
t.Errorf("refused beside an unrelated walk: %v", err)
|
t.Errorf("refused beside an unrelated walk: %v", err)
|
||||||
}
|
}
|
||||||
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")})
|
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")}, ownPathOf(nil))
|
||||||
if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") {
|
if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") {
|
||||||
t.Errorf("a retry beside a walk moving the same module: %v", err)
|
t.Errorf("a retry beside a walk moving the same module: %v", err)
|
||||||
}
|
}
|
||||||
failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err == nil {
|
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err == nil {
|
||||||
t.Error("an own-path retry went beside an open walk")
|
t.Error("an own-path retry went beside an open walk")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1589,7 +1589,9 @@ func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalog
|
|||||||
var other []string
|
var other []string
|
||||||
listID := ""
|
listID := ""
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
if _, was := sent[m.Module]; !was {
|
// A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent"
|
||||||
|
// only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment.
|
||||||
|
if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" {
|
||||||
other = append(other, m.Module+" newly assigned")
|
other = append(other, m.Module+" newly assigned")
|
||||||
}
|
}
|
||||||
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
|
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -250,6 +251,68 @@ func appliedEach(module string, since func(node string) time.Time, running []str
|
|||||||
return len(waiting) == 0, waiting
|
return len(waiting) == 0, waiting
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// gateNeed is a module the next tier waits for: the machines running it, and from when each must have reported.
|
||||||
|
type gateNeed struct {
|
||||||
|
module string
|
||||||
|
running []string
|
||||||
|
since func(node string) time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentNodes is the machines a walk sent and heard nothing from past ApplySilentAfter, on whichever tier it sent
|
||||||
|
// them (novox/hq ADR 0360 §5, issue 1160): the one rule both the gates between tiers and the walk's end read, through
|
||||||
|
// walkReports. A machine that reports on its send is not silent from then on.
|
||||||
|
func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[string]bool {
|
||||||
|
_, _, silent := walkReports(p, reports, now, prov)
|
||||||
|
out := map[string]bool{}
|
||||||
|
for _, n := range silent {
|
||||||
|
out[n] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// withoutSilent is the machines of running that are not silent: a later tier is not sent to a machine the walk heard
|
||||||
|
// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §5).
|
||||||
|
func withoutSilent(running []string, silent map[string]bool) []string {
|
||||||
|
return slices.DeleteFunc(slices.Clone(running), func(n string) bool { return silent[n] })
|
||||||
|
}
|
||||||
|
|
||||||
|
// settleTierGate is the wait between tiers on the reports of the machines running what the next tier needs: what is
|
||||||
|
// still waited for ("module on a, b"), the silent machines left out of the wait, sorted, and whether every machine the
|
||||||
|
// gate waits on is silent, so that nothing it needs was delivered anywhere (issue 1160).
|
||||||
|
func settleTierGate(needs []gateNeed, reports []inventory.Reported, silent map[string]bool) (waiting, quiet []string, lost bool) {
|
||||||
|
all, silentAll := 0, 0
|
||||||
|
seenQuiet := map[string]bool{}
|
||||||
|
seenNode := map[string]bool{}
|
||||||
|
for _, need := range needs {
|
||||||
|
for _, n := range need.running {
|
||||||
|
if !seenNode[n] {
|
||||||
|
seenNode[n] = true
|
||||||
|
all++
|
||||||
|
if silent[n] {
|
||||||
|
silentAll++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, on := appliedEach(need.module, need.since, need.running, reports)
|
||||||
|
var still []string
|
||||||
|
for _, n := range on {
|
||||||
|
if silent[n] {
|
||||||
|
if !seenQuiet[n] {
|
||||||
|
seenQuiet[n] = true
|
||||||
|
quiet = append(quiet, n)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
still = append(still, n)
|
||||||
|
}
|
||||||
|
if len(still) > 0 {
|
||||||
|
waiting = append(waiting, fmt.Sprintf("%s on %s", need.module, strings.Join(still, ", ")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(quiet)
|
||||||
|
return waiting, quiet, all > 0 && silentAll == all
|
||||||
|
}
|
||||||
|
|
||||||
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
||||||
at := map[string]*time.Time{}
|
at := map[string]*time.Time{}
|
||||||
for _, r := range reports {
|
for _, r := range reports {
|
||||||
@@ -639,11 +702,15 @@ func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Pl
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
open, err := openWalksOf(ctx, inv, started)
|
onPath, err := ownPathNow(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
mine, err := openWalkOf(ctx, inv, p)
|
open, err := openWalksOf(ctx, inv, started, onPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
mine, err := openWalkOf(ctx, inv, p, onPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
@@ -869,6 +936,22 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
// being behind. Now every module of the tier whose first machine is the same is sent there in one
|
// being behind. Now every module of the tier whose first machine is the same is sent there in one
|
||||||
// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
|
// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
|
||||||
// of the tier's machines are sent once, together, when the gates have passed.
|
// of the tier's machines are sent once, together, when the gates have passed.
|
||||||
|
// **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §5, issue 1160): the
|
||||||
|
// walk's end names it, and the gates between tiers and the sends of the tiers after do not wait for it or go to it.
|
||||||
|
allReports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
silent := silentNodes(*p, allReports, time.Now().UTC(), provLookup(ctx))
|
||||||
|
// **A machine away when a tier is sent is not sent to and not waited for** (ADR 0435 §1, issue 1161): read once,
|
||||||
|
// for a tier that has something left to send.
|
||||||
|
var away map[string]bool
|
||||||
|
for _, m := range tier {
|
||||||
|
if s := p.Modules[m]; s != nil && s.SentAt == nil && s.FirstAt == nil && s.State != planDeleted {
|
||||||
|
away = judgedAway(ctx, open, time.Now().UTC())
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
var pending []string
|
var pending []string
|
||||||
firstOn := map[string][]string{}
|
firstOn := map[string][]string{}
|
||||||
runningOf := map[string][]string{}
|
runningOf := map[string][]string{}
|
||||||
@@ -903,6 +986,19 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
if goes != nil {
|
if goes != nil {
|
||||||
running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] })
|
running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] })
|
||||||
}
|
}
|
||||||
|
if state.FirstAt == nil {
|
||||||
|
running = withoutSilent(running, silent)
|
||||||
|
passed := withoutSilent(running, away)
|
||||||
|
if len(passed) < len(running) {
|
||||||
|
for _, n := range running {
|
||||||
|
if away[n] && !slices.Contains(p.Away, n) {
|
||||||
|
fmt.Printf("%s: %s is away; %s is not sent to it and not waited for\n", p.ID, n, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rememberAway(p, running, away)
|
||||||
|
running = passed
|
||||||
|
}
|
||||||
|
}
|
||||||
runningOf[m] = running
|
runningOf[m] = running
|
||||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
|
// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
|
||||||
// of the build every machine running it was sent, it is registered with that build's artifacts —
|
// of the build every machine running it was sent, it is registered with that build's artifacts —
|
||||||
@@ -1096,13 +1192,14 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
return changed, nil
|
return changed, nil
|
||||||
}
|
}
|
||||||
// And wait for what the next tier needs running.
|
// And wait for what the next tier needs running.
|
||||||
|
leftAsWas := ""
|
||||||
needed := gates(*p, edges, rollsOut)
|
needed := gates(*p, edges, rollsOut)
|
||||||
if len(needed) > 0 {
|
if len(needed) > 0 {
|
||||||
reports, err := inv.LastReports(ctx)
|
reports, err := inv.LastReports(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
var waiting []string
|
var needs []gateNeed
|
||||||
for _, m := range needed {
|
for _, m := range needed {
|
||||||
running, err := inv.Running(ctx, m)
|
running, err := inv.Running(ctx, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1116,6 +1213,8 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
if state.State == planDeleted {
|
if state.State == planDeleted {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// A machine passed as away was sent nothing to wait on (ADR 0435 §1, issue 1161).
|
||||||
|
running = runningHeard(*p, running, state)
|
||||||
// The plan sends what it waits for. A rebuild from the same source commit is not a
|
// The plan sends what it waits for. A rebuild from the same source commit is not a
|
||||||
// move the catalogue announces — the build machine rebuilt for a controller change
|
// move the catalogue announces — the build machine rebuilt for a controller change
|
||||||
// is one — so the roll-out that opens this gate is the plan's to make, once, and
|
// is one — so the roll-out that opens this gate is the plan's to make, once, and
|
||||||
@@ -1133,14 +1232,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
for _, r := range reports {
|
for _, r := range reports {
|
||||||
byNode[r.Node] = r
|
byNode[r.Node] = r
|
||||||
}
|
}
|
||||||
if ok, on := appliedEach(m, func(node string) time.Time {
|
needs = append(needs, gateNeed{m, running, func(node string) time.Time {
|
||||||
if s := byNode[node].Sent; s != nil {
|
if s := byNode[node].Sent; s != nil {
|
||||||
return *s
|
return *s
|
||||||
}
|
}
|
||||||
return time.Time{}
|
return time.Time{}
|
||||||
}, running, reports); !ok {
|
}})
|
||||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
|
||||||
}
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||||
@@ -1160,9 +1257,23 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
}
|
}
|
||||||
return at
|
return at
|
||||||
}
|
}
|
||||||
if ok, on := appliedEach(m, sinceFor, running, reports); !ok {
|
needs = append(needs, gateNeed{m, running, sinceFor})
|
||||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
|
||||||
}
|
}
|
||||||
|
waiting, quiet, lost := settleTierGate(needs, reports, silent)
|
||||||
|
if lost {
|
||||||
|
// Every machine the next tier waits on is silent: nothing it needs was delivered anywhere, so the plan does
|
||||||
|
// not go on as if it had been (issue 1160).
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
p.Note = fmt.Sprintf("tier %d built; every machine it waits on said nothing on its send within %s: %s",
|
||||||
|
p.Tier, applySilentAfter, strings.Join(quiet, ", "))
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if len(quiet) > 0 {
|
||||||
|
fmt.Printf("%s: tier %d: %s said nothing on its send within %s and is left as it was; the plan goes on without it\n",
|
||||||
|
p.ID, p.Tier, strings.Join(quiet, ", "), applySilentAfter)
|
||||||
|
leftAsWas = fmt.Sprintf("%s said nothing on its send within %s and is left as it was", strings.Join(quiet, ", "),
|
||||||
|
applySilentAfter)
|
||||||
}
|
}
|
||||||
if len(waiting) > 0 {
|
if len(waiting) > 0 {
|
||||||
note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied"
|
note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied"
|
||||||
@@ -1204,6 +1315,13 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
|||||||
carryUserList(ctx, open, p)
|
carryUserList(ctx, open, p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if end == "" {
|
||||||
|
end = leftAsWas
|
||||||
|
}
|
||||||
|
// A machine passed as away is said where the others passed are said (ADR 0435 §1, issue 1161).
|
||||||
|
if said := awaySaying(*p); said != "" {
|
||||||
|
end = strings.TrimPrefix(end+"; "+said, "; ")
|
||||||
|
}
|
||||||
p.Tier++
|
p.Tier++
|
||||||
p.State = inventory.PlanBuilding
|
p.State = inventory.PlanBuilding
|
||||||
p.Note = end
|
p.Note = end
|
||||||
@@ -1920,6 +2038,12 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||||
moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+
|
moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+
|
||||||
"comma-separated, as the forge's announcer says them (novox/hq issue 278)")
|
"comma-separated, as the forge's announcer says them (novox/hq issue 278)")
|
||||||
|
// The what-if as data, the document `delivery plan` answers (novox/hq ADR 0450 §2, issue 1324).
|
||||||
|
data := set.Bool("data", false, "with --what-if: answer the change plan as data, the document `delivery plan` answers; "+
|
||||||
|
"without it the answer is the text a person reads")
|
||||||
|
head := set.String("head", "", "with --data: the commit at hand")
|
||||||
|
base := set.String("base", "main", "with --data: the branch it merges into")
|
||||||
|
removed := set.String("removed", "", "with --data: the files among --paths it deletes, comma-separated")
|
||||||
// Following a plan (novox/hq issue 229): `plans wait <id>`.
|
// Following a plan (novox/hq issue 229): `plans wait <id>`.
|
||||||
until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)")
|
until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)")
|
||||||
bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+
|
bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+
|
||||||
@@ -2035,6 +2159,25 @@ func plansCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if *data && *whatIf == "" {
|
||||||
|
return errors.New("--data answers a what-if: give --what-if owner/repository")
|
||||||
|
}
|
||||||
|
if *whatIf != "" && *data {
|
||||||
|
if *paths == "" {
|
||||||
|
return errors.New("plans --what-if --data: the change plan is of the files a change touches; give --paths")
|
||||||
|
}
|
||||||
|
entries, read, edges, err := theGraph(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m := changeMember(*whatIf, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs))
|
||||||
|
enc := json.NewEncoder(os.Stdout)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
return enc.Encode(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) {
|
||||||
|
u, err := inv.UpgradeOf(ctx, module)
|
||||||
|
return u, err == nil
|
||||||
|
}))
|
||||||
|
}
|
||||||
if *whatIf != "" {
|
if *whatIf != "" {
|
||||||
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules), moduleDirsOf(*moduleDirs))
|
return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules), moduleDirsOf(*moduleDirs))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -392,6 +392,32 @@ func rolloutMint(ctx context.Context, again bool) error {
|
|||||||
}
|
}
|
||||||
modules++
|
modules++
|
||||||
|
|
||||||
|
case broker.KindProcess:
|
||||||
|
// One process's own account (novox/hq ADR 0436 §1, §2), delivered as the module's own secret
|
||||||
|
// broker-<process id> — minted here exactly as `module issue` mints it, so a process issued
|
||||||
|
// once still gets a new password every time the bus's accounts are issued again.
|
||||||
|
m, inShelf := shelf[p.Module]
|
||||||
|
if !inShelf {
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pr, found := processPrincipalFor(m, p.Process)
|
||||||
|
if !found {
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := processSecretName(p.Process)
|
||||||
|
if _, reads := m.OwnSecrets[name]; !reads {
|
||||||
|
fmt.Printf(" %s's process %s declares bus but declares no `%s` secret to receive a "+
|
||||||
|
"credential in; skipped\n", p.Module, p.Process, name)
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := issueProcess(ctx, inv, m, pr, p.Node, "", known, busAddress); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
modules++
|
||||||
|
|
||||||
default:
|
default:
|
||||||
skipped++
|
skipped++
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
|
|||||||
// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
|
// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
|
||||||
first, stopFirst := context.WithCancel(ctx)
|
first, stopFirst := context.WithCancel(ctx)
|
||||||
defer stopFirst()
|
defer stopFirst()
|
||||||
startRolloutEvents(first, open, js)
|
waitFirst := startRolloutEvents(first, open, js)
|
||||||
built := time.Now().UTC()
|
built := time.Now().UTC()
|
||||||
p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built,
|
p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built,
|
||||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
@@ -199,12 +199,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
|
|||||||
|
|
||||||
// The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody.
|
// The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody.
|
||||||
stopFirst()
|
stopFirst()
|
||||||
|
waitFirst()
|
||||||
time.Sleep(300 * time.Millisecond)
|
time.Sleep(300 * time.Millisecond)
|
||||||
reports("anchor")
|
reports("anchor")
|
||||||
|
|
||||||
second, stopSecond := context.WithCancel(ctx)
|
second, stopSecond := context.WithCancel(ctx)
|
||||||
defer stopSecond()
|
defer stopSecond()
|
||||||
startRolloutEvents(second, open, js)
|
waitSecond := startRolloutEvents(second, open, js)
|
||||||
|
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
||||||
|
t.Cleanup(func() { stopSecond(); waitSecond() })
|
||||||
// Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire.
|
// Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire.
|
||||||
eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool {
|
eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool {
|
||||||
h := heldKeys()
|
h := heldKeys()
|
||||||
@@ -321,7 +324,7 @@ func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) {
|
|||||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||||
started, stop := context.WithCancel(ctx)
|
started, stop := context.WithCancel(ctx)
|
||||||
defer stop()
|
defer stop()
|
||||||
startRolloutEvents(started, open, js)
|
wait := startRolloutEvents(started, open, js)
|
||||||
time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing
|
time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing
|
||||||
|
|
||||||
// The verb as its child process runs it: the delivery command, whose saves are not the serving controller's.
|
// The verb as its child process runs it: the delivery command, whose saves are not the serving controller's.
|
||||||
@@ -333,6 +336,9 @@ func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) {
|
|||||||
return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"})
|
return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"})
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { runTheVerb = wasRun })
|
t.Cleanup(func() { runTheVerb = wasRun })
|
||||||
|
// Registered after every variable cleanup above, so it runs before them (last in, first out): the look at the plans
|
||||||
|
// has returned when checkEvents, inventory.PlanSaved and runTheVerb are restored (issue 1338).
|
||||||
|
t.Cleanup(func() { stop(); wait() })
|
||||||
handlers, _, err := seatToolHandlers()
|
handlers, _, err := seatToolHandlers()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
|
|||||||
return s, heard
|
return s, heard
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
|
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { // 1157/AC3: the send keys stay until release 6 (ADR 0433 §5)
|
||||||
s, heard := silentOnABus(t)
|
s, heard := silentOnABus(t)
|
||||||
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
|
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
|
||||||
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
|
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// seatPassing is what passing seats on needs of the outside: the record's compare-and-set, and the hand act.
|
||||||
|
type seatPassing struct {
|
||||||
|
pass func(ctx context.Context, p catalogue.SeatPass) (bool, error)
|
||||||
|
act func(ctx context.Context, p catalogue.SeatPass) error
|
||||||
|
say io.Writer
|
||||||
|
}
|
||||||
|
|
||||||
|
// passSeatsOn records, in the composition that finds it, the new holder of each seat held once whose holder on
|
||||||
|
// record is assigned nowhere and which has exactly one claimant able to hold it, and writes the controller's own
|
||||||
|
// hand act for it (novox/hq ADR 0449 §4, issue 1317). It answers the holdings as they are after it, so the
|
||||||
|
// resolution that follows reads the record it just moved. A failed write is said and leaves the record as it was:
|
||||||
|
// a composition is never refused for it.
|
||||||
|
func passSeatsOn(ctx context.Context, sp seatPassing, holdings []catalogue.Held,
|
||||||
|
assignments []catalogue.Assignment) []catalogue.Held {
|
||||||
|
passes, said := catalogue.SeatsToPassOn(holdings, assignments)
|
||||||
|
for _, line := range said {
|
||||||
|
fmt.Fprintln(sp.say, line)
|
||||||
|
}
|
||||||
|
for _, p := range passes {
|
||||||
|
moved, err := sp.pass(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(sp.say, "%s could not be passed from %s on %s to %s on %s: %v\n", p.Seat.Name,
|
||||||
|
p.From.Module, p.From.Node, p.To.Module, p.To.Node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !moved {
|
||||||
|
continue // another composition passed it first, and wrote the act
|
||||||
|
}
|
||||||
|
if err := sp.act(ctx, p); err != nil {
|
||||||
|
fmt.Fprintf(sp.say, "%s passed from %s on %s to %s on %s, and the hand act could NOT be recorded: %v\n",
|
||||||
|
p.Seat.Name, p.From.Module, p.From.Node, p.To.Module, p.To.Node, err)
|
||||||
|
}
|
||||||
|
var next []catalogue.Held
|
||||||
|
for _, h := range holdings {
|
||||||
|
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == p.Seat.Name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
next = append(next, h)
|
||||||
|
}
|
||||||
|
p.To.Site = p.From.Site
|
||||||
|
holdings = append(next, p.To)
|
||||||
|
}
|
||||||
|
return holdings
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatPassHandAct is the controller's own entry for a pass: it is the act that puts the holder on record,
|
||||||
|
// where a person's `seat --to` would have, so the record of who held what stays complete (ADR 0449 §4).
|
||||||
|
func seatPassHandAct(p catalogue.SeatPass) link.HandAct {
|
||||||
|
return link.HandAct{
|
||||||
|
By: "the controller",
|
||||||
|
Verb: "seat",
|
||||||
|
Args: []string{p.Seat.Name, "--to", p.To.Node + "/" + p.To.Module, "--was", p.From.Node + "/" + p.From.Module},
|
||||||
|
Why: fmt.Sprintf("%s on %s, the holder on record of %s, is assigned on no machine, and %s on %s is the only "+
|
||||||
|
"assignment that claims it and can hold it (ADR 0449 §4)", p.From.Module, p.From.Node, p.Seat.Name,
|
||||||
|
p.To.Module, p.To.Node),
|
||||||
|
Cause: "seat-passed",
|
||||||
|
Outcome: "done",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignmentsOf are the assignments of every machine, as the passing of a seat reads them. **A module assigned and
|
||||||
|
// missing from the shelf is still assigned**: it is an assignment with no claims, so its holder is not taken for gone.
|
||||||
|
func assignmentsOf(shelf map[string]catalogue.Manifest, assigned map[string][]string) []catalogue.Assignment {
|
||||||
|
var out []catalogue.Assignment
|
||||||
|
for node, modules := range assigned {
|
||||||
|
for _, module := range modules {
|
||||||
|
m, ok := shelf[module]
|
||||||
|
if !ok {
|
||||||
|
m = catalogue.Manifest{Module: module}
|
||||||
|
}
|
||||||
|
out = append(out, catalogue.Assignment{Node: node, Manifest: m})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatPassingOn is how a composition on this inventory passes a seat: against the store's record with the hand act
|
||||||
|
// on the bus, or, on a throwaway mesh (the merge gate's), in memory only, writing nothing anywhere.
|
||||||
|
func seatPassingOn(inv *inventory.Inventory, w io.Writer) seatPassing {
|
||||||
|
if inv.Throwaway() {
|
||||||
|
return seatPassing{say: w,
|
||||||
|
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return true, nil },
|
||||||
|
act: func(context.Context, catalogue.SeatPass) error { return nil }}
|
||||||
|
}
|
||||||
|
return seatPassing{
|
||||||
|
say: w,
|
||||||
|
pass: func(ctx context.Context, p catalogue.SeatPass) (bool, error) {
|
||||||
|
return inv.PassSeat(ctx, p.Seat.Name, p.Seat.Scope, p.From, p.To)
|
||||||
|
},
|
||||||
|
act: func(ctx context.Context, p catalogue.SeatPass) error {
|
||||||
|
return onTheBus(func(conn *nats.Conn) error {
|
||||||
|
_, err := link.RecordHandAct(ctx, conn, seatPassHandAct(p))
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// theMeshPassesSeatsOn is passSeatsOn as the composition runs it. **Only when every machine's assignments were
|
||||||
|
// read** (assignedErr nil): a machine whose read failed would look as if its modules were assigned nowhere, and
|
||||||
|
// its holder would lose the seat. Said to where the composition says its other lines (sayLeftOut).
|
||||||
|
func theMeshPassesSeatsOn(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
||||||
|
assigned map[string][]string, assignedErr error, holdings []catalogue.Held) []catalogue.Held {
|
||||||
|
var w io.Writer = os.Stderr
|
||||||
|
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
||||||
|
w = lw
|
||||||
|
}
|
||||||
|
if assignedErr != nil {
|
||||||
|
fmt.Fprintf(w, "no seat is passed on in this composition: a machine's assignments could not be read, so a "+
|
||||||
|
"holder on record might look assigned nowhere: %v\n", assignedErr)
|
||||||
|
return holdings
|
||||||
|
}
|
||||||
|
return passSeatsOn(ctx, seatPassingOn(inv, w), holdings, assignmentsOf(shelf, assigned))
|
||||||
|
}
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
func aBrokerClaiming(module string) catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: module,
|
||||||
|
Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}},
|
||||||
|
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func withBusSeat(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
was := catalogue.Seats()
|
||||||
|
t.Cleanup(func() { catalogue.UseSeats(was) })
|
||||||
|
catalogue.UseSeats([]catalogue.Seat{{Name: "mesh-broker", Scope: catalogue.ScopeMesh, Delivers: "mesh-bus", Decision: "test"}})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1317/AC1: the composition records the claimant as the holder and writes one hand act by the controller naming
|
||||||
|
// the seat, the former holder and the new one; the holdings it answers name the new holder.
|
||||||
|
func TestACompositionPassesTheSeatAndWritesOneHandActByTheController(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
var passed []catalogue.SeatPass
|
||||||
|
var acts []link.HandAct
|
||||||
|
var said bytes.Buffer
|
||||||
|
sp := seatPassing{say: &said,
|
||||||
|
pass: func(_ context.Context, p catalogue.SeatPass) (bool, error) {
|
||||||
|
passed = append(passed, p)
|
||||||
|
return true, nil
|
||||||
|
},
|
||||||
|
act: func(_ context.Context, p catalogue.SeatPass) error {
|
||||||
|
acts = append(acts, seatPassHandAct(p))
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||||
|
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||||
|
if len(got) != 1 || got[0].Module != "gate" || got[0].Node != "novox" {
|
||||||
|
t.Fatalf("the holdings after the composition do not name the gate: %+v", got)
|
||||||
|
}
|
||||||
|
if len(passed) != 1 || len(acts) != 1 {
|
||||||
|
t.Fatalf("passes=%d acts=%d, want one each", len(passed), len(acts))
|
||||||
|
}
|
||||||
|
a := acts[0]
|
||||||
|
text := strings.Join(append([]string{a.By, a.Verb, a.Why}, a.Args...), " ")
|
||||||
|
for _, want := range []string{"the controller", "mesh-broker", "novox/old-broker", "novox/gate"} {
|
||||||
|
if !strings.Contains(text, want) {
|
||||||
|
t.Fatalf("the hand act does not name %q: %+v", want, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1317/AC2: two claimants, nothing recorded, the refusal said.
|
||||||
|
func TestACompositionWithTwoClaimantsRecordsNoneAndSaysSo(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
sp := seatPassing{say: &bytes.Buffer{},
|
||||||
|
pass: func(context.Context, catalogue.SeatPass) (bool, error) {
|
||||||
|
t.Fatal("a seat with two claimants was passed")
|
||||||
|
return false, nil
|
||||||
|
},
|
||||||
|
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("an act was written"); return nil },
|
||||||
|
}
|
||||||
|
var said bytes.Buffer
|
||||||
|
sp.say = &said
|
||||||
|
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||||
|
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{
|
||||||
|
{Node: "novox", Manifest: aBrokerClaiming("gate")}, {Node: "ace", Manifest: aBrokerClaiming("other")}})
|
||||||
|
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||||
|
t.Fatalf("the record changed: %+v", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said.String(), "gate") || !strings.Contains(said.String(), "other") ||
|
||||||
|
!strings.Contains(said.String(), "seat mesh-broker --to") {
|
||||||
|
t.Fatalf("the refusal is not said with both and the remedy: %q", said.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1317/AC3 and AC4: nothing changes while the holder is assigned, nor with no claimant.
|
||||||
|
func TestACompositionChangesNothingWhileTheHolderIsAssignedOrNobodyClaims(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
sp := seatPassing{say: &bytes.Buffer{},
|
||||||
|
pass: func(context.Context, catalogue.SeatPass) (bool, error) { t.Fatal("passed"); return false, nil },
|
||||||
|
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil },
|
||||||
|
}
|
||||||
|
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||||
|
got := passSeatsOn(t.Context(), sp, holdings, []catalogue.Assignment{
|
||||||
|
{Node: "novox", Manifest: aBrokerClaiming("old-broker")}, {Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||||
|
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||||
|
t.Fatalf("holder assigned: record changed: %+v", got)
|
||||||
|
}
|
||||||
|
got = passSeatsOn(t.Context(), sp, holdings, nil)
|
||||||
|
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||||
|
t.Fatalf("no claimant: record changed: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A composition that lost the compare-and-set writes no act, and a failed write leaves the record.
|
||||||
|
func TestALostPassWritesNoActAndAFailedOneLeavesTheRecord(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||||
|
as := []catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}}
|
||||||
|
lost := seatPassing{say: &bytes.Buffer{},
|
||||||
|
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, nil },
|
||||||
|
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act without a pass"); return nil },
|
||||||
|
}
|
||||||
|
passSeatsOn(t.Context(), lost, holdings, as)
|
||||||
|
var said bytes.Buffer
|
||||||
|
sp := seatPassing{say: &said,
|
||||||
|
pass: func(context.Context, catalogue.SeatPass) (bool, error) { return false, context.DeadlineExceeded },
|
||||||
|
act: func(context.Context, catalogue.SeatPass) error { t.Fatal("act"); return nil },
|
||||||
|
}
|
||||||
|
got := passSeatsOn(t.Context(), sp, holdings, as)
|
||||||
|
if got[0].Module != "old-broker" || !strings.Contains(said.String(), "could not be passed") {
|
||||||
|
t.Fatalf("got %+v said %q", got, said.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A pass is the mesh working as ADR 0449 §4 decided, never a repair: two of them raise no healer-wanted and
|
||||||
|
// count as no repair.
|
||||||
|
func TestTwoSeatPassesAreNoRepairAndRaiseNoHealerWanted(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
now := time.Date(2026, 10, 19, 20, 0, 0, 0, time.UTC)
|
||||||
|
mk := func(at time.Time) link.HandAct {
|
||||||
|
a := seatPassHandAct(catalogue.SeatPass{Seat: catalogue.Seat{Name: "mesh-broker"},
|
||||||
|
From: catalogue.Held{Node: "novox", Module: "old"}, To: catalogue.Held{Node: "novox", Module: "gate"}})
|
||||||
|
a.At = at
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
acts := []link.HandAct{mk(now.Add(-26 * time.Hour)), mk(now.Add(-time.Hour))}
|
||||||
|
if r := repairs(acts); len(r) != 0 {
|
||||||
|
t.Fatalf("a seat pass counts as a repair: %+v", r)
|
||||||
|
}
|
||||||
|
f := calm(now)
|
||||||
|
f.handActs = acts
|
||||||
|
if got := watchHandActs(f); len(got) != 0 {
|
||||||
|
t.Fatalf("two passes raised %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed read of one machine's assignments passes no seat: its holder would look assigned nowhere.
|
||||||
|
func TestNoSeatIsPassedWhenAMachinesAssignmentsCouldNotBeRead(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
var said bytes.Buffer
|
||||||
|
ctx := leftOutSaidBy(t.Context(), &said)
|
||||||
|
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "ace", Module: "old-broker"}}
|
||||||
|
shelf := map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")}
|
||||||
|
got := theMeshPassesSeatsOn(ctx, nil, shelf, map[string][]string{"novox": {"gate"}},
|
||||||
|
errors.New("ace: the bus did not answer"), holdings)
|
||||||
|
if len(got) != 1 || got[0].Module != "old-broker" {
|
||||||
|
t.Fatalf("the record moved: %+v", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said.String(), "no seat is passed") || !strings.Contains(said.String(), "ace") {
|
||||||
|
t.Errorf("not said: %q", said.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A holder that is assigned but missing from the shelf is still assigned.
|
||||||
|
func TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
as := assignmentsOf(map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")},
|
||||||
|
map[string][]string{"novox": {"gate", "old-broker"}})
|
||||||
|
passes, _ := catalogue.SeatsToPassOn(
|
||||||
|
[]catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as)
|
||||||
|
if len(passes) != 0 {
|
||||||
|
t.Fatalf("a holder missing from the shelf lost its seat: %+v", passes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The merge gate's throwaway mesh passes the seat in memory and writes nothing: no record, no hand act.
|
||||||
|
func TestAThrowawayMeshPassesTheSeatInMemoryAndWritesNothing(t *testing.T) {
|
||||||
|
withBusSeat(t)
|
||||||
|
inv := &inventory.Inventory{}
|
||||||
|
inv.MarkThrowaway()
|
||||||
|
var said bytes.Buffer
|
||||||
|
holdings := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}
|
||||||
|
got := passSeatsOn(t.Context(), seatPassingOn(inv, &said), holdings,
|
||||||
|
[]catalogue.Assignment{{Node: "novox", Manifest: aBrokerClaiming("gate")}})
|
||||||
|
if len(got) != 1 || got[0].Module != "gate" {
|
||||||
|
t.Fatalf("the gate's view does not hold the pass: %+v", got)
|
||||||
|
}
|
||||||
|
if said.Len() != 0 {
|
||||||
|
t.Errorf("the throwaway composition said %q (a write was tried)", said.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -351,6 +352,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if d := str("module-dirs"); d != "" {
|
if d := str("module-dirs"); d != "" {
|
||||||
argv = append(argv, "--module-dirs", d)
|
argv = append(argv, "--module-dirs", d)
|
||||||
}
|
}
|
||||||
|
// The plan as data, the document delivery-plan answers (novox/hq ADR 0450 §2, issue 1324).
|
||||||
|
if str("data") == "true" {
|
||||||
|
argv = append(argv, "--data")
|
||||||
|
for _, flag := range []string{"head", "base", "removed"} {
|
||||||
|
if v := str(flag); v != "" {
|
||||||
|
argv = append(argv, "--"+flag, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
return argv, nil
|
return argv, nil
|
||||||
}
|
}
|
||||||
for _, act := range []string{"stop", "close", "retry", "go"} {
|
for _, act := range []string{"stop", "close", "retry", "go"} {
|
||||||
@@ -408,14 +418,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
if err := need("members"); err != nil {
|
if err := need("members"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return []string{"delivery", "order", "--members", str("members")}, nil
|
members, err := membersFlags(str("members"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return append([]string{"delivery", "order"}, members...), nil
|
||||||
case "delivery-check":
|
case "delivery-check":
|
||||||
if err := need("members"); err != nil {
|
if err := need("members"); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
argv := []string{"delivery", "check", "--members", str("members")}
|
members, err := membersFlags(str("members"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := append([]string{"delivery", "check"}, members...)
|
||||||
if g := str("group"); g != "" {
|
if g := str("group"); g != "" {
|
||||||
argv = []string{"delivery", "check", "--group", g, "--members", str("members")}
|
argv = append([]string{"delivery", "check", "--group", g}, members...)
|
||||||
}
|
}
|
||||||
// Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check.
|
// Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check.
|
||||||
if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" {
|
if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" {
|
||||||
@@ -1193,6 +1211,7 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
|||||||
if handingOver.Load() {
|
if handingOver.Load() {
|
||||||
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
|
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
|
||||||
}
|
}
|
||||||
|
defer removeMembersFile(argv)
|
||||||
cmd := selfCommand(ctx, argv)
|
cmd := selfCommand(ctx, argv)
|
||||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||||
caller := link.CallerIn(ctx)
|
caller := link.CallerIn(ctx)
|
||||||
@@ -1858,3 +1877,41 @@ func terminalOnly(argv []string) error {
|
|||||||
"decide who may become root on it (novox/hq ADR 0266). A verb may run node list and node show. "+
|
"decide who may become root on it (novox/hq ADR 0266). A verb may run node list and node show. "+
|
||||||
"Nothing was done", sub)
|
"Nothing was done", sub)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// membersInArgument is how long a group's members may be as one argument of the command a verb re-executes: over
|
||||||
|
// the kernel's 128 KiB for one argument the command cannot start (novox/hq issue 1272).
|
||||||
|
const membersInArgument = 32 << 10
|
||||||
|
|
||||||
|
// membersDirPrefix names the private directory a long list of members is handed over in.
|
||||||
|
const membersDirPrefix = "mesh-members-"
|
||||||
|
|
||||||
|
// membersFlags is a group's members as the flags of the command a verb re-executes: `--members <json>` while the
|
||||||
|
// list is short, as a person would type it, and `--members-file <path>` to a file in a private directory of its own
|
||||||
|
// when it is not. runVerb removes the directory once the command has ended.
|
||||||
|
func membersFlags(members string) ([]string, error) {
|
||||||
|
if len(members) <= membersInArgument {
|
||||||
|
return []string{"--members", members}, nil
|
||||||
|
}
|
||||||
|
dir, err := os.MkdirTemp("", membersDirPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the members cannot be handed to the command: %w", err)
|
||||||
|
}
|
||||||
|
file := filepath.Join(dir, "members.json")
|
||||||
|
if err := os.WriteFile(file, []byte(members), 0o600); err != nil {
|
||||||
|
os.RemoveAll(dir)
|
||||||
|
return nil, fmt.Errorf("the members cannot be handed to the command: %w", err)
|
||||||
|
}
|
||||||
|
return []string{"--members-file", file}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeMembersFile removes the private directory membersFlags made for a command line, and only such a directory.
|
||||||
|
func removeMembersFile(argv []string) {
|
||||||
|
for i, a := range argv {
|
||||||
|
if a == "--members-file" && i+1 < len(argv) {
|
||||||
|
dir := filepath.Dir(argv[i+1])
|
||||||
|
if strings.HasPrefix(filepath.Base(dir), membersDirPrefix) {
|
||||||
|
os.RemoveAll(dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and
|
||||||
|
// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes
|
||||||
|
// the key again; the look that writes it leaves it out of the set, and the condition clears.
|
||||||
|
const (
|
||||||
|
kindSendKeyRefused = "send-key-refused"
|
||||||
|
sendKeyRefusedID = "DL"
|
||||||
|
)
|
||||||
|
|
||||||
|
var refusedSendKeys = struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
by map[silentWait]error
|
||||||
|
}{}
|
||||||
|
|
||||||
|
// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found.
|
||||||
|
func noteRefusedSendKeys(refused map[silentWait]error) {
|
||||||
|
refusedSendKeys.mu.Lock()
|
||||||
|
defer refusedSendKeys.mu.Unlock()
|
||||||
|
refusedSendKeys.by = refused
|
||||||
|
}
|
||||||
|
|
||||||
|
// observeRefusedSendKeys is one condition per plan and machine whose send key was refused.
|
||||||
|
func observeRefusedSendKeys() []conditions.Observation {
|
||||||
|
refusedSendKeys.mu.Lock()
|
||||||
|
waits := make([]silentWait, 0, len(refusedSendKeys.by))
|
||||||
|
why := map[silentWait]error{}
|
||||||
|
for w, err := range refusedSendKeys.by {
|
||||||
|
waits = append(waits, w)
|
||||||
|
why[w] = err
|
||||||
|
}
|
||||||
|
refusedSendKeys.mu.Unlock()
|
||||||
|
sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) })
|
||||||
|
seen := map[[2]string]bool{}
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, w := range waits {
|
||||||
|
if seen[[2]string{w.Plan, w.Node}] {
|
||||||
|
continue // a plan sends a machine one tier at a time; one condition says it
|
||||||
|
}
|
||||||
|
seen[[2]string{w.Plan, w.Node}] = true
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node,
|
||||||
|
Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+
|
||||||
|
"names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node),
|
||||||
|
Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeSendKeyRefused reads what the newest look of the plans could not write.
|
||||||
|
func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||||
|
return observeRefusedSendKeys(), nil
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -38,6 +39,13 @@ type sendable struct {
|
|||||||
// the send; never on the wire.
|
// the send; never on the wire.
|
||||||
composedFrom int64
|
composedFrom int64
|
||||||
actingEpoch uint64
|
actingEpoch uint64
|
||||||
|
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
|
||||||
|
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
|
||||||
|
ReadsModules bool
|
||||||
|
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
|
||||||
|
// kept beside the resources, and on the wire only to a machine that reads them.
|
||||||
|
moduleOf map[string]string
|
||||||
|
buildIDs map[string]string
|
||||||
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
||||||
modules []string
|
modules []string
|
||||||
// localNetwork is every port admitted from the local network and what it resolved to (novox/hq ADR
|
// localNetwork is every port admitted from the local network and what it resolved to (novox/hq ADR
|
||||||
@@ -111,6 +119,28 @@ type adoptionEnvelope struct {
|
|||||||
// Body is the declaration's bytes, as sent and as digested.
|
// Body is the declaration's bytes, as sent and as digested.
|
||||||
func (s sendable) Body() ([]byte, error) {
|
func (s sendable) Body() ([]byte, error) {
|
||||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||||
|
if s.ReadsModules {
|
||||||
|
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
|
||||||
|
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
|
||||||
|
named := make([]map[string]any, len(s.Resources))
|
||||||
|
for n, r := range s.Resources {
|
||||||
|
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
|
||||||
|
if !owned {
|
||||||
|
named[n] = r
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
with := make(map[string]any, len(r)+1)
|
||||||
|
for k, v := range r {
|
||||||
|
with[k] = v
|
||||||
|
}
|
||||||
|
with["module"] = module
|
||||||
|
named[n] = with
|
||||||
|
}
|
||||||
|
envelope["resources"] = named
|
||||||
|
if len(s.buildIDs) > 0 {
|
||||||
|
envelope["builds"] = s.buildIDs
|
||||||
|
}
|
||||||
|
}
|
||||||
if s.Adoption != nil {
|
if s.Adoption != nil {
|
||||||
envelope["adoption"] = s.Adoption
|
envelope["adoption"] = s.Adoption
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,10 +85,7 @@ func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T)
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||||
mirror, err := inv.AssignmentsMirror(ctx)
|
mirror := inv.PlanValuesForTest(t)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var value inventory.AssignmentMirrored
|
var value inventory.AssignmentMirrored
|
||||||
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
|
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
|
||||||
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
|
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
|
||||||
|
|||||||
@@ -234,6 +234,16 @@ var signalsTable = []signalRow{
|
|||||||
newest: func(f *signalFacts) time.Time {
|
newest: func(f *signalFacts) time.Time {
|
||||||
return newestOf(f.batches, func(b batchFacts) time.Time { return b.created })
|
return newestOf(f.batches, func(b batchFacts) time.Time { return b.created })
|
||||||
}},
|
}},
|
||||||
|
{Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan",
|
||||||
|
Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)",
|
||||||
|
Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " +
|
||||||
|
"words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " +
|
||||||
|
"from the plan's last change, and ended when the plan is no longer failed",
|
||||||
|
Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3,
|
||||||
|
needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed,
|
||||||
|
newest: func(f *signalFacts) time.Time {
|
||||||
|
return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
|
||||||
|
}},
|
||||||
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
|
||||||
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
|
||||||
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
|
||||||
@@ -318,6 +328,10 @@ func ago(d time.Duration) string {
|
|||||||
return d.Round(time.Minute).String()
|
return d.Round(time.Minute).String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// silentBound is how long a machine may go unheard, where the watchman gives no verdict: three of the intervals nodes have
|
||||||
|
// always used, fixed now that no machine says its own (issue 1183).
|
||||||
|
const silentBound = heartbeatsMissed * heartbeatEvery
|
||||||
|
|
||||||
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||||
func heartbeatBound(every time.Duration) time.Duration {
|
func heartbeatBound(every time.Duration) time.Duration {
|
||||||
if every <= 0 {
|
if every <= 0 {
|
||||||
@@ -333,9 +347,9 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
|||||||
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bound := heartbeatBound(m.every)
|
bound := silentBound
|
||||||
silent := f.now.Sub(m.lastHeard)
|
silent := f.now.Sub(m.lastHeard)
|
||||||
if silent <= bound {
|
if !m.silent(f.now) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
severity := conditions.Warning
|
severity := conditions.Warning
|
||||||
@@ -357,7 +371,7 @@ func watchReports(f *signalFacts) []conditions.Observation {
|
|||||||
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
if m.silent(f.now) {
|
||||||
continue // silent: S1 says it, and a silent machine reports nothing
|
continue // silent: S1 says it, and a silent machine reports nothing
|
||||||
}
|
}
|
||||||
bound := max(reportAtLeast, 3*m.lastApply)
|
bound := max(reportAtLeast, 3*m.lastApply)
|
||||||
@@ -400,6 +414,34 @@ func watchPlans(f *signalFacts) []conditions.Observation {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// kindPlanFailed is S23's kind: plan.<id>.plan-failed.
|
||||||
|
const kindPlanFailed = "plan-failed"
|
||||||
|
|
||||||
|
// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no
|
||||||
|
// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to
|
||||||
|
// the operator). A plan failed longer ago than this is left to `plans`.
|
||||||
|
const planFailedWithin = 7 * 24 * time.Hour
|
||||||
|
|
||||||
|
// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when
|
||||||
|
// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2).
|
||||||
|
// The condition ends when the plan is no longer failed, because it is no longer among the facts.
|
||||||
|
func watchPlansFailed(f *signalFacts) []conditions.Observation {
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, p := range f.failed {
|
||||||
|
if f.now.Sub(p.since) > planFailedWithin {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
headline, explanation, needs, actions, resolved := planFailedWords(p)
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed,
|
||||||
|
Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||||
|
Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit),
|
||||||
|
p.failureSaid()),
|
||||||
|
Said: p.failureSaid() + p.refusalSaid(),
|
||||||
|
Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// kindWalkWaiting is S16's kind: plan.<id>.waiting.
|
// kindWalkWaiting is S16's kind: plan.<id>.waiting.
|
||||||
const kindWalkWaiting = "waiting"
|
const kindWalkWaiting = "waiting"
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import (
|
|||||||
func calm(now time.Time) *signalFacts {
|
func calm(now time.Time) *signalFacts {
|
||||||
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||||
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||||
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||||
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||||
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||||
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||||
@@ -160,6 +160,17 @@ var suppressions = map[string]suppression{
|
|||||||
created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}}
|
created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
// A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449).
|
||||||
|
"S23": {
|
||||||
|
inside: func(f *signalFacts) {
|
||||||
|
f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
|
||||||
|
since: f.now.Add(-planFailedWithin - time.Second)}}
|
||||||
|
},
|
||||||
|
past: func(f *signalFacts) {
|
||||||
|
f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"},
|
||||||
|
since: f.now.Add(-time.Second)}}
|
||||||
|
},
|
||||||
|
},
|
||||||
// A batch queued behind an open walk past that walk's bound, named.
|
// A batch queued behind an open walk past that walk's bound, named.
|
||||||
"S19": {
|
"S19": {
|
||||||
inside: func(f *signalFacts) {
|
inside: func(f *signalFacts) {
|
||||||
|
|||||||
@@ -0,0 +1,214 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1160/AC5, live, with the store and a real bus and no ticker: a plan of two tiers whose first tier has a machine that
|
||||||
|
// never reports, and whose second tier waits on the first (the plan's own order), passes its first tier after the bound,
|
||||||
|
// the silent key's expiry the only trigger, and sends its second tier to the machine that reported alone (issue 1160).
|
||||||
|
func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToTheOthers(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
// A quarter of an hour of silence is two seconds here.
|
||||||
|
wasSilent := applySilentAfter
|
||||||
|
applySilentAfter = 2 * time.Second
|
||||||
|
t.Cleanup(func() { applySilentAfter = wasSilent })
|
||||||
|
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := js.EnsureControllerBuckets(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacy, err := js.Conn().JetStream()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||||
|
t.Cleanup(func() { checkEvents = nil })
|
||||||
|
sentHeard := make(chan *nats.Msg, 32)
|
||||||
|
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
|
||||||
|
wasSaved := inventory.PlanSaved
|
||||||
|
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||||
|
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||||
|
|
||||||
|
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||||
|
var mu sync.Mutex
|
||||||
|
var asks []string
|
||||||
|
var sends [][]string
|
||||||
|
wasAsk := askABuild
|
||||||
|
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
||||||
|
asks = append(asks, source.Repository+"#"+id)
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = wasAsk })
|
||||||
|
digests := 0
|
||||||
|
wasSend := sendRollout
|
||||||
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
sends = append(sends, append([]string(nil), names...))
|
||||||
|
current, err := open.inventory.CurrentBuilds(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
builds := map[string]string{}
|
||||||
|
for _, m := range []string{"a", "b"} {
|
||||||
|
builds[m] = current[m].Commit
|
||||||
|
}
|
||||||
|
for _, node := range names {
|
||||||
|
digests++
|
||||||
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = wasSend })
|
||||||
|
|
||||||
|
register := func(m, commit string, asked time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||||
|
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||||
|
Asked: asked}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record := func(id, m, commit string, at time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||||
|
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
old := time.Now().Add(-2 * time.Hour)
|
||||||
|
for _, m := range []string{"a", "b"} {
|
||||||
|
record("build-"+m+"-1", m, "c1", old)
|
||||||
|
register(m, "c1", old)
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||||
|
register("a", "c2", time.Now().Add(-time.Minute))
|
||||||
|
|
||||||
|
// A machine's report on what it was sent, heard as the controller hears it.
|
||||||
|
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||||
|
seq := int64(0)
|
||||||
|
reports := func(node string) {
|
||||||
|
t.Helper()
|
||||||
|
last, found, err := inv.SentTo(ctx, node)
|
||||||
|
if err != nil || !found {
|
||||||
|
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||||
|
}
|
||||||
|
seq++
|
||||||
|
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||||
|
"report_sequence": seq})
|
||||||
|
var report link.Report
|
||||||
|
if err := json.Unmarshal(body, &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := listener.Heard(ctx, report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plan := func() inventory.Plan {
|
||||||
|
t.Helper()
|
||||||
|
p, err := inv.PlanByID(ctx, "plan-1160")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
built := time.Now().UTC()
|
||||||
|
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||||
|
"b": {},
|
||||||
|
}}
|
||||||
|
first, stopFirst := context.WithCancel(ctx)
|
||||||
|
defer stopFirst()
|
||||||
|
waitFirst := startRolloutEvents(first, open, js)
|
||||||
|
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
||||||
|
t.Cleanup(func() { stopFirst(); waitFirst() })
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, "tier 0 sent to both machines", func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return len(sends) == 1
|
||||||
|
})
|
||||||
|
reports("anchor")
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
if got := plan().Tier; got != 0 {
|
||||||
|
t.Fatalf("1160/AC5: the plan went on to tier %d before the bound, with laptop not heard from", got)
|
||||||
|
}
|
||||||
|
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
||||||
|
var moved string
|
||||||
|
eventually(t, "tier 1 passed once laptop's key expired", func() bool {
|
||||||
|
if q := plan(); q.Tier == 1 {
|
||||||
|
moved = q.Note
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") {
|
||||||
|
t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved)
|
||||||
|
}
|
||||||
|
// The ask for tier 1's build is the next step, and records its build id on the module.
|
||||||
|
eventually(t, "tier 1 asked", func() bool { m := plan().Modules["b"]; return m != nil && m.Build != "" })
|
||||||
|
b := plan().Modules["b"]
|
||||||
|
askedAt, _ := link.BuildAskedAt(b.Build)
|
||||||
|
record(b.Build, "b", "c2", askedAt)
|
||||||
|
register("b", "c2", askedAt)
|
||||||
|
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
||||||
|
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A machine silent past the bound at a tier between the first and the last is named silent and left as it was, as the
|
||||||
|
// last tier's is (novox/hq ADR 0282 §1, ADR 0360 §4-5, issue 1160): one rule, walkReports, read by the walk's end and by
|
||||||
|
// the gates between tiers.
|
||||||
|
|
||||||
|
func silentWalk(now time.Time) (inventory.Plan, []inventory.Reported) {
|
||||||
|
sent := now.Add(-time.Minute)
|
||||||
|
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
|
||||||
|
at := now.Add(-10 * time.Second)
|
||||||
|
p := inventory.Plan{Tier: 0, Tiers: [][]string{{"engine"}, {"app"}}, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"engine": {Rest: map[string]inventory.SentDeclaration{
|
||||||
|
"anchor": {Digest: "d-anchor", At: sent},
|
||||||
|
"g14": {Digest: "d-g14", At: long},
|
||||||
|
"fresh": {Digest: "d-fresh", At: sent}}},
|
||||||
|
"app": {}}}
|
||||||
|
reports := []inventory.Reported{{Node: "anchor", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-anchor"}}
|
||||||
|
return p, reports
|
||||||
|
}
|
||||||
|
|
||||||
|
func needsOf(running ...string) []gateNeed {
|
||||||
|
return []gateNeed{{module: "engine", running: running, since: func(string) time.Time { return time.Time{} }}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1160/AC1: at a middle tier the machine silent past the bound is named, not waited for, and the plan goes on with the
|
||||||
|
// machines that reported; one still inside the bound is waited for.
|
||||||
|
func TestAMiddleTierGateNamesTheSilentMachineAndGoesOnWithThoseThatReported(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
p, reports := silentWalk(now)
|
||||||
|
silent := silentNodes(p, reports, now, nil)
|
||||||
|
waiting, quiet, lost := settleTierGate(needsOf("anchor", "g14"), reports, silent)
|
||||||
|
if len(waiting) != 0 || !slices.Equal(quiet, []string{"g14"}) || lost {
|
||||||
|
t.Fatalf("1160/AC1: waiting %v, silent %v, lost %v; want g14 named silent and nothing waited for", waiting, quiet, lost)
|
||||||
|
}
|
||||||
|
waiting, quiet, _ = settleTierGate(needsOf("anchor", "g14", "fresh"), reports, silent)
|
||||||
|
if len(waiting) != 1 || waiting[0] != "engine on fresh" || !slices.Equal(quiet, []string{"g14"}) {
|
||||||
|
t.Fatalf("1160/AC1: a machine inside the bound is still waited for: waiting %v, silent %v", waiting, quiet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1160/AC2: the walk's end and the gate read the same silent machines.
|
||||||
|
func TestTheGateAndTheWalksEndShareOneRuleOfSilence(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
p, reports := silentWalk(now)
|
||||||
|
_, _, ended := walkReports(p, reports, now, nil)
|
||||||
|
got := silentNodes(p, reports, now, nil)
|
||||||
|
if len(got) != len(ended) || !got["g14"] || !slices.Equal(ended, []string{"g14"}) {
|
||||||
|
t.Fatalf("1160/AC2: the gate reads %v silent, the walk's end %v", got, ended)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1160/AC3: a later tier is not sent to a machine silent.
|
||||||
|
func TestALaterTierIsNotSentToASilentMachine(t *testing.T) {
|
||||||
|
got := withoutSilent([]string{"anchor", "g14"}, map[string]bool{"g14": true})
|
||||||
|
if !slices.Equal(got, []string{"anchor"}) {
|
||||||
|
t.Fatalf("1160/AC3: a later tier goes to %v; want anchor alone", got)
|
||||||
|
}
|
||||||
|
in := []string{"anchor", "g14"}
|
||||||
|
withoutSilent(in, map[string]bool{"g14": true})
|
||||||
|
if !slices.Equal(in, []string{"anchor", "g14"}) {
|
||||||
|
t.Fatalf("1160/AC3: the machines running it were changed: %v", in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1160/AC4: every machine the gate waits on silent: the plan is lost, not gone on as if delivered.
|
||||||
|
func TestAGateWhoseEveryMachineIsSilentIsLost(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
p, reports := silentWalk(now)
|
||||||
|
reports = nil
|
||||||
|
p.Modules["engine"].Rest = map[string]inventory.SentDeclaration{
|
||||||
|
"g14": {Digest: "d", At: now.Add(-time.Hour)}, "laptop": {Digest: "d", At: now.Add(-time.Hour)}}
|
||||||
|
silent := silentNodes(p, reports, now, nil)
|
||||||
|
_, quiet, lost := settleTierGate(needsOf("g14", "laptop"), reports, silent)
|
||||||
|
if !lost || !slices.Equal(quiet, []string{"g14", "laptop"}) {
|
||||||
|
t.Fatalf("1160/AC4: lost %v, silent %v; want both named and the gate lost", lost, quiet)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -414,18 +414,11 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
}
|
}
|
||||||
out.nodes, err = inv.Nodes(ctx)
|
out.nodes, err = open.heardNodes(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
}
|
}
|
||||||
for _, n := range out.nodes {
|
out.quiet = notHeardLately(out.nodes, time.Now())
|
||||||
// Never heard from, or not lately. Different from failing: a machine that says nothing
|
|
||||||
// may be new, switched off, or unreachable, and none of those is a machine that tried
|
|
||||||
// and could not.
|
|
||||||
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
|
|
||||||
out.quiet = append(out.quiet, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out.behind, err = inv.Behind(ctx)
|
out.behind, err = inv.Behind(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return answers{}, err
|
return answers{}, err
|
||||||
|
|||||||
@@ -0,0 +1,221 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What stopped moving (novox/hq ADR 0433 §2, §3, issue 1156). The watchman writes `stuck.<bucket>.<key>` in its own
|
||||||
|
// bucket for each key whose pulse expired, and again as moved when the key moves; it never deletes one. The controller
|
||||||
|
// reads that bucket in two places and holds no timer for either: a probe of the self-check raises each key not moved as
|
||||||
|
// a condition of the delivery scope, and the check that a machine is silent takes a node's presence key as its verdict.
|
||||||
|
const (
|
||||||
|
kindStuck = "stuck"
|
||||||
|
probeStuckID = "DM"
|
||||||
|
)
|
||||||
|
|
||||||
|
// presenceVerdict is the watchman's word on whether a machine is online, from its stuck presence key.
|
||||||
|
type presenceVerdict int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// presenceUnsaid: the watchman has written nothing for the machine; the heartbeat rule judges it, as before.
|
||||||
|
presenceUnsaid presenceVerdict = iota
|
||||||
|
// presenceHeard: the key says moved, the machine is back online.
|
||||||
|
presenceHeard
|
||||||
|
// presenceSilent: the key says not moved, the machine was offline past its bound.
|
||||||
|
presenceSilent
|
||||||
|
)
|
||||||
|
|
||||||
|
// presenceEntry is presence.<node> as the watchman wrote it: whether the machine is online and when the entry was made.
|
||||||
|
// The watchman writes it only when it changes, so for an online machine the time is when it connected.
|
||||||
|
type presenceEntry struct {
|
||||||
|
Online bool
|
||||||
|
At time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyPresence sets what the watchman says of a machine: the verdict of its stuck presence key (nil when there is none).
|
||||||
|
// Last heard is lastHeard's: now when online, the entry's time when offline, the store's last report only without an entry.
|
||||||
|
func applyPresence(m *machineFacts, stuck *sdkdelivery.Stuck, entry *presenceEntry, now time.Time) {
|
||||||
|
switch {
|
||||||
|
case stuck == nil:
|
||||||
|
m.presence = presenceUnsaid
|
||||||
|
case stuck.Moved:
|
||||||
|
m.presence = presenceHeard
|
||||||
|
default:
|
||||||
|
m.presence = presenceSilent
|
||||||
|
}
|
||||||
|
m.lastHeard = lastHeard(m.lastHeard, entry, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// silent says the machine is silent: the watchman's verdict where it gave one, else the heartbeat rule. A machine never
|
||||||
|
// heard from is not silent, it has not joined.
|
||||||
|
func (m machineFacts) silent(now time.Time) bool {
|
||||||
|
switch m.presence {
|
||||||
|
case presenceSilent:
|
||||||
|
return true
|
||||||
|
case presenceHeard:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > silentBound
|
||||||
|
}
|
||||||
|
|
||||||
|
// stuckObservations are the conditions of what stopped moving: one for each key not moved, in the delivery scope, kind
|
||||||
|
// stuck, naming the key and what it waited for. A node's presence key is left to the silent check; a moved key raises
|
||||||
|
// nothing, so its condition clears.
|
||||||
|
func stuckObservations(entries map[string]sdkdelivery.Stuck) []conditions.Observation {
|
||||||
|
keys := make([]string, 0, len(entries))
|
||||||
|
for k := range entries {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
var out []conditions.Observation
|
||||||
|
for _, k := range keys {
|
||||||
|
s := entries[k]
|
||||||
|
if s.Moved {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if s.Bucket == sdkdelivery.PresenceBucket {
|
||||||
|
if node, _, _, err := sdkdelivery.ParsePresenceKey(s.Key); err == nil && node != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
waited := s.Waited
|
||||||
|
if waited == "" {
|
||||||
|
waited = "a move"
|
||||||
|
}
|
||||||
|
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: k, Kind: kindStuck,
|
||||||
|
Severity: conditions.Warning,
|
||||||
|
Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", k, s.Since.UTC().Format("2006-01-02 15:04 MST"),
|
||||||
|
waited),
|
||||||
|
Said: fmt.Sprintf("%s: waiting for %s since %s", k, waited, s.Since.UTC().Format(time.RFC3339))})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// readStuck is every key of the stuck bucket and its value; none when the bucket is not there yet (the watchman not
|
||||||
|
// delivered). A value of another version is left out, said.
|
||||||
|
func readStuck(ctx context.Context, conn *nats.Conn) (map[string]sdkdelivery.Stuck, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||||
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the watchman's stuck bucket: %w", err)
|
||||||
|
}
|
||||||
|
keys, err := boundedCall(ctx, "listing the stuck keys", func(ctx context.Context) ([]string, error) { return kv.Keys(ctx) })
|
||||||
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]sdkdelivery.Stuck{}
|
||||||
|
for _, key := range keys {
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
s, err := sdkdelivery.DecodeStuck(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the stuck key %s is not read: %v\n", key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[key] = s
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readPresence is, by node, presence.<node>'s entry; a node without an entry is left out.
|
||||||
|
func readPresence(ctx context.Context, conn *nats.Conn, nodes []string) (map[string]presenceEntry, error) {
|
||||||
|
api, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
|
||||||
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the watchman's presence bucket: %w", err)
|
||||||
|
}
|
||||||
|
out := map[string]presenceEntry{}
|
||||||
|
for _, node := range nodes {
|
||||||
|
key, err := sdkdelivery.PresenceNodeKey(node)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
p, err := sdkdelivery.DecodePresence(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("the presence key %s is not read: %v\n", key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[node] = presenceEntry{Online: p.Online, At: e.Created()}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// withPresence says of each machine what the watchman says: its stuck presence key and its presence entry's time.
|
||||||
|
func withPresence(ctx context.Context, conn *nats.Conn, machines []machineFacts, now time.Time) error {
|
||||||
|
stuck, err := readStuck(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
names := make([]string, len(machines))
|
||||||
|
for i, m := range machines {
|
||||||
|
names[i] = m.name
|
||||||
|
}
|
||||||
|
entries, err := readPresence(ctx, conn, names)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for i := range machines {
|
||||||
|
var said *sdkdelivery.Stuck
|
||||||
|
if key, err := sdkdelivery.StuckKey(sdkdelivery.PresenceBucket, machines[i].name); err == nil {
|
||||||
|
if s, ok := stuck[key]; ok {
|
||||||
|
said = &s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var entry *presenceEntry
|
||||||
|
if e, ok := entries[machines[i].name]; ok {
|
||||||
|
entry = &e
|
||||||
|
}
|
||||||
|
applyPresence(&machines[i], said, entry, now)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeStuck is DM: every key the watchman says stopped moving is raised, but a node's presence, which the silent check
|
||||||
|
// takes.
|
||||||
|
func probeStuck(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||||
|
if d.js == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
entries, err := readStuck(ctx, d.js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return stuckObservations(entries), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1156/AC4: on the embedded bus with the watchman's buckets, a stuck key written raises a condition and written as moved
|
||||||
|
// clears it; a node's presence key not moved says the machine silent through the facts, and moved says it heard.
|
||||||
|
func TestAStuckKeyWrittenAndThenMovedRaisesAndClearsACondition(t *testing.T) { // 1156/AC4
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
if _, err := broker.RaiseBuckets(js, []broker.Bucket{
|
||||||
|
{Module: sdkdelivery.WatchmanModule, Name: "stuck"},
|
||||||
|
{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
write := func(bucket, key string, moved bool) string {
|
||||||
|
t.Helper()
|
||||||
|
stuckKey, err := sdkdelivery.StuckKey(bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := sdkdelivery.EncodeStuck(sdkdelivery.Stuck{Bucket: bucket, Key: key, Waited: "an answer", Since: time.Now().UTC(), Moved: moved})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(ctx, stuckKey, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return stuckKey
|
||||||
|
}
|
||||||
|
d := &doctor{js: js}
|
||||||
|
stuckKey := write(sdkdelivery.PulseBucket, "builds.x", false)
|
||||||
|
write(sdkdelivery.PresenceBucket, "ace", false)
|
||||||
|
got, err := probeStuck(ctx, d)
|
||||||
|
if err != nil || len(got) != 1 || got[0].ID != stuckKey || got[0].Kind != kindStuck {
|
||||||
|
t.Fatalf("a stuck key written: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
machines := []machineFacts{{name: "ace", lastHeard: time.Now()}}
|
||||||
|
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !machines[0].silent(time.Now()) {
|
||||||
|
t.Fatal("a presence key not moved did not say the machine silent")
|
||||||
|
}
|
||||||
|
write(sdkdelivery.PulseBucket, "builds.x", true)
|
||||||
|
write(sdkdelivery.PresenceBucket, "ace", true)
|
||||||
|
if got, err := probeStuck(ctx, d); err != nil || len(got) != 0 {
|
||||||
|
t.Fatalf("a stuck key written as moved: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
machines = []machineFacts{{name: "ace", lastHeard: time.Now().Add(-time.Hour)}}
|
||||||
|
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if machines[0].silent(time.Now()) {
|
||||||
|
t.Fatal("a presence key that moved still says the machine silent")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
func aStuckKey(t *testing.T, bucket, key string) string {
|
||||||
|
t.Helper()
|
||||||
|
k, err := sdkdelivery.StuckKey(bucket, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1156/AC1: each not-moved stuck key that is not a node's presence is an observation of the delivery scope, kind stuck,
|
||||||
|
// naming the key and what it waited for; a moved key raises nothing, so its condition clears.
|
||||||
|
func TestEachStuckKeyThatHasNotMovedIsAConditionOfTheDeliveryScope(t *testing.T) { // 1156/AC1
|
||||||
|
since := time.Date(2026, 10, 18, 19, 0, 0, 0, time.UTC)
|
||||||
|
pulse := aStuckKey(t, sdkdelivery.PulseBucket, "x")
|
||||||
|
moved := aStuckKey(t, sdkdelivery.HaveBucket, "ace.zsh")
|
||||||
|
node := aStuckKey(t, sdkdelivery.PresenceBucket, "ace")
|
||||||
|
part := aStuckKey(t, sdkdelivery.PresenceBucket, "part.mesh-controller.one")
|
||||||
|
entries := map[string]sdkdelivery.Stuck{
|
||||||
|
pulse: {Bucket: sdkdelivery.PulseBucket, Key: "x", Waited: "the build of zsh to be answered", Since: since},
|
||||||
|
moved: {Bucket: sdkdelivery.HaveBucket, Key: "ace.zsh", Waited: "a want", Since: since, Moved: true},
|
||||||
|
node: {Bucket: sdkdelivery.PresenceBucket, Key: "ace", Waited: "its return", Since: since},
|
||||||
|
part: {Bucket: sdkdelivery.PresenceBucket, Key: "part.mesh-controller.one", Waited: "its return", Since: since},
|
||||||
|
}
|
||||||
|
got := stuckObservations(entries)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("observations %+v, want the pulse key and the part's presence", got)
|
||||||
|
}
|
||||||
|
byID := map[string]conditions.Observation{}
|
||||||
|
for _, o := range got {
|
||||||
|
if o.Scope != conditions.ScopeDelivery || o.Kind != kindStuck {
|
||||||
|
t.Errorf("%+v is not a stuck condition of the delivery scope", o)
|
||||||
|
}
|
||||||
|
byID[o.ID] = o
|
||||||
|
}
|
||||||
|
o, ok := byID[pulse]
|
||||||
|
if !ok || !strings.Contains(o.Summary, pulse) || !strings.Contains(o.Summary, "the build of zsh to be answered") {
|
||||||
|
t.Fatalf("the condition does not name the key and what it waited for: %+v", got)
|
||||||
|
}
|
||||||
|
if _, ok := byID[part]; !ok {
|
||||||
|
t.Errorf("a part's presence is left out: %v", byID)
|
||||||
|
}
|
||||||
|
// Moved again: the same key, written as moved, raises nothing.
|
||||||
|
entries[pulse] = sdkdelivery.Stuck{Bucket: sdkdelivery.PulseBucket, Key: "x", Since: since, Moved: true}
|
||||||
|
if got := stuckObservations(entries); len(got) != 1 {
|
||||||
|
t.Fatalf("a moved key still raises: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func aMachineHeardAt(now time.Time, ago time.Duration) machineFacts {
|
||||||
|
return machineFacts{name: "laptop", lastHeard: now.Add(-ago)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1156/AC2: a node's stuck presence key not moved makes the machine silent, unless it sleeps; moved, it is heard; no
|
||||||
|
// timer is armed (the verdict is the key's, whatever the time of the last word).
|
||||||
|
func TestAStuckPresenceKeyIsTheSilentVerdict(t *testing.T) { // 1156/AC2
|
||||||
|
now := time.Now()
|
||||||
|
f := calm(now)
|
||||||
|
f.machines = append(f.machines, aMachineHeardAt(now, 10*time.Second))
|
||||||
|
notMoved := sdkdelivery.Stuck{Bucket: sdkdelivery.PresenceBucket, Key: "laptop", Waited: "its return", Since: now.Add(-3 * time.Minute)}
|
||||||
|
applyPresence(&f.machines[1], ¬Moved, nil, now)
|
||||||
|
got := watchHeartbeats(f)
|
||||||
|
if len(got) != 1 || got[0].Machine != "laptop" || got[0].Kind != "silent" {
|
||||||
|
t.Fatalf("a presence key not moved: %+v", got)
|
||||||
|
}
|
||||||
|
// Asleep: not silent.
|
||||||
|
f.machines[1].power = link.PowerState{State: "sleeping", At: now.Add(-time.Hour)}
|
||||||
|
if got := watchHeartbeats(f); len(got) != 0 {
|
||||||
|
t.Fatalf("a sleeping machine raised %+v", got)
|
||||||
|
}
|
||||||
|
// Moved: heard, whatever the store's last word says.
|
||||||
|
f.machines[1].power = link.PowerState{}
|
||||||
|
f.machines[1].lastHeard = now.Add(-2 * time.Hour)
|
||||||
|
moved := notMoved
|
||||||
|
moved.Moved = true
|
||||||
|
applyPresence(&f.machines[1], &moved, nil, now)
|
||||||
|
if got := watchHeartbeats(f); len(got) != 0 {
|
||||||
|
t.Fatalf("a presence key that moved raised %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1156/AC3: with a presence entry that says offline, last heard is the entry's time (1183/AC1); with none,
|
||||||
|
// last_seen. An entry that says online counts as heard now: the watchman writes it only when it changes, so its time is
|
||||||
|
// when the machine connected, and a machine online for hours with no stuck key is not silent, whatever last_seen says.
|
||||||
|
func TestLastHeardFollowsThePresenceEntry(t *testing.T) { // 1156/AC3
|
||||||
|
now := time.Now()
|
||||||
|
seen := now.Add(-time.Hour)
|
||||||
|
m := machineFacts{name: "laptop", lastHeard: seen}
|
||||||
|
applyPresence(&m, nil, nil, now)
|
||||||
|
if !m.lastHeard.Equal(seen) {
|
||||||
|
t.Fatalf("with no presence entry last heard is %v, want last_seen %v", m.lastHeard, seen)
|
||||||
|
}
|
||||||
|
offlineAt := now.Add(-5 * time.Minute)
|
||||||
|
applyPresence(&m, nil, &presenceEntry{Online: false, At: offlineAt}, now)
|
||||||
|
if !m.lastHeard.Equal(offlineAt) {
|
||||||
|
t.Fatalf("offline since %v: last heard is %v", offlineAt, m.lastHeard)
|
||||||
|
}
|
||||||
|
m = machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
|
||||||
|
applyPresence(&m, nil, &presenceEntry{Online: false, At: now.Add(-time.Hour)}, now)
|
||||||
|
if !m.lastHeard.Equal(now.Add(-time.Hour)) {
|
||||||
|
t.Fatalf("an offline entry's time is last heard, not a report: %v", m.lastHeard)
|
||||||
|
}
|
||||||
|
for _, lastSeen := range []time.Duration{10 * time.Second, 5 * time.Hour} {
|
||||||
|
m := machineFacts{name: "laptop", lastHeard: now.Add(-lastSeen)}
|
||||||
|
applyPresence(&m, nil, &presenceEntry{Online: true, At: now.Add(-6 * time.Hour)}, now)
|
||||||
|
if m.silent(now) {
|
||||||
|
t.Fatalf("a machine online since hours ago (last_seen %v ago) with no stuck key is silent", lastSeen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user