novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Delete the controller's plan machinery: plans, batches, merge window, tiers, gate, retry and healers H1 and H2 (release 6, issue 1945 B7)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 16:27:55 +02:00
parent 1b0fd8b442
commit dc456d5660
236 changed files with 572 additions and 44491 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/ask_bus_wait_test.go
-23
View File
@@ -1,23 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting:
// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue
// 1403, met again once the hand rebuild no longer resumed the plan first).
func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) {
since := time.Now().UTC()
p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}},
Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}}
waiting := busWaitingOnRetry(p)
_, said := retryWords(p, []string{"a"}, nil, waiting)
if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") {
t.Fatalf("a retry left for the bus is said as %q", said)
}
}
cmd/mesh-controller/asker_retry_integration_test.go
-138
View File
@@ -1,138 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
asksRecorded(t)
planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
twoTiers(t, open)
before := time.Now().UTC().Add(-time.Hour)
ended := before.Add(time.Minute)
failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
Note: "a failed to build in tier 0",
Delivery: &inventory.PlanDelivery{},
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
Why: "killed by hand"}},
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
t.Fatal(err)
}
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// The controller's seat answers `plans` as it does: by running the verb.
sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
var args map[string]string
_ = json.Unmarshal(m.Data, &args)
answer := link.Answer{Result: json.RawMessage(`"ok"`)}
if len(args) != 1 || args["retry"] == "" {
answer = link.Answer{Error: "the retry was asked with more than the plan"}
} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
answer = link.Answer{Error: err.Error()}
}
body, _ := json.Marshal(answer)
_ = m.Respond(body)
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
// What S23 raises for it, read from the store as the watchdog reads it.
raised := func(context.Context) ([]conditions.Condition, error) {
facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
if err != nil {
return nil, err
}
var out []conditions.Condition
for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
}
return out, nil
}
now := time.Now()
store := memAskedStore{}
var sent []published
var acts []link.HandAct
a := &asker{open: raised, store: store,
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
publish: func(_ context.Context, subject string, body []byte, id string) error {
sent = append(sent, published{subject, id, body})
return nil
},
call: callAction(conn),
record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
now: func() time.Time { return now },
logf: t.Logf}
cs, _ := raised(ctx)
if len(cs) != 1 || len(cs[0].Actions) != 1 {
t.Fatalf("the failed plan raised %+v", cs)
}
if err := a.reconcile(ctx); err != nil {
t.Fatal(err)
}
var w asks.Warrant
for _, r := range store {
for _, o := range r.Ask.Options {
if o.Label == "Retry" {
w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
AskDigest: r.Ask.Digest(),
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
}
}
}
if w.Ask == "" {
t.Fatalf("no ask offers Retry: %+v", store)
}
body, _ := json.Marshal(w)
if err := a.Decided(ctx, body); err != nil {
t.Fatal(err)
}
if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
t.Fatal(err)
}
p, err := open.inventory.PlanByID(ctx, failed.ID)
if err != nil {
t.Fatal(err)
}
m := p.Modules["a"]
if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" {
t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
}
if got := store[w.Ask]; got.Acted != "done" {
t.Fatalf("the ask ended %q", got.Acted)
}
if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
t.Fatalf("the warrant was recorded as %+v", acts)
}
if cs, _ = raised(ctx); len(cs) != 0 {
t.Fatalf("the plan-failed condition did not end: %+v", cs)
}
}
cmd/mesh-controller/asker_test.go
+9 -214
View File
@@ -125,8 +125,15 @@ func newAskerRig(t *testing.T) *askerRig {
}
func heldCondition() conditions.Condition {
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove,
Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}}
stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove,
Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}}
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096", Kind: "delivery-stalled",
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Headline: "Delivery of hq held for 36 hours",
Explanation: "A delivery of hq has been held for 36 hours, past its limit.",
Needs: "release it, or stop it, " + FromMeshMCPServer, Actions: []conditions.Action{release, stop}}
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
}
@@ -1026,215 +1033,3 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
}
}
// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
func failedPlanCondition(plan string) conditions.Condition {
p := aFailedPlan(plan, nil)
f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
f.failed = []failedPlanFacts{failedFactsFor(p)}
o := watchPlansFailed(f)[0]
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
}
// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
r := newAskerRig(t)
c := failedPlanCondition("plan-1792433168676870614")
r.open = []conditions.Condition{c}
if err := r.a.reconcile(context.Background()); err != nil {
t.Fatal(err)
}
sent := r.asksSent(t)
if len(sent) != 1 || len(sent[0].Options) != 1 {
t.Fatalf("asked %+v", sent)
}
o := sent[0].Options[0]
if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
t.Errorf("the option is %+v", o)
}
if err := sent[0].Check(r.now); err != nil {
t.Errorf("the ask is refused as made: %v", err)
}
if strings.Contains(sent[0].Explanation, "mesh MCP server") {
t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
}
}
// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
// the same warrant heard again does nothing.
func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
plan := "plan-1792433168676870614"
r := newAskerRig(t)
c := failedPlanCondition(plan)
r.open = []conditions.Condition{c}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, c.Key, "Retry")
answerWith(t, r, w)
answerWith(t, r, w)
if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
t.Fatalf("called %v, want exactly %s", r.called, want)
}
if len(r.acts) != 1 {
t.Fatalf("hand-acts %+v", r.acts)
}
act := r.acts[0]
if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
t.Errorf("the hand-act is %+v", act)
}
if got := r.store[w.Ask]; got.Acted != "done" {
t.Errorf("kept %+v", got)
}
}
// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is
// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before.
func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) {
plan := "plan-1792433168676870614"
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
for _, tc := range []struct {
name string
c conditions.Condition
label string
}{
{"retry", failedPlanCondition(plan), "Retry"},
{"stop", waiting, "Stop"},
} {
t.Run(tc.name, func(t *testing.T) {
r := newAskerRig(t)
r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"}
r.open = []conditions.Condition{tc.c}
_ = r.a.reconcile(context.Background())
w := r.warrantFor(t, tc.c.Key, tc.label)
answerWith(t, r, w)
answerWith(t, r, w) // heard again: told once
if len(r.told) != 1 {
t.Fatalf("told %+v", r.told)
}
got := r.told[0]
if got.headline != tc.c.Headline {
t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline)
}
said := strings.Join(got.lines, " ")
if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) {
t.Errorf("the lines do not carry the answer and the refusal's words: %q", said)
}
if why, ok := conditions.Plain(said); !ok {
t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said)
}
if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") {
t.Errorf("the failure is not recorded as before: %+v", r.acts)
}
if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") {
t.Errorf("kept %+v", kept)
}
})
}
}
// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here.
func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) {
r := newAskerRig(t)
c := failedPlanCondition("plan-1792433168676870614")
r.open = []conditions.Condition{c}
_ = r.a.reconcile(context.Background())
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
if len(r.told) != 0 {
t.Fatalf("a retry that was done told %+v", r.told)
}
r = newAskerRig(t)
r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"}
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
r.open = []conditions.Condition{module}
_ = r.a.reconcile(context.Background())
answerWith(t, r, r.warrantFor(t, module.Key, "Restart"))
if len(r.told) != 0 || len(r.acts) != 1 {
t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts)
}
}
// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was
// changed"; and the sentence a refusal falls back to fits the answer given.
func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) {
plan := "plan-1792433168676870614"
r := newAskerRig(t)
r.refuses = errors.New("mesh-controller did not answer plans within 30s")
c := failedPlanCondition(plan)
r.open = []conditions.Condition{c}
_ = r.a.reconcile(context.Background())
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
if len(r.told) != 1 {
t.Fatalf("told %+v", r.told)
}
said := strings.Join(r.told[0].lines, " ")
if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") {
t.Errorf("a call that may have run is told as %q", said)
}
if why, ok := conditions.Plain(said); !ok {
t.Errorf("not plain: %s: %q", why, said)
}
}
func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) {
plan := "plan-1792433168676870614"
for _, tc := range []struct {
act conditions.Action
want string
}{
{retryAction(plan), "a retry would not run now"},
{conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"},
{conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"},
} {
// Words the operator's channel withholds: a hash and a dotted name.
got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act)
if !strings.Contains(got, tc.want) {
t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want)
}
if _, ok := conditions.Plain(got); !ok {
t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got)
}
}
}
// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the
// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not.
func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) {
r := newAskerRig(t)
c := failedPlanCondition("plan-1792433168676870614")
c.Raised = r.now.Add(-time.Minute)
r.open = []conditions.Condition{c}
_ = r.a.reconcile(context.Background())
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
// Still open right after the answer: not asked again at once.
r.now = r.now.Add(time.Minute)
_ = r.a.reconcile(context.Background())
if n := len(r.asksSent(t)); n != 1 {
t.Fatalf("asked again while the same condition stayed open: %d", n)
}
// The retry ran, the plan left failed and the condition ended.
r.now = r.now.Add(time.Minute)
r.open = nil
_ = r.a.reconcile(context.Background())
// It fails again, ten minutes after the answer: a condition raised after the answered ask ended.
r.now = r.now.Add(8 * time.Minute)
again := failedPlanCondition("plan-1792433168676870614")
again.Raised = r.now
r.open = []conditions.Condition{again}
if err := r.a.reconcile(context.Background()); err != nil {
t.Fatal(err)
}
if n := len(r.asksSent(t)); n != 2 {
t.Fatalf("the second failure was asked %d times in all, want 2", n)
}
if len(r.called) != 1 {
t.Fatalf("something retried without an answer: %v", r.called)
}
}
cmd/mesh-controller/away_integration_test.go
-263
View File
@@ -1,263 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builderkeys"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it
// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard
// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's
// builds, so the send it gets when it is heard again carries them (AC3).
func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
// laptop is away as every tier is sent; anchor is heard.
wasAway := judgeAway
judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) {
return map[string]bool{"laptop": true}, nil
}
t.Cleanup(func() { judgeAway = wasAway })
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
legacy, err := js.Conn().JetStream()
if err != nil {
t.Fatal(err)
}
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
t.Cleanup(func() { checkEvents = nil })
sentHeard := make(chan *nats.Msg, 32)
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
wasSaved := inventory.PlanSaved
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
var mu sync.Mutex
var sends [][]string
everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it
// Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look
// goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask
// was where the walk paused).
wholeKeys := openPlanKeys
var holdB atomic.Bool
holdB.Store(true)
openPlanKeys = func() (planKeyReads, func(), error) {
all, done, err := wholeKeys()
return keysHolding{all, "b", &holdB, ""}, done, err
}
digests := 0
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
mu.Lock()
defer mu.Unlock()
sends = append(sends, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
builds := map[string]string{}
for _, m := range []string{"a", "b"} {
builds[m] = current[m].Commit
}
for _, node := range names {
digests++
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
return nil, err
}
}
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
register := func(m, commit string, asked time.Time) {
t.Helper()
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
record := func(id, m, commit string, at time.Time) {
t.Helper()
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
old := time.Now().Add(-2 * time.Hour)
for _, m := range []string{"a", "b"} {
record("build-"+m+"-1", m, "c1", old)
register(m, "c1", old)
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, n, m); err != nil {
t.Fatal(err)
}
}
}
for _, n := range []string{"anchor", "laptop"} {
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
t.Fatal(err)
}
}
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
register("a", "c2", time.Now().Add(-time.Minute))
// A machine's report on what it was sent, heard as the controller hears it.
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
seq := int64(0)
reports := func(node string) {
t.Helper()
last, found, err := inv.SentTo(ctx, node)
if err != nil || !found {
t.Fatalf("%s was sent nothing: %v", node, err)
}
seq++
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
"report_sequence": seq})
var report link.Report
if err := json.Unmarshal(body, &report); err != nil {
t.Fatal(err)
}
if _, err := listener.Heard(ctx, report); err != nil {
t.Fatal(err)
}
}
plan := func() inventory.Plan {
t.Helper()
p, err := inv.PlanByID(ctx, "plan-1160")
if err != nil {
t.Fatal(err)
}
return p
}
built := time.Now().UTC()
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
Modules: map[string]*inventory.PlanModule{
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
"b": {},
}}
first, stopFirst := context.WithCancel(ctx)
defer stopFirst()
waitFirst := startRolloutEvents(first, open, js)
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
t.Cleanup(func() { stopFirst(); waitFirst() })
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
eventually(t, "tier 0 sent to anchor alone", func() bool {
mu.Lock()
defer mu.Unlock()
return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor"
})
reports("anchor")
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
var moved string
eventually(t, "tier 1 reached on anchor's report alone", func() bool {
if q := plan(); q.Tier == 1 {
moved = q.Note
return true
}
return false
})
if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
}
// Tier 1's module is taken from its key once that is built, which the walk's look does by itself.
holdB.Store(false)
askToLook()
eventually(t, "tier 1 sent to anchor alone", func() bool {
mu.Lock()
defer mu.Unlock()
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
})
// 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again.
was, known, err := inv.SentBuilds(ctx, "laptop")
if err != nil || !known {
t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known)
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
t.Fatal(err)
}
if was["a"] == current["a"].Commit || was["a"] != "c1" {
t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit)
}
if got := plan().Away; strings.Join(got, ",") != "laptop" {
t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
}
}
// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of
// the fingerprint fp when that is given.
type keysHolding struct {
planKeyReads
module string
held *atomic.Bool
fp string
}
func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) {
if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) {
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
}
return k.planKeyReads.Build(ctx, module, fp)
}
// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared,
// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of
// the test, and the flag is set until the test lowers it.
func holdKeys(t *testing.T, modules ...string) *atomic.Bool {
t.Helper()
held := &atomic.Bool{}
held.Store(true)
was := openPlanKeys
openPlanKeys = func() (planKeyReads, func(), error) {
all, done, err := was()
var keys planKeyReads = all
for _, m := range modules {
keys = keysHolding{keys, m, held, ""}
}
return keys, done, err
}
t.Cleanup(func() { openPlanKeys = was })
return held
}
cmd/mesh-controller/backups_before_send_integration_test.go
-312
View File
@@ -1,312 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"os"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and
// answers `backed-up` with the last night it is told (novox/hq ADR 0404).
type aBackupSeatOnTheBus struct {
mu sync.Mutex
nows []string
night map[string]any // module → lastNight
}
func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) {
t.Helper()
reply := func(m *nats.Msg, result any) {
body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node})
_ = m.Respond(body)
}
now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) {
var a struct {
Module string `json:"module"`
}
_ = json.Unmarshal(m.Data, &a)
f.mu.Lock()
f.nows = append(f.nows, a.Module)
f.mu.Unlock()
reply(m, map[string]any{"started": a.Module})
})
if err != nil {
t.Fatal(err)
}
read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
f.mu.Lock()
defer f.mu.Unlock()
var out []map[string]any
for module, n := range f.night {
out = append(out, map[string]any{"module": module, "lastNight": n})
}
reply(m, out)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() })
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
}
func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) {
f.mu.Lock()
defer f.mu.Unlock()
f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}}
}
func (f *aBackupSeatOnTheBus) asks() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.nows)
}
// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits
// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the
// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the
// backups again, and the plan goes on. One bus and one store for the scenarios.
func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := asksRecorded(t)
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
`"resources":[{"id":"files","type":"directory"}]}`))
if err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
if _, err := assign(ctx, open, node, "keeper"); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, node, "notes"); err != nil {
t.Fatal(err)
}
}
seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}}
for node, s := range seats {
s.serve(t, js.Conn(), node)
}
// Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup.
wasWithin, wasEvery := backupsWithin, backupsPollEvery
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
newPlan := func(id string) {
t.Helper()
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
}
read := func(id string) inventory.Plan {
t.Helper()
p, err := inv.PlanByID(ctx, id)
if err != nil {
t.Fatal(err)
}
return p
}
// 1. Both machines are asked; while one has not answered, no build is asked.
newPlan("plan-one")
advancePlans(ctx, open)
if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 {
t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked)
}
seats["anchor"].answers("notes", true, "")
advancePlans(ctx, open)
if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending {
t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups)
}
seats["laptop"].answers("notes", true, "")
advancePlans(ctx, open)
if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone {
t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups)
}
finish := read("plan-one")
finish.State = inventory.PlanDone
if err := inv.SavePlan(ctx, &finish); err != nil {
t.Fatal(err)
}
// 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised.
seats["anchor"].night, seats["laptop"].night = nil, nil
newPlan("plan-two")
advancePlans(ctx, open)
seats["anchor"].answers("notes", true, "")
seats["laptop"].answers("notes", false, "no space left on device")
advancePlans(ctx, open)
held := read("plan-two")
if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 {
t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked)
}
var failed *inventory.PlanBackup
for i := range held.Backups.Asked {
if held.Backups.Asked[i].State == inventory.BackupFailed {
failed = &held.Backups.Asked[i]
}
}
if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" ||
strings.Contains(held.Note, "tried again") {
t.Fatalf("the failed backup: %+v note=%q", failed, held.Note)
}
advancePlans(ctx, open) // not retried on the next look
if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 ||
seats["laptop"].asks() != 2 {
t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks())
}
said, err := probeBackupHeld(ctx, &doctor{open: open})
if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") {
t.Fatalf("the condition: %+v %v", said, err)
}
// 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on.
seats["laptop"].answers("notes", false, "no space left on device")
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by",
"mesh-delivery for jochen"}); err != nil {
t.Fatal(err)
}
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil {
t.Fatal("a plan with no failed backup was asked again")
}
seats["laptop"].night = nil
advancePlans(ctx, open)
if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 {
t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks())
}
seats["laptop"].answers("notes", true, "")
advancePlans(ctx, open)
// One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key
// build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes
// it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key.
if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 ||
got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" {
t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked)
}
if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
t.Fatalf("the condition stays: %+v", said)
}
}
// Issue 1086: a plan cut from a merge asked its first tier at the cut, so a module keeping data was sent without the
// backup the plan promised. The cut now takes the backups first (askFirstTierAtCut, batches.go), and the walk's own
// step asks the tier once they are done.
func TestAPlanCutFromAMergeOfABackedUpModuleAsksItsBackupAndNotItsTierUntilItIsDone(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
settings := t.TempDir() + "/merge-window.json"
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv(mergeWindowFileVar, settings)
everyHeadWaits(t)
asked := asksRecorded(t)
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
`"resources":[{"id":"files","type":"directory"}]}`))
if err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/notes", Seat: "git", Ref: "main",
BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
for _, m := range []string{"keeper", "notes"} {
if _, err := assign(ctx, open, "anchor", m); err != nil {
t.Fatal(err)
}
}
seat := &aBackupSeatOnTheBus{}
seat.serve(t, js.Conn(), "anchor")
wasWithin, wasEvery := backupsWithin, backupsPollEvery
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
// 1. A merge of notes is heard and its window closes: the batch is cut through cutBatch.
hear(t, open, repoMerge("notes", "c-notes", t0), t0.Add(time.Second))
cutAt(t, open, t0.Add(time.Second+90*time.Second))
ws, _ := walks(t, open)
if len(ws) != 1 {
t.Fatalf("%d walk(s)", len(ws))
}
// 2. Right after the cut no tier is asked, the record has anchor/notes pending, the seat heard one `now`.
cut, err := inv.PlanByID(ctx, ws[0].ID)
if err != nil {
t.Fatal(err)
}
if len(*asked) != 0 || askedInTier(cut) {
t.Fatalf("the first tier was asked at the cut: %v %+v", *asked, cut.Modules)
}
if cut.Backups == nil || cut.Backups.State != inventory.BackupPending || len(cut.Backups.Asked) != 1 ||
cut.Backups.Asked[0].Machine != "anchor" || cut.Backups.Asked[0].Module != "notes" {
t.Fatalf("the backups of the cut: %+v", cut.Backups)
}
if seat.asks() != 1 {
t.Fatalf("the seat heard %d `now`", seat.asks())
}
// 3. The backup answers done: the walk's own step asks the first tier, once, and asks no second backup.
seat.answers("notes", true, "")
advancePlans(ctx, open)
advancePlans(ctx, open)
after, err := inv.PlanByID(ctx, ws[0].ID)
if err != nil {
t.Fatal(err)
}
if len(*asked) != 1 || after.Backups.State != inventory.BackupDone || seat.asks() != 1 {
t.Fatalf("builds=%v backups=%+v asks=%d", *asked, after.Backups, seat.asks())
}
}
cmd/mesh-controller/batches.go
-1797
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/batches_integration_test.go
-727
View File
@@ -1,727 +0,0 @@
//go:build integration
package main
import (
"bytes"
"io"
"os"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
func windowed(t *testing.T) *stores {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
settings := t.TempDir() + "/merge-window.json"
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
t.Fatal(err)
}
t.Setenv(mergeWindowFileVar, settings)
noGroups(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
Head: "c0"}); err != nil {
t.Fatal(err)
}
for _, m := range []struct{ module, repository, path string }{
{"app", "novox/mesh-catalog", "modules/app"},
{"notes", "novox/mesh-catalog", "modules/notes"},
{"one", "novox/one", ""},
{"two", "novox/two", ""},
{"three", "novox/three", ""},
} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
Head: "c0"}); err != nil {
t.Fatal(err)
}
}
return open
}
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
t.Helper()
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
t.Fatal(err)
}
}
func cutAt(t *testing.T, open *stores, now time.Time) {
t.Helper()
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
t.Fatal(err)
}
}
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
claude := catalogueMerge("553b7191claude", "app", t0)
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
hear(t, open, claude, t0.Add(time.Second))
hear(t, open, dunst, t0.Add(18*time.Second))
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
}
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
if ws, _ := walks(t, open); len(ws) != 0 {
t.Fatalf("cut before the window closed: %+v", ws)
}
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
ws, bs := walks(t, open)
if len(ws) != 1 || len(bs) != 0 {
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
}
w := ws[0]
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
}
for _, m := range []string{"app", "notes"} {
if _, in := w.Modules[m]; !in {
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
}
}
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
}
// The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at
// which commit: the one the branch holds, which is the later merge the walk carries.
for _, m := range w.Tiers[w.Tier] {
if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit {
t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s)
}
}
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
hear(t, open, claude, t0.Add(5*time.Minute))
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
}
}
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "feat/three")
asksRecorded(t)
for i, r := range []string{"one", "two", "three"} {
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
t0.Add(time.Duration(i)*20*time.Second+time.Second))
}
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
ws, bs := walks(t, open)
if len(ws) != 1 || len(bs) != 0 {
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
}
w := ws[0]
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
}
for _, r := range []string{"one", "two", "three"} {
if w.CommitOf("novox/"+r) != "c-"+r {
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
}
if _, in := w.Modules[r]; !in {
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
}
}
// Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build).
for _, r := range w.Tiers[w.Tier] {
if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r {
t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s)
}
}
}
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "feat/both", "feat/busy")
asksWithPaths(t)
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
cutAt(t, open, t0.Add(100*time.Second))
if ws, _ := walks(t, open); len(ws) != 0 {
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
}
cutAt(t, open, t0.Add(170*time.Second))
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
}
busy := windowed(t)
groupsWaiting(t, "feat/both", "feat/busy")
asksWithPaths(t)
var last time.Time
for i := 0; i < 12; i++ {
last = t0.Add(time.Duration(i) * time.Minute)
repo := []string{"one", "two", "three"}[i%3]
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
if ws, _ := walks(t, busy); len(ws) > 0 {
if i != 10 {
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
}
break
}
}
ws, _ := walks(t, busy)
if len(ws) != 1 {
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
}
}
func TestTheAssemblingBatchIsShown(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/553b7191claude")
asksWithPaths(t)
var said []inventory.Plan
was := inventory.PlanSaved
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
t.Cleanup(func() { inventory.PlanSaved = was })
now := time.Now().UTC()
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
hear(t, open, claude, now.Add(-19*time.Second))
hear(t, open, dunst, now.Add(-time.Second))
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
lines := strings.Split(out, "\n")
first := lines[0]
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
if !strings.Contains(first, want) {
t.Fatalf("plans' first line %q does not say %q", first, want)
}
}
// Under it, one line per repository: the pull request, what it answers, what it moves.
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
"#174 claude-code: an agent proposes a section) · app, notes" ||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
t.Fatalf("the grouped list reads %q", lines[1:4])
}
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
len(said[len(said)-1].Delivery.Merges) != 3 {
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
}
}
func captured(t *testing.T, run func() error) string {
t.Helper()
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
was := os.Stdout
os.Stdout = w
runErr := run()
os.Stdout = was
_ = w.Close()
var b bytes.Buffer
_, _ = io.Copy(&b, r)
if runErr != nil {
t.Fatal(runErr)
}
return b.String()
}
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(90*time.Second))
ws, _ := walks(t, open)
if len(ws) != 1 || !ws[0].Open() {
t.Fatalf("the first batch was not cut: %+v", ws)
}
first := ws[0]
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
cutAt(t, open, t0.Add(5*time.Minute))
ws, bs := walks(t, open)
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
}
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
t.Fatalf("the started walk was %s by the next merge", got.State)
}
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
}
first.State = inventory.PlanDone
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
t.Fatal(err)
}
cutAt(t, open, t0.Add(6*time.Minute))
ws, bs = walks(t, open)
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
}
}
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
ctx := t.Context()
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(90*time.Second))
ws, _ := walks(t, open)
if len(ws) != 1 || !ws[0].Waiting() {
t.Fatalf("the walk does not wait for its word: %+v", ws)
}
waiting := ws[0]
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
ws, bs := walks(t, open)
if len(bs) != 0 || len(ws) != 2 {
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
}
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
if err != nil {
t.Fatal(err)
}
newer := ws[0]
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
}
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
}
if _, in := newer.Modules["one"]; !in {
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
}
}
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
ctx := t.Context()
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
hear(t, open, later, t0.Add(18*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
w := ws[0]
w.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &w); err != nil {
t.Fatal(err)
}
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
got, _ := open.inventory.PlanByID(ctx, w.ID)
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
Carried: later.Commit}) {
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
}
// Moves app, which that walk never built: the next batch walks it.
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
_, bs := walks(t, open)
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
}
}
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
ctx := t.Context()
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
for i, m := range []link.SourceMoved{oldest, middle, newest} {
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
}
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
failed := ws[0]
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
t.Fatal(err)
}
cutAt(t, open, t0.Add(3*time.Minute))
got, _ := open.inventory.PlanByID(ctx, failed.ID)
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
}
ws, _ = walks(t, open)
alone := ws[0]
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
}
// The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the
// module is taken from its key at the commit the merge was walked on.
if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit {
t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules)
}
// Retried, the failed walk would name merges the search answers now.
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
}
alone.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
t.Fatal(err)
}
cutAt(t, open, t0.Add(4*time.Minute))
ws, bs := walks(t, open)
if ws[0].ID != alone.ID || len(bs) != 0 {
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
}
done, _ := open.inventory.PlanByID(ctx, alone.ID)
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
Commit: oldest.Commit, Carried: middle.Commit}) {
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
}
// A stopped walk starts no search: a person ended it.
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
cutAt(t, open, t0.Add(8*time.Minute))
ws, _ = walks(t, open)
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
t.Fatal(err)
}
cutAt(t, open, t0.Add(9*time.Minute))
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
}
}
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
for _, grouped := range []bool{true, false} {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
ctx := t.Context()
for _, m := range []string{"mesh-host"} {
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
host := repoMerge("mesh-host", "h1", t0)
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
host.Head, ctl.Head = "feat/together", "feat/together"
if !grouped {
ctl.Head = "feat/alone"
}
hear(t, open, ctl, t0.Add(2*time.Second))
hear(t, open, host, t0.Add(3*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
if len(ws) != 1 {
t.Fatalf("%d walks", len(ws))
}
tiers := ws[0].Tiers
if grouped {
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
}
} else if len(tiers) != 1 {
t.Fatalf("two merges of no group were ordered: %v", tiers)
}
}
}
func TestABatchSurvivesARestart(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "feat/both")
asksWithPaths(t)
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
// A new controller: nothing of the first one's but the store.
again, err := openStores(t.Context())
if err != nil {
t.Fatal(err)
}
t.Cleanup(again.Close)
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
cutAt(t, again, t0.Add(119*time.Second))
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
}
cutAt(t, again, t0.Add(120*time.Second))
ws, bs := walks(t, again)
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
}
}
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
ctx := t.Context()
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
hear(t, open, c2, t0)
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
w := ws[0]
w.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &w); err != nil {
t.Fatal(err)
}
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
got, _ := open.inventory.PlanByID(ctx, w.ID)
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
Commit: "c1", Carried: "c2"}) {
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
}
// A walk of two that built only two: a late merge of one is not its to name.
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
cutAt(t, open, t0.Add(18*time.Minute))
ws, _ = walks(t, open)
w = ws[0]
delete(w.Modules, "two")
w.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &w); err != nil {
t.Fatal(err)
}
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
}
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
t.Fatalf("the late merge did not join the next batch: %+v", bs)
}
}
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
open := windowed(t)
asked := asksWithPaths(t)
// The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with
// the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other.
wholeKeys, held := openPlanKeys, &atomic.Bool{}
held.Store(true)
openPlanKeys = func() (planKeyReads, func(), error) {
all, done, err := wholeKeys()
return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err
}
ctx := t.Context()
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(2*time.Minute))
started, _ := walks(t, open)
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
t.Fatal(err)
}
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
}
before := len(*asked)
advanceHeld(ctx, open)
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
}
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
later := time.Now().Add(time.Hour)
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
t.Fatalf("a deferred walk reads %q", line)
}
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
if err != nil {
t.Fatal(err)
}
for _, f := range facts {
if f.id == got.ID {
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
}
}
// Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale.
done, err := open.inventory.PlanByID(ctx, started[0].ID)
if err != nil {
t.Fatal(err)
}
done.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &done); err != nil {
t.Fatal(err)
}
advanceHeld(ctx, open)
if len(*asked) != before+1 {
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
}
}
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asked := asksWithPaths(t)
ctx := t.Context()
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
_, bs := walks(t, open)
var ownBatch, catalogueBatch inventory.Plan
for _, b := range bs {
if b.OwnPath() {
ownBatch = b
} else {
catalogueBatch = b
}
}
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
len(ownBatch.Delivery.Merges) != 1 {
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
}
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
}
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
// is cut when the controller's walk ended, and waits for its word with both merges.
cutAt(t, open, t0.Add(2*time.Minute))
ws, bs := walks(t, open)
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
}
ownWalk := ws[0]
for _, m := range []string{"app", "notes"} {
if _, in := ownWalk.Modules[m]; in {
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
}
}
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
}
ownWalk.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
t.Fatal(err)
}
cutAt(t, open, t0.Add(3*time.Minute))
ws, bs = walks(t, open)
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
}
for _, m := range []string{"app", "notes"} {
if _, in := ws[0].Modules[m]; !in {
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
}
}
for _, a := range *asked {
if a[1] == "modules/app" || a[1] == "modules/notes" {
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
}
}
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
catalogueWalk := ws[0]
// The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the
// stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk.
wholeKeys, held := openPlanKeys, &atomic.Bool{}
held.Store(true)
openPlanKeys = func() (planKeyReads, func(), error) {
all, done, err := wholeKeys()
return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err
}
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
cutAt(t, open, t0.Add(6*time.Minute))
ws, _ = walks(t, open)
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
}
if _, in := ws[0].Modules["app"]; in {
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
}
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
t.Fatal(err)
}
before := len(*asked)
advanceHeld(ctx, open)
if len(*asked) != before {
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
}
// Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale.
own2, err := open.inventory.PlanByID(ctx, ws[0].ID)
if err != nil {
t.Fatal(err)
}
own2.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
t.Fatal(err)
}
advanceHeld(ctx, open)
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
}
}
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksWithPaths(t)
ctx := t.Context()
for _, m := range []struct {
module string
claims []catalogue.Claim
}{
{"nats", nil},
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
} {
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
ModuleDirsSaid: true}
hear(t, open, mixed, t0.Add(time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
if len(ws) != 1 || ws[0].Waiting() {
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
}
done := ws[0]
done.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &done); err != nil {
t.Fatal(err)
}
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
got, _ := open.inventory.PlanByID(ctx, done.ID)
_, bs := walks(t, open)
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
got.Delivery.Merges, bs)
}
}
cmd/mesh-controller/batches_test.go
-293
View File
@@ -1,293 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
func init() {
mergeWindowDefault = 0
// No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change
// (delivery_edges.go). A test that wants some replaces these (forgeHolding).
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
return catalogue.Manifest{}, false, nil
}
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
}
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
// settings file says them, and
// modules built from the catalogue (app, notes) and from three repositories of their own.
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
var t0 = time.Now().UTC().Truncate(time.Second)
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
// whose merges wait for the window as a group's says so (groupsWaiting).
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
}
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
}
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
func noGroups(t *testing.T) {
t.Helper()
was := deliveryGroupOf
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
t.Cleanup(func() { deliveryGroupOf = was })
}
// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named
// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and
// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds
// its batch open however long it takes (ADR 0418 clause 3), is groupStates'.
func groupsWaiting(t *testing.T, heads ...string) {
t.Helper()
was := deliveryGroupOf
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
if slices.Contains(heads, head) {
return nil, false, errors.New("the test's mesh-delivery does not say")
}
return nil, false, nil
}
t.Cleanup(func() { deliveryGroupOf = was })
}
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
// before novox/hq ADR 0304: mesh-delivery cannot say any head's group.
func everyHeadWaits(t *testing.T) {
t.Helper()
was := deliveryGroupOf
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
return nil, false, errors.New("the test's mesh-delivery does not say")
}
t.Cleanup(func() { deliveryGroupOf = was })
}
// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the
// members merge.
func groupStates(t *testing.T, head string, states map[string]string) {
t.Helper()
groupStatesFrom(t, head, states, true)
}
// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the
// composed check's: only the first holds the group's batch open for its last member.
func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) {
t.Helper()
was := deliveryGroupOf
deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) {
if asked != head {
return nil, false, nil
}
var members []groupMember
for repository, state := range states {
members = append(members, groupMember{Repository: repository, State: state, Composed: composed})
}
return members, true, nil
}
t.Cleanup(func() { deliveryGroupOf = was })
}
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
m.Head = head
return m
}
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
// related to a batch assembling (ADR 0304 decision 1).
func lone(m link.SourceMoved) link.SourceMoved {
m.Head = ""
return m
}
// hear is a merge heard at a moment.
// cutAt is the cutter looking at a moment.
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
t.Helper()
recent, err := open.inventory.RecentPlans(t.Context(), 50)
if err != nil {
t.Fatal(err)
}
for _, p := range recent {
if p.Batch() {
batches = append(batches, p)
} else {
walks = append(walks, p)
}
}
return walks, batches
}
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
// later commit, which names both merges — the earlier carried by the later.
// Merges of three repositories in one window are one walk, one commit for each.
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
// closes at most ten minutes after its first merge however busy.
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
// captured is what a command printed.
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
// names the walk that took it over.
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
// moves — named at once on a walk done — and joins the next batch when it does not.
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
// first; the first delivered answers the older ones, and the search stops.
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
// The window's settings reach the controller in its settings file, read at every look; one that says no
// duration, or no file, is the default.
func TestTheWindowIsTheControllersSetting(t *testing.T) {
file := t.TempDir() + "/merge-window.json"
t.Setenv(mergeWindowFileVar, file)
for _, c := range []struct {
content string
window, atMost time.Duration
}{
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
} {
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
t.Fatal(err)
}
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
}
}
t.Setenv(mergeWindowFileVar, file+".gone")
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
t.Errorf("no file reads as %s and %s", w, m)
}
}
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
f := calm(t0)
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
got := watchWaits(f)
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
t.Fatalf("the waiting walk does not name both merges: %+v", got)
}
}
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
// nothing, and the merge joins the next batch.
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
b, _ := json.Marshal(m)
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
}
got := combinedMerges([]inventory.BatchedMerge{
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
ev("bb", []string{"x/module.json"}, nil),
ev("ccc", []string{"z.go"}, nil)})
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
len(got[0].Paths) != 3 {
t.Fatalf("combined as %+v", got)
}
}
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
// still waits for the word, so no catalogue delivery skips its turn.
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
// request is named by its commit, and a title is cut at fifty runes.
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
running := func(module string) []string {
if module == "messenger" || module == "telegram" {
return []string{"novox"}
}
return nil
}
if got, want := planLineOn(p, now, tierAtLeast, running),
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
t.Fatalf("the line reads %q, want %q", got, want)
}
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
t.Fatalf("without the machines the line reads %q", got)
}
old := p
old.Commits, old.Delivery = nil, nil
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
t.Fatalf("a record naming no pull request reads %q", got)
}
long := p
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
t.Fatalf("a long title is not cut at fifty runes: %q", got)
}
}
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
func sameMerge(a, b inventory.PlanMerge) bool {
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
}
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
// its delivery would skip its turn. It joins the next catalogue batch.
cmd/mesh-controller/batches_watch.go
-153
View File
@@ -1,153 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
// The kinds S18 and S19 raise.
const (
kindBatchNotCut = "batch-not-cut"
kindBatchBehindWalk = "batch-behind-walk"
// kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after
// its first (novox/hq ADR 0418 clause 3).
kindGroupNotMerged = "group-not-merged"
)
// batchFacts is one batch not yet cut, as the watchdogs read it.
type batchFacts struct {
id, state, grouped string
// atMost is when its window closes at the latest; closed when it closed.
atMost, closed time.Time
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
// tiers.
behind string
walkBound time.Duration
// holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first
// merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting).
holds map[string]string
created time.Time
atMostWindow time.Duration
}
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
batches, err := inv.Batches(ctx)
if err != nil || len(batches) == 0 {
return nil, err
}
bounds, err := measuredTierBounds(ctx, inv, now)
if err != nil {
return nil, err
}
var out []batchFacts
for _, b := range batches {
_, atMostWindow := mergeWindowOf()
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow}
if w := b.Delivery; w != nil && w.Batch != nil {
f.holds = w.Batch.Holds
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
if f.atMost.Before(f.closed) {
f.closed = f.atMost
}
if w.Batch.CutAtOnce && !b.Created.IsZero() {
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
f.atMost, f.closed = b.Created, b.Created
}
}
if f.behind != "" {
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
}
}
out = append(out, f)
}
return out, nil
}
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, b := range f.batches {
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter ||
len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition
continue
}
late := f.now.Sub(b.atMost)
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
Severity: conditions.Warning,
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
Headline: "Merged changes are not being delivered",
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
Resolved: "Merged changes are being delivered again"})
}
return out
}
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, b := range f.batches {
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
continue
}
in := f.now.Sub(b.closed)
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
Severity: conditions.Warning,
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
b.grouped, b.behind),
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
Headline: "Merged changes wait behind a slow delivery",
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
Resolved: "Merged changes no longer wait behind a slow delivery"})
}
return out
}
// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge
// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of
// the group is delivered while it waits.
func watchGroupsNotMerged(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, b := range f.batches {
if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow {
continue
}
heads := make([]string, 0, len(b.holds))
for h := range b.holds {
heads = append(heads, h)
}
sort.Strings(heads)
var why []string
for _, h := range heads {
why = append(why, b.holds[h])
}
waited := f.now.Sub(b.created)
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id,
Kind: kindGroupNotMerged, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+
"of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id),
Said: "a group waits for a member that has not merged",
Headline: "A change waits for the rest of its group to merge",
Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+
"One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.",
humanDuration(waited)),
Resolved: "The group's last change merged"})
}
return out
}
cmd/mesh-controller/beside_test.go
-117
View File
@@ -1,117 +0,0 @@
//go:build integration
package main
import (
"strings"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
// Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in
// builder's key done at once, one step would carry a one-module walk to done.
wholeKeys, held := openPlanKeys, &atomic.Bool{}
held.Store(true)
openPlanKeys = func() (planKeyReads, func(), error) {
all, done, err := wholeKeys()
return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err
}
ctx := t.Context()
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(90*time.Second))
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
t.Fatalf("the first walk is not open: %+v", ws)
}
heard := t0.Add(100 * time.Second)
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
advanceHeld(ctx, open)
ws, bs := walks(t, open)
if len(ws) != 2 || len(bs) != 0 {
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
}
for _, w := range ws {
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
deferred(w), askedAny(w), w.Note)
}
}
}
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
ctx := t.Context()
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(90*time.Second))
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
heard := t0.Add(100 * time.Second)
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
ws, bs := walks(t, open)
if len(ws) != 2 || len(bs) != 1 {
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
}
b := bs[0]
words := batchWords(b, heard)
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
}
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
t.Fatalf("the started walk was %s by the related merge", got.State)
}
first.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &first); err != nil {
t.Fatal(err)
}
cutAt(t, open, heard.Add(2*time.Second))
if _, bs := walks(t, open); len(bs) != 0 {
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
}
}
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
// the two share (issue 973): it replaces the controller, so it goes alone.
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
ctx := t.Context()
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(90*time.Second))
ws, _ := walks(t, open)
first := ws[0]
heard := t0.Add(100 * time.Second)
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
ws, bs := walks(t, open)
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
}
first.State = inventory.PlanDone
if err := open.inventory.SavePlan(ctx, &first); err != nil {
t.Fatal(err)
}
cutAt(t, open, heard.Add(time.Second))
ws, _ = walks(t, open)
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
}
own := ws[0]
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
later := heard.Add(time.Minute)
hear(t, open, lone(repoMerge("two", "c2", later)), later)
ws, bs = walks(t, open)
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
}
}
cmd/mesh-controller/build.go
-4
View File
@@ -229,10 +229,6 @@ type answers struct {
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
// recorded at send, not at apply).
reported []inventory.Reported
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
plans []inventory.Plan
// tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296).
tierBounds tierBounds
// refused is why a machine cannot be worked out at all, by name. A different thing from every
// other answer here: those are about a machine that was told something, and this is about one
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
cmd/mesh-controller/bus_step.go
+1 -1
View File
@@ -229,7 +229,7 @@ func busCommand(ctx context.Context, args []string) error {
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
sent, err := sendToEach(withBusStep(ctx), open, moving)
if err != nil {
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
cmd/mesh-controller/bus_waits.go
-195
View File
@@ -1,195 +0,0 @@
package main
import (
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// A send held for the bus's planned step (novox/hq issue 336).
//
// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned
// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every
// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only
// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move.
// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what
// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it
// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the
// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks'
// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB).
// kindBusStepWaiting is S17's kind: bus.<module>.step-waiting.
const kindBusStepWaiting = "bus-step-waiting"
// busFacts is a new bus build waiting for its planned step, and the sends held for it.
type busFacts struct {
module, to string
// from is the build each machine of the bus runs; machines those whose bus the step would replace.
from map[string]string
machines []string
waits []busWaitFacts
}
// busWaitFacts is one walk whose send was refused because it would replace the bus.
type busWaitFacts struct {
plan, repository, commit string
// modules are what its tier sends: what waits.
modules []string
// since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal.
since time.Time
}
// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note
// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save.
var busRefusedFirst = &firstSeen{at: map[string]time.Time{}}
type firstSeen struct {
mu sync.Mutex
at map[string]time.Time
}
// mark keeps the first moment an id was seen.
func (s *firstSeen) mark(id string, at time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
if _, seen := s.at[id]; !seen {
s.at[id] = at
}
}
// of is when an id was first seen; zero when it was not.
func (s *firstSeen) of(id string) time.Time {
s.mu.Lock()
defer s.mu.Unlock()
return s.at[id]
}
// keepOnly forgets every id not given: a walk no longer held is not held since then.
func (s *firstSeen) keepOnly(ids map[string]bool) {
s.mu.Lock()
defer s.mu.Unlock()
for id := range s.at {
if !ids[id] {
delete(s.at, id)
}
}
}
// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step.
func refusedForTheBus(err error) bool {
return err != nil && strings.Contains(err.Error(), errBusWaits.Error())
}
// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its
// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace
// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not.
func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts {
f := busFacts{module: b.module, to: b.to, from: b.from}
for _, n := range b.machines {
if b.moves(n) {
f.machines = append(f.machines, n)
}
}
if len(f.machines) == 0 {
return f
}
for _, p := range plans {
if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) {
continue
}
since := p.Updated
if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) {
since = seen
}
var modules []string
if p.Tier >= 0 && p.Tier < len(p.Tiers) {
modules = append(modules, p.Tiers[p.Tier]...)
} else {
modules = planModules(p)
}
sort.Strings(modules)
f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules,
since: since})
}
sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) })
return f
}
// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out.
func (b busFacts) heldByTheBus() map[string]bool {
out := map[string]bool{}
for _, w := range b.waits {
out[w.plan] = true
}
return out
}
// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with
// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true").
const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`"
// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator.
func watchBusWaits(f *signalFacts) []conditions.Observation {
b := f.bus
if len(b.waits) == 0 || len(b.machines) == 0 {
return nil
}
since := b.waits[0].since
var walks, what []string
for _, w := range b.waits {
walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit),
strings.Join(w.modules, ", ")))
what = append(what, deliveryWhat(w.modules, w.repository))
}
var from []string
for _, n := range b.machines {
from = append(from, short(orNotKnown(b.from[n])))
}
machines := namesWords(b.machines, 3)
in := f.now.Sub(since)
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting",
Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Machine: b.machines[0], Also: b.machines[1:],
Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+
"the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "),
b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb,
since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")),
Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)),
Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax),
Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+
"installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines,
namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax),
Needs: "start the bus upgrade " + FromMeshMCPServer,
Resolved: "Resolved: the bus upgrade started, and sends go on"}}
}
// sortedUnique is a list sorted, each once.
func sortedUnique(xs []string) []string {
seen := map[string]bool{}
var out []string
for _, x := range xs {
if !seen[x] {
seen[x] = true
out = append(out, x)
}
}
sort.Strings(out)
return out
}
// clipWords keeps a plain sentence within a bound, at a word.
func clipWords(s string, n int) string {
if len(s) <= n {
return s
}
cut := strings.LastIndex(s[:n-1], " ")
if cut <= 0 {
cut = n - 1
}
return s[:cut] + "…"
}
cmd/mesh-controller/bus_waits_integration_test.go
-70
View File
@@ -1,70 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
b := aBusOnAnchor()
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
Updated: updated}
}
for _, c := range []struct {
name string
bus busPending
plans []inventory.Plan
first map[string]time.Time
want []string
since time.Time
}{
{"a walk refused for the bus", b,
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
{"refused earlier than its last save, as this controller saw it", b,
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
{"a walk refused for another reason", b,
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
nil, time.Time{}},
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
{"the bus already runs the build held: its step started", func() busPending {
o := aBusOnAnchor()
o.from = map[string]string{"anchor": o.to}
return o
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
return []inventory.Plan{p}
}(), nil, nil, time.Time{}},
} {
t.Run(c.name, func(t *testing.T) {
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
var ids []string
for _, w := range got.waits {
ids = append(ids, w.plan)
}
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
}
if len(c.want) > 0 {
if !got.waits[0].since.Equal(c.since) {
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
}
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
t.Errorf("behind %+v", got)
}
}
})
}
}
cmd/mesh-controller/bus_waits_test.go
-99
View File
@@ -1,99 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
)
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that
// ends it — and the wait is not read as a walk running late.
// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held.
func aBusOnAnchor() busPending {
return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"},
to: "32307bd1bbbb", same: map[string]bool{}}
}
// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it.
func refusedNote(b busPending, tier int) string {
held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) +
"), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)"
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
}
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
// condition clears once the bus's machine runs the new build.
func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) {
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
b := aBusOnAnchor()
held := func(entered time.Duration) *signalFacts {
f := calm(now)
f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1,
entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}}
f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"},
waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122",
modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}}
return f
}
f := held(time.Minute)
got := watchBusWaits(f)
if len(got) != 1 {
t.Fatalf("a send held for the bus a minute ago raised %+v", got)
}
o := got[0]
if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator {
t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver)
}
for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus",
`"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} {
if !strings.Contains(o.Summary, want) {
t.Errorf("its summary does not say %q: %s", want, o.Summary)
}
}
if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" {
t.Errorf("its words do not say what waits and what the operator does: %+v", o)
}
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok {
t.Errorf("its words are not plain: %s", why)
}
// Past S3's bound: still the bus's wait, never a stalled walk.
late := held(45 * time.Minute)
if s3 := watchPlans(late); len(s3) != 0 {
t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3)
}
// A walk held for something else past its bound is still stalled.
other := held(45 * time.Minute)
other.plans[0].bus = false
if s3 := watchPlans(other); len(s3) != 1 {
t.Fatalf("a walk held for something else past its bound raised %+v", s3)
}
// Through the keeper: open at the first tick, cleared when the step started.
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{},
Now: func() time.Time { return now }})
defer k.Close(context.Background())
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
w.see(t.Context(), held(time.Minute))
open, err := k.Open(t.Context())
if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" {
t.Fatalf("after the first tick, open: %+v (%v)", open, err)
}
started := held(time.Minute)
started.bus = busFacts{module: "nats", to: b.to}
started.plans[0].bus = false
w.see(t.Context(), started)
if open, _ := k.Open(t.Context()); len(open) != 0 {
t.Fatalf("the step started, and open: %+v", open)
}
}
cmd/mesh-controller/busy.go
-103
View File
@@ -1,103 +0,0 @@
package main
import (
"fmt"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
//
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
// its gate starts as before.
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
var busyGateSlack = 5 * time.Minute
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
// controller's clock from when it first read the wait.
const busyEngineBound = 15 * time.Minute
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
// machine nor a wrong word from it holds a walk longer.
func busyGateEnds(b *inventory.GateBusy) time.Time {
until := b.Seen.Add(busyEngineBound)
if !b.Until.IsZero() && b.Until.Before(until) {
until = b.Until
}
return until.Add(busyGateSlack)
}
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
// gate's Since to its end. Pure but for the gate it is handed.
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
if g.Busy != nil && g.Busy.Ended != nil {
return false
}
for _, j := range pairs {
b, waits := busyOn(j, f)
if !waits {
continue
}
if g.Busy == nil {
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
Seen: now}
}
if g.Busy.Seen.IsZero() {
g.Busy.Seen = now
}
g.Busy.Until, g.Busy.Said = b.Until, b.Said
if now.After(busyGateEnds(g.Busy)) {
// The engine said it would have replaced it by now: the judging starts, and finds the module not
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
end := now
g.Busy.Ended, g.Busy.PastBound = &end, true
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
return false
}
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
g.Read(now, false, g.Busy.Words())
return true
}
if g.Busy == nil {
return false
}
// Replaced: the judging starts at the report that said so.
end := now
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
end = r.At.UTC()
}
g.Busy.Ended = &end
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
return false
}
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
// running because it said busy.
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
r, said := f.reports[j.node]
if !said || r.At == nil || !f.reportedOn(j.node, r) {
return inventory.BusyWait{}, false
}
return inventory.WaitingFor(r.Busy, j.module)
}
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
func busyStill(j judged, f gateFacts) (string, bool) {
b, waits := busyOn(j, f)
if !waits {
return "", false
}
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
}
cmd/mesh-controller/busy_test.go
-148
View File
@@ -1,148 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
reported := sent.Add(20 * time.Second)
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
// As judgeMoves hands it: the report held against the gate's own send.
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
}
now := reported.Add(time.Minute)
if !busyWait(g, pairs, facts(reported, waiting), now) {
t.Fatal("a gate whose module's container waits as busy judged it")
}
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
}
for _, said := range []string{gateLine(g), g.Last} {
if !strings.Contains(said, "waits for a busy container on novox") {
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
}
}
if !strings.Contains(g.Last, "check of #280") {
t.Errorf("the wait does not say what the container holds: %s", g.Last)
}
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
now = reported.Add(10 * time.Minute)
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
}
// The machine reports it replaced: the judging starts at that report.
replacedAt := reported.Add(12 * time.Minute)
now = replacedAt.Add(5 * time.Second)
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
t.Fatal("the gate still waits after the container was replaced")
}
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
}
if strings.Contains(gateLine(g), "waits for a busy container") {
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
}
// And a later report listing it waiting again does not hold a gate that started.
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
t.Error("a gate that started waits again")
}
}
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
pairs := []judged{{module: "build-agent", node: "novox"}}
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
At: &sent, Current: true, Busy: waiting}}}
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
if busyWait(g, pairs, f, now) {
t.Fatal("a gate waited past the engine's bound and the slack")
}
if !g.Busy.PastBound || !g.Since.Equal(now) {
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
}
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
}
}
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
// report not about the gate's send says nothing of it.
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
pairs := []judged{{module: "build-agent", node: "ace"}}
at := sent.Add(time.Minute)
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
At: &at, Declared: "d2", ReportedSequence: 9}}}
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
}
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
if busyWait(g, pairs, elsewhere, at) {
t.Fatal("a wait reported about an earlier declaration held the gate")
}
}
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
pairs := []judged{{module: "build-agent", node: "novox"}}
ahead := seen.Add(time.Hour)
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
}
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
t.Fatal("the gate stopped waiting within the bound on its own clock")
}
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
}
}
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
{"id": "agent-env", "type": "file"}}}}
lines := busyNotSent(modules, false)
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
!strings.Contains(lines[0], "replaced at once") {
t.Fatalf("the plan does not say the busy not sent: %v", lines)
}
if lines := busyNotSent(modules, true); len(lines) != 0 {
t.Fatalf("a busy sent is said as not sent: %v", lines)
}
}
cmd/mesh-controller/calls_verb_test.go
-24
View File
@@ -24,28 +24,4 @@ func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
}
}
// A push — named or through command — answers before it runs: it sends the machine holding the bus
// first, and the broker reloading its user list forgets the answer it was about to permit.
func TestAPushAnswersBeforeItSends(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want bool
}{
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
{"push", map[string]any{"why": "w"}, true},
{"command", map[string]any{"command": "builds"}, false},
{"status", map[string]any{}, false},
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
} {
argv, err := argvFor(c.verb, c.args)
if err != nil {
t.Fatalf("%s %v: %v", c.verb, c.args, err)
}
if got := answersFirst(argv); got != c.want {
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
}
}
}
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
cmd/mesh-controller/catch_up.go
-643
View File
@@ -1,643 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"net/url"
"path"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **The controller catches up on merges it did not hear** (novox/hq ADR 0457, issue 1336). A merge the forge announced while the
// controller was being replaced was never heard, and nothing asked for it again: the trunk held a merge no walk answered. At its
// start, and each time it hears a merge, the controller reads the last catchUpWindow merge commits of a trunk from the forge, and
// hears each one it holds no record of as if the forge had announced it then, on the warrant the router holds for it.
// It lasts until release 6 of design 47 deletes delivery plans and batches (ADR 0429 §7, §8).
// catchUpWindow is how many of a trunk's newest merge commits are read; an older merge is a person's to decide.
const catchUpWindow = 50
// kindMergeNotHeard is the condition of a merge the controller did not hear and holds no warrant to plan.
const kindMergeNotHeard = "merge-not-heard"
// trunkMerge is a merge commit of a trunk as the forge lists it.
type trunkMerge struct {
Commit string
// Head is the merge commit's second parent: the pull request's head that was merged.
Head string
Subject string
At time.Time
}
// trunkCommit is a commit of a trunk as the forge lists it, newest first: a merge commit has a second parent.
type trunkCommit struct {
Commit, SecondParent, Subject string
At time.Time
}
// trunkCommitsPage is one page (fifty) of a trunk's commits, newest first; empty past the last. A variable so a test needs no
// forge.
var trunkCommitsPage = func(ctx context.Context, owner, repo, branch string, page int) ([]trunkCommit, error) {
var found struct {
Result []struct {
SHA string `json:"sha"`
Parents []struct {
SHA string `json:"sha"`
} `json:"parents"`
Commit struct {
Message string `json:"message"`
Committer struct {
Date string `json:"date"`
} `json:"committer"`
} `json:"commit"`
} `json:"result"`
}
apiPath := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&limit=50&page=%d&stat=false&verification=false&files=false",
owner, repo, url.QueryEscape(branch), page)
missing, err := askForgeApi(ctx, apiPath, &found)
if err != nil {
return nil, err
}
if missing {
return nil, fmt.Errorf("the forge holds no branch %s of %s/%s", branch, owner, repo)
}
var out []trunkCommit
for _, c := range found.Result {
at, _ := time.Parse(time.RFC3339, c.Commit.Committer.Date)
tc := trunkCommit{Commit: c.SHA, Subject: firstLine(c.Commit.Message), At: at.UTC()}
if len(c.Parents) >= 2 {
tc.SecondParent = c.Parents[1].SHA
}
out = append(out, tc)
}
return out, nil
}
// readTrunkMerges is the last catchUpWindow merge commits of a repository's trunk, oldest first: pages are read until that many
// merges are held or the trunk ends.
func readTrunkMerges(ctx context.Context, owner, repo, branch string) ([]trunkMerge, error) {
var merges []trunkMerge
for page := 1; page <= 8 && len(merges) < catchUpWindow; page++ {
commits, err := trunkCommitsPage(ctx, owner, repo, branch, page)
if err != nil {
return nil, err
}
if len(commits) == 0 {
break
}
for _, c := range commits {
// A merge of the trunk is a pull request's merge into it. A commit with a second parent that is none is a merge made
// inside some branch ("Merge remote-tracking branch 'origin/main' into fix/..."), which the trunk's history lists too and
// which no Approve ever covered: read as a gap it would raise a condition for nothing a person decides.
if c.SecondParent != "" && mergesIntoTrunk(c.Subject, branch) {
merges = append(merges, trunkMerge{Commit: c.Commit, Head: c.SecondParent, Subject: c.Subject, At: c.At})
}
}
}
return lastMerges(merges), nil
}
// sinceTheRecord are the merges that can be gaps (novox/hq ADR 0463 §2): those made since the controller's record of merges begins on
// the mesh (first, the earliest merge it holds of any repository and branch, less mergeGrace as owedLate judges). A merge before it is
// history on every trunk, and a trunk with no merge kept of its own still has the merges made since as gaps. With no merge kept
// anywhere there is no record to have a gap in.
func sinceTheRecord(merges []trunkMerge, kept bool, first time.Time) []trunkMerge {
if !kept {
return nil
}
var out []trunkMerge
for _, m := range merges {
if !m.At.Before(first.Add(-mergeGrace)) {
out = append(out, m)
}
}
return out
}
// lastMerges is the newest catchUpWindow of merges, given newest first, oldest first.
func lastMerges(newestFirst []trunkMerge) []trunkMerge {
if len(newestFirst) > catchUpWindow {
newestFirst = newestFirst[:catchUpWindow]
}
out := slices.Clone(newestFirst)
slices.Reverse(out)
return out
}
var mergeSubject = regexp.MustCompile(`^Merge pull request '(.*)' \(#(\d+)\) from (\S+) into (\S+)$`)
// pullOfSubject is the pull request a merge commit's subject names: its title and number.
func pullOfSubject(subject string) (title string, number int, ok bool) {
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
if m == nil {
return "", 0, false
}
n, err := strconv.Atoi(m[2])
if err != nil || n <= 0 {
return "", 0, false
}
return m[1], n, true
}
// mergesIntoTrunk says a commit's subject is a pull request's merge into this trunk.
func mergesIntoTrunk(subject, branch string) bool {
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
return m != nil && m[4] == branch
}
// headBranchOfSubject is the branch a merge commit's subject says was merged.
func headBranchOfSubject(subject string) string {
if m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)); m != nil {
return m[3]
}
return ""
}
// readMergeEvent is the merge as the forge would have announced it: the commit, the pull request, the files it changed and the
// directories among them that hold a module.json. A variable so a test needs no forge.
var readMergeEvent = func(ctx context.Context, owner, repo, branch string, tm trunkMerge) (link.SourceMoved, error) {
m := link.SourceMoved{Owner: owner, Repo: repo, Base: branch, Head: headBranchOfSubject(tm.Subject), HeadSHA: tm.Head,
Commit: tm.Commit, MergedAt: tm.At.UTC().Format(time.RFC3339)}
if title, number, ok := pullOfSubject(tm.Subject); ok {
m.Number, m.Title = number, title
}
var found struct {
Result struct {
Files []struct {
Filename string `json:"filename"`
Status string `json:"status"`
} `json:"files"`
} `json:"result"`
}
missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(tm.Commit), &found)
if err != nil {
return m, err
}
if missing || len(found.Result.Files) == 0 {
return m, fmt.Errorf("the forge named no files for the merge %.8s of %s/%s", tm.Commit, owner, repo)
}
for _, f := range found.Result.Files {
m.Paths = append(m.Paths, f.Filename)
if f.Status == "removed" || f.Status == "deleted" {
m.Removed = append(m.Removed, f.Filename)
}
}
// The forge lists a merge's files by the page; a list as long as a page may be a beginning.
m.PathsTruncated = len(m.Paths) >= 300
dirs := map[string]bool{}
for _, p := range m.Paths {
for d := path.Dir(p); ; d = path.Dir(d) {
dirs[d] = true
if d == "." || d == "/" {
break
}
}
}
if len(dirs) > 64 {
return m, nil // too many to look at: the old rule stands, a directory is a module when its manifest changed
}
var holding []string
for d := range dirs {
file := "module.json"
if d != "." {
file = d + "/module.json"
}
_, exists, err := manifestCached(ctx, owner, repo, file, tm.Commit)
if err != nil {
return m, err
}
if exists {
holding = append(holding, d)
}
}
sort.Strings(holding)
m.ModuleDirs, m.ModuleDirsSaid = holding, true
return m, nil
}
// mergeAskPrefix is what the id of a merge ask made by the retired module mesh-delivery begins with: the pull request's number
// and the head's first twelve digits (`merge-<number>-<head>-<try>`). The teller's ids name neither (see findMergeWarrant).
func mergeAskPrefix(number int, head string) string {
if len(head) > 12 {
head = head[:12]
}
return fmt.Sprintf("merge-%d-%s-", number, head)
}
// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking
// mesh-delivery's (ADR 0450 §8; issue 1318).
func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} }
// caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant
// answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437
// §6, as movedOnApprove judges an announced one). The heads the Approve was bound to are checked by the caller, where it can
// name them.
func caughtUpWarrant(asker string, ask *asks.Ask, w *asks.Warrant) (string, error) {
if err := askerAccepted(asker); err != nil {
return "", err
}
switch {
case ask == nil || w == nil:
return "", errors.New("the router holds no closed record of the ask")
case ask.Who != asks.Operator:
return "", fmt.Errorf("the ask %s is for %q to answer, not the operator", ask.ID, ask.Who)
}
option, err := w.For(asker, *ask)
if err != nil {
return "", err
}
if option.ID != optionApprove || w.Option != optionApprove {
return "", fmt.Errorf("the operator chose %q on the ask %s, not an Approve", w.Option, w.Ask)
}
if w.By == nil || w.By.Who != asks.Operator || w.By.Verified == "" {
return "", fmt.Errorf("the ask %s was not answered by the operator on a channel that proves who answered", w.Ask)
}
return w.Says() + " (ask " + w.Ask + ")", nil
}
// routerAskIDs are the ids of the asks an asker has in the router's record. A variable so a test needs no bus.
var routerAskIDs = func(ctx context.Context, inv catalogueReader, asker string) ([]string, error) {
serving := servingBus.Load()
if serving == nil {
return nil, errors.New("only the serving controller reads the router's record")
}
bucket, err := asksRecords(ctx, inv)
if err != nil {
return nil, err
}
if bucket == "" {
return nil, errors.New("no module declares the operator channel's records")
}
api, err := jetstream.New(serving.Conn())
if err != nil {
return nil, err
}
kv, err := api.KeyValue(ctx, bucket)
if err != nil {
return nil, err
}
return askIDsOf(ctx, kv, asker)
}
// askIDsOf are the ids of the asks an asker has in the router's bucket.
func askIDsOf(ctx context.Context, kv jetstream.KeyValue, asker string) ([]string, error) {
lister, err := kv.ListKeysFiltered(ctx, asker+".>")
if err != nil {
return nil, err
}
var ids []string
for key := range lister.Keys() {
ids = append(ids, strings.TrimPrefix(key, asker+"."))
}
// The lister stops its own watch when it has sent the last key, so a stop of ours finds it stopped: nats answers that with
// "invalid subscription", which is no failure of the read (issue 1336). Any other error stays one.
if err := lister.Stop(); err != nil && !errors.Is(err, nats.ErrBadSubscription) {
return nil, err
}
return ids, nil
}
// closedRecord is a router record of an ask that ended: it never changes, so it is read once.
type closedRecord struct {
ask *asks.Ask
w *asks.Warrant
}
var closedRecords sync.Map // asker + "." + id -> closedRecord
// closedRecordOf is the router's record of an ask, kept once it is closed. A nil ask is an ask still open, ended without a
// choice, or not there.
func closedRecordOf(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
if v, ok := closedRecords.Load(asker + "." + id); ok {
r := v.(closedRecord)
return r.ask, r.w, nil
}
ask, w, err := routerAskOf(ctx, inv, asker, id)
if err != nil {
return nil, nil, err
}
if ask != nil && w != nil {
closedRecords.Store(asker+"."+id, closedRecord{ask, w})
}
return ask, w, nil
}
// deliveryIDOf is the id the gate gives a delivery, and so the key of its head in the heads an Approve binds: the repository as
// owner/name, lower-cased, `@` and the first twelve characters of the head.
func deliveryIDOf(owner, repo, head string) string {
if len(head) > 12 {
head = head[:12]
}
return strings.ToLower(owner+"/"+repo) + "@" + head
}
// coversHead says an ask the operator approved is for this merge: its Approve is bound to the heads of the pull request alone
// (mergeAct over {delivery id: head}, which is how a pull request not in a group is asked), or, for a group's ask, whose other
// members' heads the router's record does not carry in full, the ask as published names this pull request at this head in its
// details ("<repo>#<number> ... at <head>:"), and the digest the warrant answers covers those words.
func coversHead(ask *asks.Ask, owner, repo string, number int, head string) (how string, ok bool) {
option, found := ask.Option(optionApprove)
if !found {
return "", false
}
heads := map[string]string{deliveryIDOf(owner, repo, head): head}
if option.Performs(mergeAct(ask.ID, optionApprove, heads)) == nil {
return "bound to the head of this pull request", true
}
short := head
if len(short) > 12 {
short = short[:12]
}
line := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(repo) + `#` + strconv.Itoa(number) + ` .* at ` + regexp.QuoteMeta(short) + `:`)
if line.MatchString(ask.Details) {
return "a group's ask naming this pull request at this head", true
}
return "", false
}
// findMergeWarrant finds the warrant of a merge, with no read the controller does not make today: the router's chosen merge
// ask for it. Under the teller (releases 3e on) the id is `merge-<group>-<heads hash>-<tail>`, which names neither the pull
// request nor the head, so the teller's closed asks are read and the one the operator approved whose heads cover the merge's
// second parent is the warrant (coversHead). Under mesh-delivery, which made the earlier merges, the id names the pull request
// and the head and is found by them. Why it holds; or found false with why not, for a person to read; or an error when the
// router could not be read, which is no answer. A variable so a test needs no bus.
var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, repo string, number int, head string) (why string, found bool, reason string, err error) {
reason = "the router holds no chosen merge ask of " + strings.Join(askersOfMerges(), " or ") + " that covers the head " + head[:min(12, len(head))]
for _, asker := range askersOfMerges() {
ids, err := routerAskIDs(ctx, inv, asker)
if err != nil {
return "", false, "", err
}
for _, id := range ids {
named := number > 0 && strings.HasPrefix(id, mergeAskPrefix(number, head))
if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") {
continue
}
ask, w, err := closedRecordOf(ctx, inv, asker, id)
if err != nil {
return "", false, "", err
}
if ask == nil || w == nil {
continue
}
warrant, err := caughtUpWarrant(asker, ask, w)
if err != nil {
if named {
reason = fmt.Sprintf("%s's ask %s holds no operator's Approve: %v", asker, id, err)
}
continue
}
how, covers := coversHead(ask, owner, repo, number, head)
if !covers && !named {
continue
}
if !covers {
how = "the ask's id names this pull request and head"
}
return warrant + ", " + how, true, "", nil
}
}
return "", false, reason, nil
}
// caughtTrunk is one trunk the catalogue builds from.
type caughtTrunk struct{ Owner, Repo, Branch string }
func (t caughtTrunk) repository() string { return t.Owner + "/" + t.Repo }
// trunksOfCatalogue are the trunks of the repositories on the mesh's forge the catalogue builds a module from.
func trunksOfCatalogue(forms *sourceForms, entries []inventory.Entry) []caughtTrunk {
var out []caughtTrunk
for _, e := range entries {
if e.Provided || e.Source.Repository == "" {
continue
}
owner, name, onForge := forms.onTheForge(e.Source.Repository, e.Source.Seat)
if !onForge {
continue
}
branch := followedBranch(e.Source.Ref)
if branch == "" {
branch = "main"
}
t := caughtTrunk{owner, name, branch}
if !slices.Contains(out, t) {
out = append(out, t)
}
}
return out
}
// catchUpBeside runs a catch-up beside the hearing of the merge that asked for it, so the bus's acknowledgement of that merge
// waits for no forge. A variable so a test runs it where it stands.
var catchUpBeside = func(run func()) { go run() }
// catchingUp makes one catch-up at a time.
var catchingUp sync.Mutex
// settledMerges are the merges this process found nothing to plan for: heard, and no record kept, because nothing reads them.
// They are not read from the forge again.
var settledMerges sync.Map
// catchUp hears the merges of the catalogue's trunks (only one, when only names it) that the controller holds no record of.
// It says whether a trunk was left for later: the forge, the router or the store did not answer.
func (f following) catchUp(ctx context.Context, only *caughtTrunk) (later bool) {
catchingUp.Lock()
defer catchingUp.Unlock()
inv := f.open.inventory
entries, err := inv.Catalogued(ctx)
if err != nil {
fmt.Printf("catch-up: the catalogue cannot be read, so no merge is caught up now: %v\n", err)
return true
}
kept, first, err := inv.EarliestKept(ctx)
if err != nil {
fmt.Printf("catch-up: what the controller kept cannot be read, so no merge is caught up now: %v\n", err)
return true
}
if kept {
fmt.Printf("catch-up: the controller's record of merges begins %s; a merge made before it, less %s, is history on every trunk\n",
first.UTC().Format(time.RFC3339), mergeGrace)
} else {
fmt.Println("catch-up: the controller holds no merge of any trunk yet, so none can have been missed")
}
trunks := trunksOfCatalogue(newSourceForms(ctx, inv), entries)
if only != nil {
trunks = slices.DeleteFunc(trunks, func(t caughtTrunk) bool {
return !strings.EqualFold(t.repository(), only.repository()) || t.Branch != only.Branch
})
}
for _, t := range trunks {
if f.catchUpTrunk(ctx, t, kept, first) {
later = true
}
}
return later
}
func (f following) catchUpTrunk(ctx context.Context, t caughtTrunk, kept bool, first time.Time) (later bool) {
inv := f.open.inventory
repository := t.repository()
merges, err := readTrunkMerges(ctx, t.Owner, t.Repo, t.Branch)
if err != nil {
// Not nothing to catch up: left for the next start or the next heard merge (ADR 0457 clause 3).
fmt.Printf("catch-up: the forge did not answer for the last merges of %s %s, so they are not caught up now; the next start "+
"or the next merge heard there does it: %v\n", repository, t.Branch, err)
return true
}
// **A merge made before the controller's record of merges begins is history, never a gap** (novox/hq ADR 0463 §2).
read := len(merges)
merges = sinceTheRecord(merges, kept, first)
fmt.Printf("catch-up: %s %s: %d of the last %d pull request merges were made since the record began\n", repository, t.Branch,
len(merges), read)
unwarranted := map[string]bool{}
for _, tm := range merges {
id := fmt.Sprintf("%s_%.8s", repository, tm.Commit)
if _, done := settledMerges.Load(repository + "@" + tm.Commit); done {
continue
}
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err != nil {
fmt.Printf("catch-up: the record of %s cannot be read, so %s is not caught up now: %v\n", repository, tm.Commit[:8], err)
return true
} else if known {
continue
}
m, err := readMergeEvent(ctx, t.Owner, t.Repo, t.Branch, tm)
if err != nil {
fmt.Printf("catch-up: the merge %.8s of %s cannot be read from the forge, so it is not caught up now: %v\n", tm.Commit, repository, err)
later = true
continue
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return true
}
read, err := readForPlanning(ctx, inv)
if err != nil {
return true
}
facts, err := factsOfMerge(ctx, inv, m, time.Now().UTC(), entries, read)
if err != nil {
fmt.Printf("catch-up: %s %.8s cannot be judged now: %v\n", repository, tm.Commit, err)
later = true
continue
}
// The very test hearMerge makes: a merge it would plan, or take as a first build, is a gap; any other it would say
// nothing reads, and is no gap for a person to decide.
if !facts.plans() && !facts.addsModuleToBuilds {
settledMerges.Store(repository+"@"+tm.Commit, true)
continue
}
why, found, reason, err := findMergeWarrant(ctx, inv, t.Owner, t.Repo, m.Number, tm.Head)
if err != nil {
fmt.Printf("catch-up: the router cannot be read, so %s %.8s is not caught up now: %v\n", repository, tm.Commit, err)
later = true
continue
}
if !found {
unwarranted[id] = true
f.raiseMergeNotHeard(ctx, repository, t.Branch, tm, m, reason)
continue
}
now := time.Now().UTC()
said := fmt.Sprintf("caught up: merged %s, not heard, planned %s", tm.At.Format(time.RFC3339), now.Format(time.RFC3339))
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard; %s; on the warrant %s\n", repository, t.Branch, tm.Commit, said, why)
m.Title = strings.TrimSpace(m.Title + " (" + said + ")")
if err := f.hearMerge(ctx, m, now); err != nil {
fmt.Printf("catch-up: %s %.8s could not be heard now: %v\n", repository, tm.Commit, err)
later = true
continue
}
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err == nil && !known {
settledMerges.Store(repository+"@"+tm.Commit, true) // heard, and nothing reads it
}
}
// Cleared only when every gap was judged: one left for later may still be open.
if !later {
f.clearMergesHeard(ctx, repository, unwarranted)
}
return later
}
// raiseMergeNotHeard keeps open, and observes again, the condition of a merge with no warrant to plan it. One for each
// repository and commit, however many catch-ups find it.
func (f following) raiseMergeNotHeard(ctx context.Context, repository, branch string, tm trunkMerge, m link.SourceMoved, reason string) {
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard, and is not planned: %s\n", repository, branch, tm.Commit, reason)
if conditionsFrom == nil {
return
}
o := conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, tm.Commit), Kind: kindMergeNotHeard,
Severity: conditions.Warning, Source: "catch-up", Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("the merge %.8s of %s into %s was not heard by the controller, and no operator's Approve of it is on "+
"record to plan it on: a person decides", tm.Commit, repository, branch),
Said: fmt.Sprintf("%s %.8s (pull request %d, head %.8s, merged %s): %s", repository, tm.Commit, m.Number, tm.Head,
tm.At.Format(time.RFC3339), reason),
Headline: fmt.Sprintf("A merge into %s was never delivered", repository),
Explanation: fmt.Sprintf("A merge into %s was made while the controller did not hear it, and no Approve of yours for it is on "+
"record that the controller can plan it on. It stays open while the merge is among the latest %d merges of that repository. "+
"The details say what was looked for.", repository, catchUpWindow),
Needs: "decide whether that merge should be delivered, and say so if it should.",
Resolved: "the merge is planned, or is no longer among the latest merges of its repository"}
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
fmt.Printf("catch-up: the condition of %s %.8s could not be kept: %v\n", repository, tm.Commit, err)
}
}
// clearMergesHeard clears the open conditions of a repository's merges not heard that this catch-up did not find again: the
// gap was planned, or left the window.
func (f following) clearMergesHeard(ctx context.Context, repository string, still map[string]bool) {
if conditionsFrom == nil {
return
}
open, err := conditionsFrom.Open(ctx)
if err != nil {
fmt.Printf("catch-up: the open conditions cannot be read, so none is cleared now: %v\n", err)
return
}
for _, c := range open {
if c.Kind != kindMergeNotHeard || c.Subject.Scope != conditions.ScopeMerge ||
!strings.HasPrefix(c.Subject.ID, repository+"_") || still[c.Subject.ID] {
continue
}
if _, err := conditionsFrom.Clear(ctx, c.Key, "the merge is planned, or left the last merges of its trunk"); err != nil {
fmt.Printf("catch-up: the condition %s could not be cleared: %v\n", c.Key, err)
}
}
}
// catchUpAtStart catches up every trunk once the controller serves, and again every minute while a trunk was left for later.
func catchUpAtStart(ctx context.Context, open *stores) {
select {
case <-ctx.Done():
return
case <-time.After(30 * time.Second):
}
for try := 0; try < 20; try++ {
if !(following{open: open}).catchUp(ctx, nil) {
return
}
select {
case <-ctx.Done():
return
case <-time.After(time.Minute):
}
}
fmt.Println("catch-up: a trunk is still left for later after twenty tries; the next merge heard there does it")
}
cmd/mesh-controller/catch_up_integration_test.go
-352
View File
@@ -1,352 +0,0 @@
//go:build integration
package main
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear.
// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the
// router holds an operator's Approve for.
type aTrunk struct {
merges []trunkMerge
module map[string]string
approved map[string]bool
read []string // the trunks asked of the forge, as owner/repo
events int // merges read in full from the forge
silent bool // the forge does not answer
}
func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk {
t.Helper()
tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}}
for i, m := range modules {
c := fmt.Sprintf("%s-c%02d", repo, i+1)
tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1),
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c),
At: t0.Add(time.Duration(i) * time.Minute)})
tr.module[c] = m
tr.approved[c] = true
}
was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant
trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) {
tr.read = append(tr.read, owner+"/"+name)
if tr.silent {
return nil, errors.New("the forge did not answer")
}
if page > 1 || owner+"/"+name != "novox/"+repo {
return nil, nil
}
var newest []trunkCommit
for i := len(tr.merges) - 1; i >= 0; i-- {
m := tr.merges[i]
newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At})
}
return newest, nil
}
readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
tr.events++
m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At)
m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head
_, m.Number, _ = pullOfSubject(tm.Subject)
return m, nil
}
findMergeWarrant = func(_ context.Context, _ catalogueReader, _, _ string, number int, head string) (string, bool, string, error) {
for _, m := range tr.merges {
if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] {
return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil
}
}
return "", false, "the router holds no chosen merge ask for the pull request and head", nil
}
wasBeside := catchUpBeside
catchUpBeside = func(run func()) { run() }
t.Cleanup(func() {
trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside
settledMerges.Clear()
})
return tr
}
// aKeptMergeBefore is a merge the controller heard before the merges of a test: where its record begins, so what a trunk holds from
// then on can be a gap (novox/hq ADR 0463 §2).
func aKeptMergeBefore(t *testing.T, open *stores) {
t.Helper()
hear(t, open, catalogueMerge("mesh-catalog-seed", "app", t0.Add(-time.Hour)), t0.Add(-time.Hour))
}
func mergeKnown(t *testing.T, open *stores, commit string) bool {
t.Helper()
_, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit)
if err != nil {
t.Fatal(err)
}
return known
}
// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said
// as caught up; a second start plans nothing new.
func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second))
hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second))
if mergeKnown(t, open, "mesh-catalog-c02") {
t.Fatal("the merge to be caught up is known before the catch-up")
}
if (following{open: open}).catchUp(t.Context(), nil) {
t.Fatal("the catch-up says a trunk was left for later")
}
if !mergeKnown(t, open, "mesh-catalog-c02") {
t.Fatal("the merge the controller did not hear was not planned")
}
kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02")
if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") {
t.Fatalf("the merge does not say it was caught up: %q", said)
}
cutAt(t, open, t0.Add(10*time.Minute))
ws, _ := walks(t, open)
if len(ws) == 0 {
t.Fatal("no walk answers the caught-up merge")
}
if _, in := ws[0].Modules["notes"]; !in {
t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules)
}
before := tr.events
if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before {
t.Fatalf("a second start read %d merge(s) in full again", tr.events-before)
}
}
// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the
// earlier first, and the forge is read for that repository's trunk only.
func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
var order []string
inner := readMergeEvent
readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
order = append(order, tm.Commit)
return inner(ctx, owner, name, branch, tm)
}
t.Cleanup(func() { readMergeEvent = inner })
tr.read = nil
heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute))
if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil {
t.Fatal(err)
}
for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} {
if !mergeKnown(t, open, c) {
t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c)
}
}
if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" {
t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order)
}
if len(tr.read) == 0 {
t.Fatal("the forge was not read for the trunk of the merge heard")
}
for _, r := range tr.read {
if r != "novox/mesh-catalog" {
t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read)
}
}
}
// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window.
func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
modules := make([]string, 51)
for i := range modules {
modules[i] = "app"
}
aTrunkOf(t, "mesh-catalog", modules...)
// Fifty are known; the oldest is not.
for i := 2; i <= 51; i++ {
c := fmt.Sprintf("mesh-catalog-c%02d", i)
hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second))
}
// Paged as the forge pages: fifty a page, and a trunk of another repository is empty (a short page, or none).
pages := trunkCommitsPage
trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) {
all, err := pages(ctx, owner, name, branch, 1)
if err != nil || page < 1 || (page-1)*50 >= len(all) {
return nil, err
}
return all[(page-1)*50 : min(page*50, len(all))], nil
}
t.Cleanup(func() { trunkCommitsPage = pages })
(following{open: open}).catchUp(t.Context(), nil)
if mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatal("the oldest of fifty-one merges was read and planned, outside the window")
}
}
// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and
// commit; a second start leaves that one condition open, observed twice; planning the gap clears it.
func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
aKeptMergeBefore(t, open)
keeper, _ := withConditionsInMemory(t)
tr := aTrunkOf(t, "mesh-catalog", "notes")
tr.approved["mesh-catalog-c01"] = false
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
catch()
if mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatal("a merge with no Approve on record was planned")
}
raised := func() []conditions.Condition {
t.Helper()
all, err := keeper.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var out []conditions.Condition
for _, c := range all {
if c.Kind == kindMergeNotHeard {
out = append(out, c)
}
}
return out
}
one := raised()
if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") {
t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one)
}
catch()
if again := raised(); len(again) != 1 || again[0].Observations < 2 {
t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again)
}
tr.approved["mesh-catalog-c01"] = true
catch()
if !mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatal("the gap was not planned once its Approve was on record")
}
if left := raised(); len(left) != 0 {
t.Fatalf("planning the gap left its condition open: %+v", left)
}
}
// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it.
func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
aKeptMergeBefore(t, open)
tr := aTrunkOf(t, "mesh-catalog", "notes")
tr.silent = true
if later := (following{open: open}).catchUp(t.Context(), nil); !later {
t.Fatal("a forge that did not answer counted as nothing to catch up")
}
if mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatal("a merge was planned without the forge's word")
}
tr.silent = false
if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatalf("the next start did not catch up (left for later: %v)", later)
}
}
// Finding 2 of the review: a gap that adds a module is a gap, judged as hearMerge judges it. With the operator's Approve on record it
// is heard (hearMerge asks the new module in its walk); without, it raises the condition and is not lost in silence.
func TestAGapThatAddsAModuleIsHeardOnItsWarrantAndRaisedWithout(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
aKeptMergeBefore(t, open)
keeper, _ := withConditionsInMemory(t)
tr := aTrunkOf(t, "mesh-catalog", "newmodule")
tr.approved["mesh-catalog-c01"] = false
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
catch()
if mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatal("a merge adding a module was heard with no Approve on record")
}
all, err := keeper.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var raised int
for _, c := range all {
if c.Kind == kindMergeNotHeard {
raised++
}
}
if raised != 1 {
t.Fatalf("a merge adding a module with no Approve raised %d condition(s), not one: it was lost in silence", raised)
}
tr.approved["mesh-catalog-c01"] = true
catch()
if !mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatal("a merge adding a module was not heard once its Approve was on record")
}
}
// A trunk with merges from before the controller began keeping merges, and merges since: only the later ones are gaps.
func TestAMergeFromBeforeTheRecordBeganIsHistoryAndNoCondition(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
keeper, _ := withConditionsInMemory(t)
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes", "notes")
tr.approved["mesh-catalog-c01"], tr.approved["mesh-catalog-c02"], tr.approved["mesh-catalog-c03"] = false, false, false
// The record begins at the second merge: the first is two days older.
tr.merges[0].At = t0.Add(-48 * time.Hour)
hear(t, open, catalogueMerge("mesh-catalog-c02", "notes", t0.Add(time.Minute)), t0.Add(time.Minute))
delete(tr.approved, "mesh-catalog-c02")
(following{open: open}).catchUp(t.Context(), nil)
// The first merge (two days before the record) was not even read from the forge: only the third, which is unknown, was.
if tr.events != 1 {
t.Fatalf("%d merge(s) were read from the forge: the one from before the record is history and is not read", tr.events)
}
all, err := keeper.Open(t.Context())
if err != nil {
t.Fatal(err)
}
var raised int
for _, c := range all {
if c.Kind == kindMergeNotHeard {
raised++
}
}
if raised != 1 {
t.Fatalf("%d condition(s) raised: only the merge since the record began (the third) is a gap", raised)
}
}
// novox/hq ADR 0463 §2: the record begins at the earliest merge kept on ANY trunk. A trunk the controller kept no merge of has the
// merges made since as gaps (mesh-delivery #1 on novox), and the merges made before as history.
func TestATrunkWithNoKeptMergeHasItsLaterMergesAsGapsAndItsEarlierOnesAsHistory(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asksRecorded(t)
// The record begins on another trunk, at t0 less an hour.
hear(t, open, repoMerge("one", "c-one", t0.Add(-time.Hour)), t0.Add(-time.Hour))
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes")
tr.merges[0].At = t0.Add(-72 * time.Hour) // before the record: history
(following{open: open}).catchUp(t.Context(), nil)
if mergeKnown(t, open, "mesh-catalog-c01") {
t.Fatal("a merge from before the record began was heard")
}
if !mergeKnown(t, open, "mesh-catalog-c02") {
t.Fatal("a merge since the record began, on a trunk with no merge kept, was not caught up")
}
}
cmd/mesh-controller/catch_up_test.go
-318
View File
@@ -1,318 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// The pieces of the catch-up that need no store (novox/hq ADR 0457, issue 1336).
func TestAMergeCommitsSubjectNamesItsPullRequest(t *testing.T) {
title, number, ok := pullOfSubject("Merge pull request 'Fix keycloak's sign-in (issue 1332)' (#378) from fix/1332-sign-in into main")
if !ok || number != 378 || title != "Fix keycloak's sign-in (issue 1332)" {
t.Fatalf("read %q %d %v", title, number, ok)
}
if b := headBranchOfSubject("Merge pull request 'x' (#378) from fix/1332-sign-in into main"); b != "fix/1332-sign-in" {
t.Fatalf("the head branch read as %q", b)
}
if _, _, ok := pullOfSubject("Merge branch 'main' of somewhere"); ok {
t.Fatal("a merge that names no pull request read as one")
}
}
func TestOnlyTheLastFiftyMergesOfATrunkAreRead(t *testing.T) {
pages := map[int][]trunkCommit{}
var all []trunkCommit // newest first: 51 merges, a plain commit between each pair
for i := 51; i >= 1; i-- {
all = append(all, trunkCommit{Commit: fmt.Sprintf("merge-%02d", i), SecondParent: fmt.Sprintf("head-%02d", i),
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from fix/x into main", i, i)},
trunkCommit{Commit: fmt.Sprintf("plain-%02d", i)})
}
for i := 0; i < len(all); i += 50 {
pages[i/50+1] = all[i:min(i+50, len(all))]
}
was := trunkCommitsPage
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { return pages[page], nil }
t.Cleanup(func() { trunkCommitsPage = was })
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
if err != nil {
t.Fatal(err)
}
if len(got) != catchUpWindow || got[0].Commit != "merge-02" || got[len(got)-1].Commit != "merge-51" {
t.Fatalf("read %d merges from %s to %s: the oldest of 51 is outside the window", len(got), got[0].Commit, got[len(got)-1].Commit)
}
if got[0].Head != "head-02" {
t.Fatalf("a merge's head is its second parent, not %q", got[0].Head)
}
}
func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) {
was := trunkCommitsPage
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) {
return nil, errors.New("the forge did not answer")
}
t.Cleanup(func() { trunkCommitsPage = was })
if got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main"); err == nil {
t.Fatalf("a forge that did not answer read as %d merges: nothing to catch up", len(got))
}
}
func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) {
ok := approvedMerge(t)
if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil {
t.Errorf("the operator's Approve under the teller was refused: %v", err)
}
// 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken.
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil {
t.Error("a warrant under mesh-delivery was accepted")
}
if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil {
t.Error("a warrant of an asker the controller takes none from was accepted")
}
declined := ok.warrant
declined.Option, declined.Label = "decline", "Decline"
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &declined); err == nil {
t.Error("a Decline was accepted as a warrant")
}
unproved := ok.warrant
by := *unproved.By
by.Verified = ""
unproved.By = &by
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &unproved); err == nil {
t.Error("an answer whose channel proves nobody was accepted")
}
if _, err := caughtUpWarrant(theDelivery, nil, nil); err == nil {
t.Error("a router that holds nothing was accepted")
}
}
// aTellersAsk is the ask the delivery gate composes for a pull request, as the router holds it once the operator chose Approve
// (the gate's composeMergeAsk: its id from sdkdelivery.AskingID of the group and the heads, its Approve bound to mergeAct over the
// heads, the pull request and its head's first twelve digits in the details), with the warrant the router wrote.
func aTellersAsk(t *testing.T, group string, members ...sdkdelivery.Head) (asks.Ask, asks.Warrant) {
t.Helper()
key, err := sdkdelivery.MergeAskKey(group, members)
if err != nil {
t.Fatal(err)
}
id := sdkdelivery.AskingID(key, 1)
heads := map[string]string{}
var details []string
for _, m := range members {
owner, repo, _ := strings.Cut(m.Repository, "/")
heads[deliveryIDOf(owner, repo, m.Head)] = m.Head
details = append(details, fmt.Sprintf("%s#%d %q at %s: issues: none named; review: approved; reaches: no delivery plan yet", repo, m.Number, "title", m.Head[:12]))
}
bind := func(option string) string {
d, err := asks.ActDigest(mergeAct(id, option, heads))
if err != nil {
t.Fatal(err)
}
return d
}
at := time.Date(2026, 10, 20, 12, 2, 1, 0, time.UTC)
ask := asks.Ask{ID: id, Headline: "Merge into main", Explanation: "passed its checks.", Who: asks.Operator, Expires: at.Add(24 * time.Hour),
OnExpiry: "nothing is merged; the ask is made again", Details: strings.Join(details, "\n"),
Options: []asks.Option{
{ID: optionApprove, Label: "Approve", Does: "merges", Level: asks.Approve, Binds: bind(optionApprove)},
{ID: "decline", Label: "Decline", Does: "nothing is merged", Level: asks.Approve, Binds: bind("decline")}}}
w := asks.Warrant{Ask: id, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve",
Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"},
At: at.Add(time.Minute), AskDigest: ask.Digest()}
return ask, w
}
// withRouter hands findMergeWarrant these asks as the router's records, by asker.
func withRouter(t *testing.T, records map[string]map[string]struct {
ask asks.Ask
w asks.Warrant
}) {
t.Helper()
wasIDs, wasAsk := routerAskIDs, routerAskOf
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
routerAskIDs = func(_ context.Context, _ catalogueReader, asker string) ([]string, error) {
var ids []string
for id := range records[asker] {
ids = append(ids, id)
}
return ids, nil
}
routerAskOf = func(_ context.Context, _ catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
r, ok := records[asker][id]
if !ok {
return nil, nil, nil
}
return &r.ask, &r.w, nil
}
t.Cleanup(func() {
routerAskIDs, routerAskOf = wasIDs, wasAsk
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
})
}
type aRouterRecord = struct {
ask asks.Ask
w asks.Warrant
}
// The teller's asks, as the router held them on 2026-10-20: the id is the router's own key, rebuilt here from the group and the
// head the gate composed it for (mesh-controller#405 at 8a330afd, mesh-catalog#381 at 1c702422, mesh-host#124 at 52600ec7). It names
// neither the pull request nor the head, so the warrant is found by the heads the Approve is bound to.
func TestATellersWarrantIsFoundByTheHeadItsApproveIsBoundTo(t *testing.T) {
real := []struct {
group, key, owner, repo string
number int
head string
}{
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "novox", "mesh-controller", 405, "8a330afd44ba852873339cf1210e90989dca46e1"},
{"fix/1351-remove-todays-delivery-module", "merge-fix_1351_remove_todays_delivery_module-64a353-d93b97b9697b", "novox", "mesh-catalog", 381, "1c7024224dd77c5b603299378883e0993b2e2fdd"},
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-6a59ba48da14-424e053ebc9c", "novox", "mesh-host", 124, "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"},
}
teller := map[string]aRouterRecord{}
for _, r := range real {
ask, w := aTellersAsk(t, r.group, sdkdelivery.Head{Repository: r.owner + "/" + r.repo, Number: r.number, Head: r.head})
if ask.ID != r.key {
t.Fatalf("the id of %s#%d is %q, and the router held it as %q: the fixture is not the router's", r.repo, r.number, ask.ID, r.key)
}
teller[ask.ID] = aRouterRecord{ask, w}
}
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: teller})
for _, r := range real {
why, found, reason, err := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, r.head)
if err != nil || !found || !strings.Contains(why, r.key) || !strings.Contains(why, "bound to the head of this pull request") {
t.Errorf("%s#%d: the warrant was not found: %q %v %q %v", r.repo, r.number, why, found, reason, err)
}
// Another head of the same pull request is another merge: the Approve was not given for it.
if _, found, _, _ := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, "ffffffffffffffffffffffffffffffffffffffff"); found {
t.Errorf("%s#%d: another head found the warrant of this one", r.repo, r.number)
}
}
// One pull request's Approve is none for another repository's merge at the same head digits.
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-tools", 405, real[0].head); found {
t.Error("a merge of another repository found mesh-controller#405's warrant")
}
}
// A group's ask binds several heads, and the router's record carries only twelve digits of the other members': the pull request is
// found in the ask as it was published, which the warrant's digest covers.
func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testing.T) {
a := sdkdelivery.Head{Repository: "novox/mesh-host", Number: 124, Head: "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"}
b := sdkdelivery.Head{Repository: "novox/mesh-controller", Number: 405, Head: "8a330afd44ba852873339cf1210e90989dca46e1"}
ask, w := aTellersAsk(t, "feat/r4b-sdk-users", a, b)
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, w}}})
for _, h := range []sdkdelivery.Head{a, b} {
owner, repo, _ := strings.Cut(h.Repository, "/")
why, found, reason, err := findMergeWarrant(t.Context(), nil, owner, repo, h.Number, h.Head)
if err != nil || !found || !strings.Contains(why, "a group's ask naming this pull request at this head") {
t.Errorf("%s#%d: not found in the group's ask: %q %v %q %v", repo, h.Number, why, found, reason, err)
}
}
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 381, a.Head); found {
t.Error("a pull request the group's ask does not name found its warrant")
}
// A Decline is no warrant.
declined := w
declined.Option, declined.Label = "decline", "Decline"
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, declined}}})
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-host", 124, a.Head); found {
t.Error("a Decline found a warrant")
}
}
// A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do.
func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) {
was := trunkCommitsPage
t.Cleanup(func() { trunkCommitsPage = was })
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
if page > 1 {
return nil, nil
}
return []trunkCommit{{Commit: "m2", SecondParent: "h2", Subject: "Merge pull request 'two' (#2) from fix/b into main"},
{Commit: "p"}, {Commit: "m1", SecondParent: "h1", Subject: "Merge pull request 'one' (#1) from fix/a into main"}}, nil
}
got, err := readTrunkMerges(t.Context(), "novox", "x", "main")
if err != nil || len(got) != 2 || got[0].Commit != "m1" || got[1].Commit != "m2" {
t.Fatalf("a short trunk read as %+v %v", got, err)
}
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { return nil, nil }
if got, err := readTrunkMerges(t.Context(), "novox", "x", "main"); err != nil || len(got) != 0 {
t.Fatalf("an empty trunk read as %+v %v", got, err)
}
}
// ADR 0253: what the operator reads of a merge not heard is plain: no hash, no identifier, nothing to do "by hand".
func TestTheConditionOfAMergeNotHeardIsInPlainWords(t *testing.T) {
keeper, _ := withConditionsInMemory(t)
tm := trunkMerge{Commit: "c0880c4d1f2e3a4b5c6d7e8f9a0b1c2d3e4f5a6b", Head: "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372"}
(following{}).raiseMergeNotHeard(t.Context(), "novox/mesh-catalog", "main", tm, link.SourceMoved{Number: 378},
"the router holds no chosen merge ask of mesh-delivery-teller or mesh-delivery that covers the head b9104454f9a5")
open, err := keeper.Open(t.Context())
if err != nil || len(open) != 1 {
t.Fatalf("opened %d conditions, %v", len(open), err)
}
c := open[0]
if why, ok := conditions.PlainWords(conditions.Words{Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs,
Resolved: c.Resolved}); !ok {
t.Fatalf("the condition's words are not plain: %s\n%q\n%q\n%q", why, c.Headline, c.Explanation, c.Needs)
}
}
// The trunk's history lists the merges made inside a pull request's branch too. On novox (2026-10-20) 15 of the 15 non-pull-request
// "gaps" of mesh-catalog and mesh-controller were such merges; only a pull request's merge into this trunk is a merge to catch up.
func TestAMergeMadeInsideABranchIsNoMergeOfTheTrunk(t *testing.T) {
was := trunkCommitsPage
t.Cleanup(func() { trunkCommitsPage = was })
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
if page > 1 {
return nil, nil
}
return []trunkCommit{
{Commit: "c0880c4d", SecondParent: "b9104454", Subject: "Merge pull request 'Read the grant file' (#378) from fix/1332 into main"},
{Commit: "6394a62a", SecondParent: "x1", Subject: "Merge remote-tracking branch 'origin/main' into fix/1081-withdraw"},
{Commit: "896f1d28", SecondParent: "x2", Subject: "Merge feat/agents-write-git-ssh into feat/agents-write-git-ssh-f (issue 955 F)"},
{Commit: "4d03755c", SecondParent: "x3", Subject: "Bring the branch up to main after #354 (issue 1047)"},
{Commit: "other", SecondParent: "x4", Subject: "Merge pull request 'Elsewhere' (#5) from fix/y into nox-mesh"},
}, nil
}
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
if err != nil || len(got) != 1 || got[0].Commit != "c0880c4d" {
t.Fatalf("read %+v %v: only keycloak's pull request merge into main is a merge to catch up", got, err)
}
}
// novox/hq ADR 0463 §2: the record of merges begins at the earliest merge the controller holds on any trunk. A merge before it is
// history on every trunk; a trunk with no merge kept of its own still has the merges made since as gaps (mesh-delivery #1 on novox).
func TestOnlyTheMergesSinceTheRecordBeganAreGapsOnEveryTrunk(t *testing.T) {
first := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) // the first merge it heard, on some trunk of the mesh
trunk := func(prefix string) []trunkMerge {
return []trunkMerge{
{Commit: prefix + "-old-1", At: first.Add(-30 * 24 * time.Hour)},
{Commit: prefix + "-old-2", At: first.Add(-2 * time.Hour)},
{Commit: prefix + "-within-grace", At: first.Add(-mergeGrace / 2)},
{Commit: prefix + "-since", At: first.Add(24 * time.Hour)},
}
}
// Two trunks, one of which the controller has kept no merge of: both read by the one starting point.
for _, prefix := range []string{"kept-trunk", "trunk-with-no-kept-merge"} {
var commits []string
for _, m := range sinceTheRecord(trunk(prefix), true, first) {
commits = append(commits, m.Commit)
}
if want := prefix + "-within-grace " + prefix + "-since"; strings.Join(commits, " ") != want {
t.Errorf("%s: the gaps are %v, want %q: earlier ones are history on every trunk, later ones are gaps", prefix, commits, want)
}
}
if none := sinceTheRecord(trunk("x"), false, time.Time{}); len(none) != 0 {
t.Errorf("with no merge kept anywhere there are %d gap(s), but no record to have one in", len(none))
}
}
cmd/mesh-controller/chain_integration_test.go
-232
View File
@@ -1,232 +0,0 @@
//go:build integration
package main
import (
"context"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/chain"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there,
// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live
// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's
// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send
// is recorded as the controller records one it made, and the gate's facts are the store's.
func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asked := asksRecorded(t)
ctx := t.Context()
inv := open.inventory
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
t.Fatal(err)
}
var sent [][]string
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
sent = append(sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
digest := fmt.Sprintf("d-%s-%d", node, len(sent))
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest,
map[string]string{"notes": current["notes"].Commit}); err != nil {
return nil, err
}
}
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
wasGather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
f.reports[r.Node] = r
}
return f, nil
}
t.Cleanup(func() { gatherGateFacts = wasGather })
// A merge of notes is heard, and its window closes: the batch is cut into the walk.
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
cutAt(t, open, t0.Add(time.Second+90*time.Second))
ws, _ := walks(t, open)
if len(ws) != 1 {
t.Fatalf("%d walk(s) after the cut", len(ws))
}
rollout := ws[0].ID
// The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the
// outcome in as it does.
asking, err := inv.PlanByID(ctx, rollout)
if err != nil {
t.Fatal(err)
}
for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ {
advancePlans(ctx, open)
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
t.Fatal(err)
}
}
buildID := asking.Modules["notes"].Build
if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) {
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
}
// The walk sends the tier to its first machine.
for try := 0; try < 3 && len(sent) == 0; try++ {
advancePlans(ctx, open)
}
if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) {
t.Fatalf("sent %v", sent)
}
// One read of the assignment.
on, err := inv.AssignmentsOn(ctx, "anchor")
a, ok := assignmentOf(on, "notes")
if err != nil || !ok {
t.Fatalf("the assignments of anchor: %+v %v", on, err)
}
// Beside it a module pulled in by the assignment, which no send of this walk put anywhere.
if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") {
t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard)
}
if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout ||
a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() {
t.Fatalf("the assignment names %+v, not commit c-notes, build %s, rollout %s", a, buildID, rollout)
}
// One read of the rollout.
p, err := inv.PlanByID(ctx, rollout)
if err != nil {
t.Fatal(err)
}
wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}}
if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 ||
p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil {
t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange)
}
// And one read of the build names the same commit.
built, found, err := inv.BuildByID(ctx, buildID)
if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" {
t.Fatalf("the build: %+v %v %v", built, found, err)
}
// The verbs say the links.
if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") ||
!strings.Contains(lines[1], buildID) {
t.Fatalf("`plans` says %v", lines)
}
if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) {
t.Fatalf("`node show` says %v", lines)
}
}
// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks
// nothing (issue 1095).
func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil {
t.Fatal(err)
}
node := nodeID(t, open, "anchor")
inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}})
if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil {
t.Fatal(err)
}
// Sent again by hand, with no rollout: the assignment still names the rollout that put the build there.
if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil {
t.Fatal(err)
}
on, _ := inv.AssignmentsOn(ctx, "anchor")
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
t.Fatalf("the assignment names %+v", on)
}
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
t.Fatal(err)
}
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
t.Fatalf("a send of unknown builds left %+v", on)
}
// A module sent a commit no build is kept of is named by its commit alone.
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
t.Fatal(err)
}
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" ||
runOf(on, "notes").Rollout != "" {
t.Fatalf("the assignment after a push of c2 by hand names %+v", on)
}
}
// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it.
func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) {
for _, a := range on {
if a.Module == module {
return a, true
}
}
return chain.Assignment{}, false
}
func runOf(on []chain.Assignment, module string) chain.Assignment {
a, _ := assignmentOf(on, module)
return a
}
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
// and the table of pending assignments is read by nobody. No key is written by hand.
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
open := aCatalogueMesh(t)
ctx := t.Context()
inv := open.inventory
asksWithPaths(t)
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
t.Fatalf("an assignment while its build runs was refused: %v", err)
}
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
!strings.HasPrefix(waiting.Pending.ID, "p-") {
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
}
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
t.Fatalf("a module only waiting is assigned: %v", got)
}
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
t.Fatal(err)
}
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
}
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
t.Fatal("the build registered sensors and it was not assigned")
}
}
cmd/mesh-controller/chain_no_table_integration_test.go
-77
View File
@@ -1,77 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
)
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
asked := asksRecorded(t)
ctx := t.Context()
inv := open.inventory
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
t.Fatal(err)
}
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
cutAt(t, open, t0.Add(91*time.Second))
ws, _ := walks(t, open)
if len(ws) != 1 {
t.Fatalf("%d walk(s) after the cut", len(ws))
}
rollout := ws[0].ID
asking, err := inv.PlanByID(ctx, rollout)
if err != nil {
t.Fatal(err)
}
for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ {
advancePlans(ctx, open)
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
t.Fatal(err)
}
}
buildID := asking.Modules["notes"].Build
if buildID == "" || len(*asked) != 1 {
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
}
// What the live path made is in the chain.
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
}
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
}
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
t.Fatal(err)
} else if _, ok := assignmentOf(on, "notes"); !ok {
t.Fatalf("the assignment is not read from the chain: %+v", on)
}
// And the catalog of the store holds none of them.
have := map[string]bool{}
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
have[strings.Trim(row[0], `"`)] = true
}
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
if have[table] {
t.Errorf("the table %s is in the controller's store", table)
}
}
for _, table := range []string{"build_gate", "assignment_generation"} {
if !have[table] {
t.Errorf("the table %s, which stays, is gone", table)
}
}
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
}
}
cmd/mesh-controller/changeplan.go
-192
View File
@@ -1,192 +0,0 @@
package main
import (
"fmt"
"path"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
// planner, so the two can be compared.
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
type policyOf func(module string) (inventory.Upgrade, bool)
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
holder := ""
for _, e := range entries {
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
holder = e.On[0]
}
}
var granting []string
machines := map[string]*link.MachinePlan{}
machine := func(name string) *link.MachinePlan {
if machines[name] == nil {
machines[name] = &link.MachinePlan{Machine: name}
}
return machines[name]
}
for _, tier := range r.Plan.Tiers {
for _, name := range tier {
e, held := byName[name]
if !held {
continue
}
u, known := policy(name)
waits := known && !u.RollOut
for _, on := range e.On {
if waits {
machine(on).Waits = append(machine(on).Waits, name)
} else {
machine(on).Receives = append(machine(on).Receives, name)
}
}
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
!slices.Contains(granting, name) {
granting = append(granting, name)
}
switch {
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
// until a person takes the step, so merging it is a promise to take it.
p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+
"nothing else is sent to %s either — unless the new build turns out the same as the one running there "+
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData):
// Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377).
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+
"its move recreates none of its containers and moves none of its data, which the walk judges and then sends",
name, orNone(u.From)))
case waits && len(e.On) > 0:
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
orNone(u.From)))
}
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
var offers []string
for _, o := range e.Manifest.Provides {
offers = append(offers, o.Name)
}
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
name, strings.Join(offers, ", ")))
}
if e.Manifest.Data != nil && len(e.On) > 0 {
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
}
}
}
if len(granting) > 0 {
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
}
var names []string
for name := range machines {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
p.Machines = append(p.Machines, *machines[name])
}
p.Summary = summaryOf(p)
p.Class = changeClassOf(p, entries)
return p
}
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
const classNone = "none"
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
// resolver claim its manifest makes.
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
return classNone
}
modules := append(append([]string{}, p.Moved...), p.Dependents...)
for _, dir := range p.New {
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
modules = append(modules, name)
}
}
return classOfModules(modules, entries)
}
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
const grantsStepWord = "grants step"
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
func summaryOf(p link.ChangePlan) string {
if len(p.Moved) == 0 && len(p.New) == 0 {
return "builds nothing: the change touches no module of the mesh's graph"
}
var parts []string
what := strings.Join(p.Moved, ", ")
if len(p.Dependents) > 0 {
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
}
if len(p.New) > 0 {
if what != "" {
what += ", "
}
what += "new: " + strings.Join(p.New, ", ")
}
var to []string
for _, m := range p.Machines {
if len(m.Receives) > 0 {
to = append(to, m.Machine)
}
}
if len(to) > 0 {
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
} else {
parts = append(parts, "builds "+what+", sent nowhere")
}
bus := "no bus step"
for _, s := range p.Steps {
switch {
case strings.HasPrefix(s, busUpgradeNeeded):
bus = busUpgradeNeeded
case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step":
bus = "a bus step"
}
}
parts = append(parts, bus)
waiting := 0
for _, m := range p.Machines {
waiting += len(m.Waits)
}
if waiting > 0 {
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
}
return strings.Join(parts, "; ")
}
cmd/mesh-controller/changeplan_test.go
-75
View File
@@ -1,75 +0,0 @@
package main
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
// person, a provider whose consumers follow it.
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
entry := func(name, path string, on ...string) inventory.Entry {
e := fromRepo(name, catalogue_, path)
e.Source.BuiltFrom = "old"
e.On = on
return e
}
nats := entry("nats", "modules/nats", "anchor")
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
gitea := entry("gitea", "modules/gitea", "anchor")
held := entry("photos", "modules/photos", "anchor", "laptop")
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
tools.On = []string{"anchor", "laptop"}
entries := []inventory.Entry{nats, gitea, held, tools}
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
policy := func(module string) (inventory.Upgrade, bool) {
if module == "photos" {
return inventory.Upgrade{RollOut: false, From: "a person"}, true
}
return inventory.Upgrade{RollOut: true}, true
}
plan := func(paths ...string) link.ChangePlan {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
}
p := plan("modules/gitea/index.ts")
if p.Summary != "builds gitea → anchor; no bus step" {
t.Errorf("one module's change reads %q", p.Summary)
}
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
t.Errorf("the bus and a held module read %q", p.Summary)
}
got := map[string]string{}
for _, m := range p.Machines {
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
}
// The bus first, what stands on it after: the build plan's order, per machine.
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
t.Errorf("the deploy plan reads %v", got)
}
text := strings.Join(p.Steps, "\n")
// Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a
// person runs the bus's step, and the verb that does.
for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor",
"mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person",
"nats provides mesh-bus"} {
if !strings.Contains(text, want) {
t.Errorf("the steps do not say %q:\n%s", want, text)
}
}
p = plan("merge-check.sh")
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
t.Errorf("a root file's plan reads %+v", p)
}
}
cmd/mesh-controller/checks.go
+3 -6
View File
@@ -38,10 +38,8 @@ type checkScope struct {
New []string
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
Manifests []string
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
// module's build reads.
Width, Tiers int
Unread []string
// Unread the changed files no module's build reads.
Unread []string
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
// core's, or the change adds a module to it.
Mesh bool
@@ -62,8 +60,7 @@ func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
ModuleDirsSaid: p.ModuleDirsSaid}
r := reachOfMerge(m, entries, read, edges)
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, Reach: r}
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
}
cmd/mesh-controller/checks_test.go
-57
View File
@@ -93,60 +93,3 @@ func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
}
}
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
// module in both, and what stands on a touched module is named as its dependents in both.
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
nats := fromRepo("nats", catalogue, "modules/nats")
nats.Source.BuiltFrom = "old"
gitea := fromRepo("gitea", catalogue, "modules/gitea")
gitea.Source.BuiltFrom = "old"
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
entries := []inventory.Entry{nats, gitea, tools}
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
read := map[string][]inventory.ReadRepository{}
for _, c := range []struct {
what string
paths []string
moved, dependents string
width int
unread string
}{
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
} {
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
s := pullScope(p, entries, read, edges)
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
r := reachOfMerge(m, entries, read, edges)
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
}
if got := strings.Join(s.Dependents, ","); got != c.dependents {
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
}
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
}
if got := strings.Join(s.Unread, ","); got != c.unread {
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
}
}
// A repository whose build another module's recipe packages: that module is reached through the
// planner's `read`, and what stands on it after it.
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
Paths: []string{"src/index.ts"}}, entries, read, edges)
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
}
}
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
cmd/mesh-controller/compose_input.go
-25
View File
@@ -267,19 +267,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build}
}
}
// A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its
// manifest and the commit, applied to this node's own resolution only.
// What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it).
kept, err := recordedKept(ctx, open, n.Name)
if err != nil {
*refused = err
} else if len(kept) > 0 {
out.KeptBuilds = kept
if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil {
*refused = err
out.KeptBuilds, out.Kept = nil, nil
}
}
stated, has, err := inv.HealthOf(ctx, n.Name)
if err != nil {
return out, err
@@ -291,18 +278,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
return out, nil
}
// keptManifests is the manifest of each module kept at the build a machine runs: keepRecorded's reads, and its refusals in its
// words, without putting any of it into a shelf (that is the node's own Kept, applied by the composition).
func keptManifests(ctx context.Context, open *stores, node string, kept map[string]string) (map[string]catalogue.Manifest, error) {
// applyKept puts each manifest into the shelf it is given and refuses a build that cannot be read; the shelf here is a
// scratch one, whose entries are the kept manifests.
scratch := map[string]catalogue.Manifest{}
if err := applyKept(ctx, open, node, kept, scratch); err != nil {
return nil, err
}
return scratch, nil
}
func readTunnels(ctx context.Context, inv *inventory.Inventory) (map[string]compose.FoundTunnel, error) {
found, err := inv.Tunnels(ctx)
if err != nil {
cmd/mesh-controller/compose_input_integration_test.go
-65
View File
@@ -6,11 +6,9 @@ import (
"reflect"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"github.com/novox/mesh-controller/internal/link"
)
// 1382: the adapter's Input names, for every node, what the inventory answers for it, each compared with the read named
@@ -132,69 +130,6 @@ func nodePorts(in compose.Input, name string) map[string]map[int]int {
return n.Ports
}
// 1382/AC2: a node keeping a recorded module at the build it runs has that module in Kept and KeptBuilds, and the Input's
// Catalogue is the current shelf, not the kept manifest (R4a review round 1, finding 1).
func TestANodeKeepingARecordedBuildHasItInKeptAndTheCatalogueStaysCurrent(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
image := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64)
for _, b := range []link.BuildResult{
aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, map[string][2]string{"server": {image, ""}}),
} {
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
t.Fatal(err)
}
}
if _, err := inv.Assign(ctx, "anchor", "postgres"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"postgres": "c1111111"}); err != nil {
t.Fatal(err)
}
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute),
map[string][2]string{"server": {image, "runtime"}})); err != nil {
t.Fatal(err)
}
// A send that is not a person's push keeps the build the machine runs.
in, err := readComposeInput(keepingRecorded(ctx), open)
if err != nil {
t.Fatal(err)
}
anchor, _ := nodeOf(in.Input, "anchor")
if anchor.KeptBuilds["postgres"] != "c1111111" {
t.Fatalf("anchor keeps %v, want postgres at c1111111", anchor.KeptBuilds)
}
kept, found := anchor.Kept["postgres"]
if !found {
t.Fatalf("the kept manifest of postgres is not in anchor's Kept: %v", anchor.Kept)
}
current, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(in.Catalogue["postgres"], current["postgres"]) {
t.Error("the Input's Catalogue is not the current shelf")
}
if reflect.DeepEqual(kept, current["postgres"]) {
t.Error("the kept manifest is the current one: nothing is kept")
}
if laptop, _ := nodeOf(in.Input, "laptop"); len(laptop.Kept) != 0 || len(laptop.KeptBuilds) != 0 {
t.Errorf("laptop keeps what only anchor runs: %v", laptop.KeptBuilds)
}
// A person's push keeps nothing.
pushed, err := readComposeInput(ctx, open)
if err != nil {
t.Fatal(err)
}
if a, _ := nodeOf(pushed.Input, "anchor"); len(a.Kept) != 0 || len(a.KeptBuilds) != 0 {
t.Errorf("a person's push keeps %v", a.KeptBuilds)
}
}
// 1382/AC3: the seats on record, the tunnels, the adopted tunnel, its carried peers and the nodes joining equal the
// inventory's answers; the range and the suffix are read from the environment once, here.
func TestTheAdaptersInputCarriesTheNetworkAsTheInventoryHoldsIt(t *testing.T) {
cmd/mesh-controller/compose_sdk.go
+1 -9
View File
@@ -48,15 +48,7 @@ func freshInput(ctx context.Context) context.Context {
// keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs
// (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send.
func keptSignature(ctx context.Context) string {
skip, on := keptExcept(ctx)
var names []string
for m := range skip {
names = append(names, m)
}
sort.Strings(names)
every, _ := ctx.Value(keepEveryBuildKey{}).(string)
settings, _ := ctx.Value(keepForSettingsKey{}).(string)
return fmt.Sprintf("kept=%v;except=%v;every=%s;settings=%s", on, names, every, settings)
return ""
}
// inputFor is the Input for ctx's operation.
cmd/mesh-controller/compose_sdk_test.go
-24
View File
@@ -1,7 +1,6 @@
package main
import (
"context"
"reflect"
"strings"
"testing"
@@ -86,26 +85,3 @@ func TestADeclarationThatStandsInForAValueIsStillForeseen(t *testing.T) {
}
}
}
// 1386: an Input read for a person's push is never the one read for a send that keeps recorded builds, and the cache is
// emptied by what a send made.
func TestAnInputIsCachedPerWhatItKeepsAndForgottenAfterASend(t *testing.T) {
ctx := withInputCache(context.Background())
plain := keptSignature(ctx)
kept := keptSignature(keepingRecorded(ctx))
bus := keptSignature(keepingRecorded(ctx, "nats"))
every := keptSignature(keepingEveryBuild(ctx, "anchor"))
for name, sig := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
for other, o := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
if name != other && sig == o {
t.Errorf("%s and %s read the same Input", name, other)
}
}
}
cache := ctx.Value(inputCacheKey{}).(*inputCache)
cache.by[plain] = &meshInput{}
forgetInput(ctx)
if len(cache.by) != 0 {
t.Fatal("what a send made left the Input it read in the cache")
}
}
cmd/mesh-controller/conditions_brief_test.go
-76
View File
@@ -1,76 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
)
// stalledConditions are n open conditions like those of 2026-10-08: one per delivery held past its bound,
// each keeping all its evidence.
func stalledConditions(n int) []conditions.Condition {
at := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
var out []conditions.Condition
for i := 0; i < n; i++ {
id := fmt.Sprintf("d-%012x", 0x51a11ed+i)
lines := stalledObservations([]stalledLine{{ID: id, State: "checking", For: "13h2m", Bound: "45m",
H2: "close: recheck", Says: "the check was asked and nothing has answered"}})
o := lines[0]
c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Subject: conditions.Subject{Scope: o.Scope, ID: o.ID},
Severity: o.Severity, Summary: o.Summary, Source: probeDeliveriesID, Raised: at, LastObserved: at,
Observations: 160, Count: 1, Resolver: conditions.ResolverSelf}
for e := 0; e < conditions.KeptEvidence; e++ {
c.Evidence = append(c.Evidence, conditions.Evidence{At: at.Add(time.Duration(e) * 5 * time.Minute),
Said: "checking for " + (13*time.Hour + time.Duration(e)*time.Minute).String()})
}
out = append(out, c)
}
return out
}
// novox/hq issue 314: a list of hundreds of open conditions carries each one's newest evidence, with
// how much more it keeps and where to read it, and the verb's answer says it once — as data, not again
// as the text the command printed. On 2026-10-08, 689 stalled deliveries made that answer more than the
// bus carries in one message; now it is less than half of one.
func TestAListOfConditionsCarriesEachInBrief(t *testing.T) {
list := stalledConditions(689)
printed, _ := json.MarshalIndent(map[string]any{"conditions": inBrief(list), "open": len(list),
"counted": counted(list)}, "", " ")
var parsed any
if err := json.Unmarshal(printed, &parsed); err != nil {
t.Fatal(err)
}
wire, _ := json.Marshal(map[string]any{"result": verbAnswer{OK: true, Answer: parsed,
Output: "its answer, as data, is `answer`\n"}})
before, _ := json.MarshalIndent(map[string]any{"conditions": list}, "", " ")
t.Logf("689 stalled deliveries: %d bytes on the wire, where the whole list printed was %d and was sent twice",
len(wire), len(before))
if len(wire) >= 512<<10 {
t.Fatalf("the answer of 689 conditions is %d bytes, not under half of one message of the bus", len(wire))
}
var doc struct {
Conditions []conditionInBrief `json:"conditions"`
Counted map[string]int `json:"counted"`
}
if err := json.Unmarshal(printed, &doc); err != nil {
t.Fatal(err)
}
c := doc.Conditions[0]
if len(doc.Conditions) != 689 || len(c.Evidence) != 1 || c.EvidenceKept != conditions.KeptEvidence ||
c.Key != list[0].Key || c.Summary != list[0].Summary || !strings.Contains(c.More, "key="+c.Key) {
t.Fatalf("a condition in brief reads %+v", c)
}
if doc.Counted["D14 stalled"] != 689 {
t.Fatalf("counted %v", doc.Counted)
}
if one := inBrief(list[:1])[0]; len(list[0].Evidence) != conditions.KeptEvidence || one.Evidence[0] != list[0].Evidence[0] {
t.Fatal("the brief changed the condition it was made from, or kept other than the newest")
}
}
// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and
// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314).
cmd/mesh-controller/core_box.go
+1 -1
View File
@@ -246,7 +246,7 @@ const kindVaultSilent = "vault-silent"
// vaultSilentObservation is that, as a condition: `machine.<machine>.vault-silent`.
func vaultSilentObservation(node string, b coreBoxes) conditions.Observation {
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent,
Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource}
Machine: node, Severity: conditions.Warning, Source: "sends"}
var said []string
if len(b.notPlaced) > 0 {
o.Severity = conditions.Urgent
cmd/mesh-controller/core_downgrade.go
-215
View File
@@ -1,215 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
// sent the control node the controller build before it. The older binary read the row it could not serve and the
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
//
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
// newer merge, or `rebuild mesh-controller`.
//
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
// schema already (issue 352).
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
const kindCoreDowngrade = "core-downgrade-kept"
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
const coreDowngradeSource = "sends"
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
// store and its seat's row, and the node-engine, which reads what the controller declares.
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
type askedBuild struct {
asked time.Time
failedGate bool
}
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
// variable, so a test reads its own order.
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
builds, err := inv.Builds(ctx, module, askedLookBack)
if err != nil {
return askedBuild{}, false, err
}
for _, b := range builds {
if b.Worked() && sameCommit(b.Commit, commit) {
failed, err := inv.GateFailed(ctx, b.ID)
if err != nil {
return askedBuild{}, false, err
}
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
}
}
return askedBuild{}, false, nil
}
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
const askedLookBack = 50
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
type coreDowngrade struct {
module, node, component string
// sent is the build the machine was last sent, record the one the mesh holds.
sent, record string
sentAsked, recordAsked time.Time
}
func (d coreDowngrade) String() string {
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
d.module)
}
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
// but for asked.
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
var out []coreDowngrade
for module, was := range sent {
component := coreComponent(module)
now := record[module]
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
continue
}
if scope.puttingBack && scope.modules[module] {
continue
}
from, known, err := asked(module, was)
if err != nil {
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
}
if !known || from.failedGate {
continue
}
to, known, err := asked(module, now)
if err != nil {
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
}
if known && to.asked.Before(from.asked) {
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
sentAsked: from.asked, recordAsked: to.asked})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
return out, nil
}
func guarded(component string) bool {
for _, c := range downgradeGuarded {
if c == component {
return true
}
}
return false
}
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
// was last sent, module → commit, each said and raised as a condition.
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
current map[string]inventory.CurrentBuild) (map[string]string, error) {
record := map[string]string{}
for m, c := range current {
record[m] = c.Commit
}
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
return buildAsked(ctx, inv, module, commit)
})
if err != nil || len(found) == 0 {
return nil, err
}
out := map[string]string{}
for _, d := range found {
out[d.module] = d.sent
sayCoreDowngrade(ctx, d)
}
return out, nil
}
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
Summary: summary}
}
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
fmt.Printf("KEPT: %s\n", d)
keeper := conditionsFrom
if keeper == nil {
return
}
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
}
}
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
keeper := conditionsFrom
if keeper == nil || carried == nil {
return
}
var open []string
for _, c := range downgradeGuarded {
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
open = append(open, c)
}
}
if len(open) == 0 {
return
}
current, err := inv.CurrentBuilds(ctx)
if err != nil {
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
return
}
for module, now := range current {
component := coreComponent(module)
if !guarded(component) {
continue
}
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
continue
}
key := coreDowngradeObservation(component, node, "").Key()
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
}
}
}
cmd/mesh-controller/delivery.go
-1213
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/delivery_conditions.go
-262
View File
@@ -1,262 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table
// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the
// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery.<id>.stalled`; healer
// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only
// the next observation says whether it worked (ADR 0231).
// probeDeliveriesID is the probe that reads the delivery's owner.
const probeDeliveriesID = "D14"
// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope.
const kindDeliveryStalled = "stalled"
// deliveryOwnerWithin is how long the owner is given to answer.
var deliveryOwnerWithin = 10 * time.Second
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
type stalledLine struct {
ID string `json:"id"`
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
Number int `json:"number,omitempty"`
State string `json:"state"`
// Walk is the walk a line is about: a waiting walk no live delivery holds, or, asked with `walks`, each walk
// waiting for the owner's word and what holds it (novox/hq issues 362, 529).
Walk string `json:"walk,omitempty"`
// Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each:
// "mesh/check pending since <UTC time>", "mesh/check never set" (novox/hq issue 438).
Waiting []string `json:"waiting,omitempty"`
// Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
// says none, and Waiting is said as it reads.
Checks []waitingCheck `json:"checks,omitempty"`
// Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held
// (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no
// delivery, and from a mesh-delivery that does not say it yet; the words are never read for it.
Held string `json:"held,omitempty"`
For string `json:"for"`
Bound string `json:"bound"`
H2 string `json:"h2"`
Says string `json:"says"`
// Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty
// or anything else is a warning, as before.
Severity string `json:"severity,omitempty"`
// Needs is "new-head" for a delivery whose cure is its author's new head (novox/hq ADR 0505 §5), Author that author
// as the operator reads it ("the author of feat/x", with the session that pushed it when its pull request names one)
// and Branch its branch. A mesh-delivery from before says none, and the words are as before.
Needs string `json:"needs,omitempty"`
Author string `json:"author,omitempty"`
Branch string `json:"branch,omitempty"`
}
// needsNewHead is what a stalled line needs when its cure is its author's new head (ADR 0505 §5).
const needsNewHead = "new-head"
// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise.
func (l stalledLine) severity() conditions.Severity {
if l.Severity == string(conditions.Urgent) {
return conditions.Urgent
}
return conditions.Warning
}
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say.
type waitingCheck struct {
Context string `json:"context"`
State string `json:"state"`
Since string `json:"since,omitempty"`
}
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
var wordsNow = time.Now
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
var wordsZone = func() *time.Location { return time.Local }
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
// a seam a test replaces. The holder's own refusal is an error naming it.
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
granted := false
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
granted = granted || v.Verb == verb
}
if !granted {
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
}
if conn == nil {
return nil, errors.New("this controller is not on the bus")
}
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
if err != nil {
return nil, err
}
if answer.Error != "" {
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
}
return unwrapToolResult(answer.Result), nil
}
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
// protocol's content list holding it as text.
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
var wrapped struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
return json.RawMessage(wrapped.Content[0].Text)
}
return raw
}
// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record,
// or none answers (D3 says that one).
func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) {
if !held {
return nil, nil
}
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{})
if errors.Is(err, link.ErrNothingServes) {
return nil, nil // the holder not running is D3's holder-silent, said once there
}
if err != nil {
return nil, err
}
var lines []stalledLine
if err := json.Unmarshal(raw, &lines); err != nil {
return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err)
}
return lines, nil
}
// walkWaits are, by walk, the lines mesh-delivery says of a walk waiting for its word: what holds it.
type walkWaits map[string][]stalledLine
// walksWaiting asks the delivery's owner what holds each walk waiting for its word (novox/hq issue 529): its
// `stalled` asked with `walks`, so a line of `plans` names a held delivery, its reason and the verb that releases
// it, instead of pointing at a hand start. Nil when no holder is on record or none answers: the line then reads as
// before. An owner from before this answers its stalled lines, of which only those naming a walk are kept.
func walksWaiting(ctx context.Context, conn *nats.Conn, held bool) walkWaits {
if !held {
return nil
}
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{"walks": true})
if err != nil {
return nil
}
var lines []stalledLine
if json.Unmarshal(raw, &lines) != nil {
return nil
}
out := walkWaits{}
for _, l := range lines {
if l.Walk != "" {
out[l.Walk] = append(out[l.Walk], l)
}
}
return out
}
// stalledObservations are the conditions of what is stalled.
func stalledObservations(lines []stalledLine) []conditions.Observation {
out := make([]conditions.Observation, 0, len(lines))
for _, l := range lines {
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
Severity: l.severity(),
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
if l.operatorsOnly() {
o.Resolver = conditions.ResolverOperator
}
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
out = append(out, o)
}
return out
}
// probeDeliveries is D14: every delivery held past its state's bound is said.
func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
entries, err := d.open.inventory.Catalogued(ctx)
if err != nil {
return nil, err
}
var conn *nats.Conn
if d.js != nil {
conn = d.js.Conn()
}
lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries))
if err != nil {
return nil, err
}
return stalledObservations(lines), nil
}
// --- H2, for a delivery ---------------------------------------------------------------------------------
// connOf is the bus a healer asks over.
func (h *healing) connOf() *nats.Conn {
if h.js == nil {
return nil
}
return h.js.Conn()
}
// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table
// lets H2 take; never one whose state is the operator's.
func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
lines, err := deliveriesStalled(ctx, h.connOf(), true)
if err != nil {
return "", false, "", err
}
for _, l := range lines {
if l.ID != c.Subject.ID {
continue
}
if l.operatorsOnly() {
return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil
}
return c.Key, true, "", nil
}
return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil
}
// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the
// transition its table names. A refusal is no repair, said; the next observation says whether it worked.
func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key})
if err != nil {
if errors.Is(err, link.ErrNothingServes) {
return "", "", err
}
return "closed nothing", oneLine(err.Error()), nil
}
var said string
if json.Unmarshal(raw, &said) != nil {
said = string(raw)
}
return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil
}
cmd/mesh-controller/delivery_conditions_test.go
-174
View File
@@ -1,174 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's
// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`.
// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records
// what close was asked.
func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string {
t.Helper()
var closed []string
was := askDeliveryOwner
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
switch verb {
case "stalled":
raw, _ := json.Marshal(lines)
return raw, nil
case "close":
closed = append(closed, args["id"].(string))
if closeErr != nil {
return nil, closeErr
}
return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil
}
t.Fatalf("asked the owner %s", verb)
return nil, nil
}
t.Cleanup(func() { askDeliveryOwner = was })
return &closed
}
func holdTheDeliverySeat(t *testing.T, open *stores) {
t.Helper()
m := catalogue.Manifest{Module: "mesh-delivery", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
}
var twoStalled = []stalledLine{
{ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s",
H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"},
{ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s",
H2: "none: the state is the operator's", Says: "it waits for the operator"},
}
func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
ownerAnswers(t, twoStalled, nil)
d := &doctor{open: open}
got, err := probeDeliveries(ctx, d)
if err != nil || len(got) != 0 {
t.Fatalf("with no holder on record D14 said %+v %v", got, err)
}
holdTheDeliverySeat(t, open)
got, err = probeDeliveries(ctx, d)
if err != nil || len(got) != 2 {
t.Fatalf("D14 said %+v %v", got, err)
}
if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning ||
got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") {
t.Fatalf("the first is %+v", got[0])
}
if got[1].Resolver != conditions.ResolverOperator {
t.Fatalf("a held delivery is not the operator's: %+v", got[1])
}
// The holder not running is D3's to say, not D14's.
was := askDeliveryOwner
askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) {
return nil, link.ErrNothingServes
}
defer func() { askDeliveryOwner = was }()
if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 {
t.Fatalf("an owner that is down was said by D14: %+v %v", got, err)
}
}
// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a
// refusal is no repair.
func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
closed := ownerAnswers(t, twoStalled, nil)
for _, o := range stalledObservations(twoStalled) {
o.Source = probeDeliveriesID
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
t.Fatal(err)
}
}
h.tick(ctx)
if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) {
t.Fatalf("close was asked for %v", *closed)
}
acts := told.said()
if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" {
t.Fatalf("said %+v", acts)
}
// A refusal: closed nothing, said so.
c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled")
ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering"))
act, said, err := repairDelivery(ctx, h, c)
if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") {
t.Fatalf("a refused close reads %q %q %v", act, said, err)
}
}
// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else
// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read.
func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
// And release and stop, which the operator's warrant chooses (novox/hq ADR 0259).
for _, verb := range []string{"stalled", "close", "release", "stop"} {
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
}
}
if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") {
t.Fatalf("a verb the grant does not name was asked: %v", err)
}
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := deliveriesStalled(t.Context(), conn, true); err != nil {
t.Fatalf("an owner not running is D3's, not an error: %v", err)
}
lines, _ := json.Marshal(twoStalled)
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}})
sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) {
body, _ := json.Marshal(link.Answer{Result: wrapped})
_ = m.Respond(body)
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = sub.Unsubscribe() }()
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
got, err := deliveriesStalled(ctx, conn, true)
if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" {
t.Fatalf("over the bus: %+v %v", got, err)
}
}
cmd/mesh-controller/delivery_edges.go
-170
View File
@@ -1,170 +0,0 @@
package main
import (
"fmt"
"slices"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
)
// The order of a plan's modules is read from the module graph and from what the change says of the manifests
// (novox/hq ADR 0419), per module and never per repository: no `after:` line is read, and no member of a group
// is ordered against another. Pure, so a test needs no store.
// The rules that draw an edge of a plan's order, in the words the plan shows.
const (
ruleVerbBeforeCaller = "verb before caller"
ruleVersionSkew = "version skew"
ruleEngineFirst = "engine before controller"
)
// The two modules the rollout rules name.
const (
theController = "mesh-controller"
theEngine = "mesh-host"
)
// moduleChange is what a change says of one module's manifest: the seat verbs ("<seat>.<verb>") it adds to what the
// module serves, the seat verbs it adds to what the module invokes, and whether the manifest changes at all.
type moduleChange struct {
Serves []string
Invokes []string
Manifest bool
}
// planOrder is the edges the rules draw among a plan's modules, and the cycle they make when they make one.
type planOrder struct {
Edges []inventory.Edge
// Cycle is every edge of a cycle among the order's edges and the graph's, the module graph's included.
Cycle []inventory.Edge
}
// CycleSaid names every module and edge of the cycle, for a check to fail with (ADR 0419 clause 6).
func (o planOrder) CycleSaid() string {
if len(o.Cycle) == 0 {
return ""
}
var parts []string
for _, e := range o.Cycle {
parts = append(parts, edgeSaid(e))
}
return "the modules of the plan are ordered in a cycle: " + strings.Join(parts, "; ")
}
// edgeSaid is an edge as a person reads it: the module after, the module before, and why.
func edgeSaid(e inventory.Edge) string {
why := e.Why
if why == "" {
why = e.Kind
}
return fmt.Sprintf("%s after %s (%s)", e.From, e.To, why)
}
// orders is whether an edge of the module graph orders a plan's tiers (tiersOf).
func orders(e inventory.Edge) bool {
return e.Kind != inventory.EdgePackages && e.Kind != inventory.EdgeBuiltBy && e.From != e.To
}
// orderEdges draws the edges of a plan's order among its moved modules (ADR 0419 clause 3, 4): a module whose change
// adds a verb to a seat goes before one whose change adds a call of that verb; the controller, when the plan moves
// it, goes before every module whose manifest the plan changes, the node-engine excepted; and the node-engine goes
// before the controller by default, when the plan moves both and no edge orders them either way, transitively. An
// edge is "From after To". graph is the module graph, whose edges order too and are never drawn again.
func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge) planOrder {
var out planOrder
sorted := append([]string(nil), moved...)
sort.Strings(sorted)
for _, server := range sorted {
for _, caller := range sorted {
if server == caller {
continue
}
for _, verb := range changes[server].Serves {
if slices.Contains(changes[caller].Invokes, verb) {
out.Edges = append(out.Edges, inventory.Edge{From: caller, To: server, Kind: edgeChangeOrder,
Why: ruleVerbBeforeCaller + ": " + verb})
}
}
}
}
if slices.Contains(sorted, theController) {
for _, m := range sorted {
if m != theController && m != theEngine && changes[m].Manifest {
out.Edges = append(out.Edges, inventory.Edge{From: m, To: theController, Kind: edgeChangeOrder,
Why: ruleVersionSkew})
}
}
}
if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) {
all := orderingEdges(graph, out.Edges)
switch {
case pathBetween(all, theController, theEngine) != nil:
// An edge already puts the controller after the engine: the default says the same.
case pathBetween(all, theEngine, theController) != nil:
// An edge puts the engine after the controller: it overrules the default, and says so.
path := pathBetween(all, theEngine, theController)
for i := range out.Edges {
if out.Edges[i] == path[0] {
out.Edges[i].Why += " (overrules " + ruleEngineFirst + ")"
break
}
}
default:
out.Edges = append(out.Edges, inventory.Edge{From: theController, To: theEngine, Kind: edgeChangeOrder,
Why: ruleEngineFirst})
}
}
all := orderingEdges(graph, out.Edges)
for _, e := range out.Edges {
if back := pathBetween(all, e.To, e.From); back != nil {
out.Cycle = append(out.Cycle, e)
out.Cycle = append(out.Cycle, back...)
break
}
}
return out
}
// orderingEdges is the graph's edges that order and the edges drawn, together.
func orderingEdges(graph, drawn []inventory.Edge) []inventory.Edge {
var out []inventory.Edge
for _, e := range graph {
if orders(e) {
out = append(out, e)
}
}
return append(out, drawn...)
}
// pathBetween is the edges that lead from a module to another along "depends on" (From to To), the shortest, or nil
// when the first does not depend on the second.
func pathBetween(edges []inventory.Edge, from, to string) []inventory.Edge {
if from == to {
return nil
}
via := map[string]inventory.Edge{}
seen := map[string]bool{from: true}
queue := []string{from}
for len(queue) > 0 {
at := queue[0]
queue = queue[1:]
for _, e := range edges {
if e.From != at || seen[e.To] {
continue
}
seen[e.To] = true
via[e.To] = e
if e.To == to {
var path []inventory.Edge
for m := to; m != from; m = via[m].From {
path = append([]inventory.Edge{via[m]}, path...)
}
return path
}
queue = append(queue, e.To)
}
}
return nil
}
cmd/mesh-controller/delivery_edges_test.go
-183
View File
@@ -1,183 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// The order of a plan's modules is read from the module graph and the manifests' change alone (novox/hq ADR 0419):
// no `after:` line, no member of a group. Pure, like orderOf was.
func hasEdge(edges []inventory.Edge, from, to string) (inventory.Edge, bool) {
for _, e := range edges {
if e.From == from && e.To == to {
return e, true
}
}
return inventory.Edge{}, false
}
// A module whose change adds a verb to a seat goes before the module whose change adds a call of it: the caller
// depends on the server, the plan puts them in two tiers, and the edge says why.
func TestAVerbIsServedBeforeItsCallerIsSent(t *testing.T) {
changes := map[string]moduleChange{
"server": {Serves: []string{"mesh-delivery.groups"}, Manifest: true},
"caller": {Invokes: []string{"mesh-delivery.groups"}, Manifest: true},
}
got := orderEdges([]string{"caller", "server"}, changes, nil)
e, ok := hasEdge(got.Edges, "caller", "server")
if !ok || !strings.Contains(e.Why, "verb before caller") || !strings.Contains(e.Why, "mesh-delivery.groups") {
t.Fatalf("the caller does not follow the server by the verb: %+v", got.Edges)
}
if len(got.Cycle) != 0 {
t.Fatalf("a cycle: %+v", got.Cycle)
}
if tiers := tiersOf([]string{"caller", "server"}, got.Edges); len(tiers) != 2 || tiers[0][0] != "server" {
t.Fatalf("tiers %v, want the server first", tiers)
}
}
// Edges of one rule both ways are a cycle: every module and edge is named.
func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) {
changes := map[string]moduleChange{
"x": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}},
"y": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}},
}
got := orderEdges([]string{"x", "y"}, changes, nil)
if len(got.Cycle) < 2 {
t.Fatalf("no cycle found: %+v", got)
}
said := got.CycleSaid()
for _, want := range []string{"x", "y", "s.a", "s.b"} {
if !strings.Contains(said, want) {
t.Fatalf("the cycle is said as %q, without %s", said, want)
}
}
}
// The node-engine goes before the controller by default, and says so; an edge of the manifests the other way
// overrules it, and the edge that did is said.
func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T) {
got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil)
e, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host")
if !ok || e.Why != "engine before controller" {
t.Fatalf("the engine does not go first: %+v", got.Edges)
}
changes := map[string]moduleChange{
"mesh-host": {Serves: []string{"s.v"}},
"mesh-controller": {Invokes: []string{"s.v"}},
}
got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
if _, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host"); !ok || len(got.Edges) != 1 ||
!strings.Contains(got.Edges[0].Why, "verb before caller") {
t.Fatalf("the verb edge does not place the controller after the engine: %+v", got.Edges)
}
// The other way: the engine calls what the controller brings.
changes = map[string]moduleChange{
"mesh-controller": {Serves: []string{"s.v"}},
"mesh-host": {Invokes: []string{"s.v"}},
}
got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
e, ok = hasEdge(got.Edges, "mesh-host", "mesh-controller")
if !ok || len(got.Edges) != 1 || !strings.Contains(e.Why, "overrules engine before controller") {
t.Fatalf("the verb edge does not say it overruled the default: %+v", got.Edges)
}
}
// The controller goes before every module whose manifest the plan changes (version skew), the node-engine excepted.
func TestTheControllerGoesBeforeEveryChangedManifest(t *testing.T) {
changes := map[string]moduleChange{"app": {Manifest: true}, "other": {}}
got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil)
if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" {
t.Fatalf("the changed manifest does not follow the controller: %+v", got.Edges)
}
if _, ok := hasEdge(got.Edges, "other", "mesh-controller"); ok {
t.Fatalf("a module with an unchanged manifest was ordered: %+v", got.Edges)
}
if got := orderEdges([]string{"app", "other"}, changes, nil); len(got.Edges) != 0 {
t.Fatalf("version skew ordered a plan that does not move the controller: %+v", got.Edges)
}
}
// The order is per module, not per repository: of two modules of one repository only the one with the edge moves.
func TestTheOrderIsPerModuleNotPerRepository(t *testing.T) {
changes := map[string]moduleChange{
"a": {Serves: []string{"s.v"}},
"c": {Invokes: []string{"s.v"}},
}
got := orderEdges([]string{"a", "b", "c"}, changes, nil)
tiers := tiersOf([]string{"a", "b", "c"}, got.Edges)
if len(tiers) != 2 || strings.Join(tiers[0], ",") != "a,b" || strings.Join(tiers[1], ",") != "c" {
t.Fatalf("tiers %v, want [a b] then [c]", tiers)
}
}
// The edges a plan's order was drawn with are kept on the plan, and its gates read them: a later tier waits for the
// earlier to be applied although the catalogue's graph holds no such edge (ADR 0419, the lead's settlement Q4).
func TestAPlansOwnOrderGatesItsTiersWhateverTheStoreGraphHolds(t *testing.T) {
p := inventory.Plan{Tiers: [][]string{{"server"}, {"caller"}}, Tier: 0,
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "caller", To: "server", Kind: edgeChangeOrder,
Why: ruleVerbBeforeCaller + ": s.v"}}}}
if got := gates(p, nil, func(string) bool { return true }); len(got) != 1 || got[0] != "server" {
t.Fatalf("the caller's tier waits for %v; want the server", got)
}
p.Delivery.Order = nil
if got := gates(p, nil, func(string) bool { return true }); len(got) != 0 {
t.Fatalf("a plan with no order waits for %v", got)
}
}
// A verb an earlier merge of the same repository in the batch added is added by the batch: the change is compared with
// what the repository held before the batch, not with the first parent of its latest merge.
func TestAVerbAnEarlierMergeOfTheBatchAddedDrawsItsEdge(t *testing.T) {
was, wasParent := manifestAt, parentOf
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
manifestsReadMu.Lock()
clear(manifestsRead)
manifestsReadMu.Unlock()
const first, second, before = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "0000000000000000000000000000000000000000"
parentOf = func(_ context.Context, _, _, commit string) (string, error) {
if commit == first {
return before, nil
}
return first, nil // the latest merge's parent holds the verb already
}
manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) {
m := catalogue.Manifest{Module: "server"}
if ref != before {
m.DefinesSeats = []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "v"}}}}
}
return m, true, nil
}
src := changeSource{Owner: "novox", Repo: "r", New: second, Paths: []string{"module.json"}}
if got, err := changesOf(context.Background(), []changeSource{src}); err != nil || len(got["server"].Serves) != 0 {
t.Fatalf("against the first parent of the latest merge: %+v %v", got, err)
}
src.Since = first
got, err := changesOf(context.Background(), []changeSource{src})
if err != nil || len(got["server"].Serves) != 1 || got["server"].Serves[0] != "s.v" {
t.Fatalf("against what the batch found: %+v %v", got, err)
}
}
// A change that names only a beginning of its files is read as changing the manifests of the modules it moves.
func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T) {
entries := []inventory.Entry{
{Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}},
{Manifest: catalogue.Manifest{Module: "b"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/b"}},
{Manifest: catalogue.Manifest{Module: "c"}, Source: inventory.Source{Repository: "novox/other", Path: "modules/c"}},
}
got := truncatedPaths(entries, "novox", "mesh-catalog", []string{"a", "c"})
if len(got) != 1 || got[0] != "modules/a/module.json" {
t.Fatalf("read as changed: %v", got)
}
members := []orderMember{{ID: "m", Repository: "novox/mesh-catalog", Head: "h", PathsTruncated: true, Paths: []string{"x"}}}
src := changeSourcesOf(members, entries, map[string]orderReach{"m": {Moved: []string{"a"}}})
if len(src) != 1 || src[0].Truncated || len(src[0].Paths) != 1 || src[0].Paths[0] != "modules/a/module.json" {
t.Fatalf("the member's change is read as %+v", src)
}
}
cmd/mesh-controller/delivery_integration_test.go
-216
View File
@@ -1,216 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := asksRecorded(t)
withConditionsInMemory(t)
for _, name := range []string{"app", "mesh-delivery"} {
m := catalogue.Manifest{Module: name, Version: "1"}
if name == "mesh-delivery" {
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
}
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
merge := func(commit string, paths ...string) inventory.Plan {
t.Helper()
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
recent, err := inv.RecentPlans(ctx, 1)
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
}
return recent[0]
}
// finish ends a walk as done, so the next merge has nothing of it to take over.
finish := func(id string) {
t.Helper()
w, err := inv.PlanByID(ctx, id)
if err != nil {
t.Fatal(err)
}
w.State = inventory.PlanDone
if err := inv.SavePlan(ctx, &w); err != nil {
t.Fatal(err)
}
}
// heardOf has this mesh's own view of the plans take in the plan as the bucket holds it. A verb of the
// controller (deliveryCommand, plansCommand) opens an inventory of its own and writes the plan there; this
// mesh's inventory learns of the write from the bucket's watch, a moment later, and a loaded machine makes the
// moment long enough for advanceHeld to read the plan as it was, still waiting for its word, and ask nothing
// (issue 1399). Reading the plan from the bucket puts it into the view at once, with no wait to bound.
heardOf := func(id string) {
t.Helper()
if _, err := inv.PlanByID(ctx, id); err != nil {
t.Fatal(err)
}
}
// No holder on record: the merge starts its walk, as before.
// A cut takes its first tier from the builder's keys and a step of the walk records the take (the walk asks no
// build), so each walk that starts is stepped once before its takes are counted.
p := merge("c1aaaaaaaa", "modules/app/index.ts")
advanceHeld(ctx, open)
if p.Waiting() || len(*asked) != 1 {
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
}
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
finish(p.ID)
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
p = merge("c2bbbbbbbb", "modules/app/index.ts")
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
}
advanceHeld(ctx, open)
if len(*asked) != 1 {
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
}
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
t.Errorf("a waiting walk reads %q", line)
}
// The delivery's word starts it.
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
t.Fatal(err)
}
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
t.Fatal("a walk was let go twice")
}
heardOf(p.ID)
advanceHeld(ctx, open)
if len(*asked) != 2 {
t.Fatalf("the word did not start the walk: %v", *asked)
}
got, err := inv.PlanByID(ctx, p.ID)
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
}
// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
// ADR 0276): the merge's batch is cut once it ended.
finish(p.ID)
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
advanceHeld(ctx, open)
if p.Waiting() || len(*asked) != 3 {
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
}
// The owner down: a person starts a waiting walk, with why.
finish(p.ID)
p = merge("c4dddddddd", "modules/app/index.ts")
if !p.Waiting() {
t.Fatal("the walk did not wait")
}
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
t.Fatalf("a walk was started by hand without why: %v", err)
}
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
t.Fatal(err)
}
heardOf(p.ID)
advanceHeld(ctx, open)
got, _ = inv.PlanByID(ctx, p.ID)
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
t.Fatalf("a person's word did not start the walk: %+v, asked %v, state %q, tier %d, note %q", got.Delivery, *asked, got.State, got.Tier, got.Note)
}
// The owner's stop: failed, said as stopped by it.
finish(p.ID)
p = merge("c5eeeeeeee", "modules/app/index.ts")
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
"mesh-delivery for jochen"}); err != nil {
t.Fatal(err)
}
got, _ = inv.PlanByID(ctx, p.ID)
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
!strings.Contains(got.Note, "the operator stopped it") {
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
}
// Unassigned: the mesh is back on the controller's own path.
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
asks := len(*asked)
p = merge("c6ffffffff", "modules/app/index.ts")
advanceHeld(ctx, open)
if p.Waiting() || len(*asked) != asks+1 {
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
}
}
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
before := handActConn
handActConn = js.Conn()
t.Cleanup(func() { handActConn = before })
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {}},
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
t.Fatal(err)
}
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
var said inventory.Plan
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
said.Delivery.Go == nil {
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
}
case <-time.After(5 * time.Second):
t.Fatal("a walk let go by a verb was not said")
}
if checkEvents != nil || inventory.PlanSaved != nil {
t.Fatal("the command's own bus was left in place")
}
}
cmd/mesh-controller/delivery_order_test.go
-152
View File
@@ -1,152 +0,0 @@
package main
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
type recordingDelivery struct {
did []string
grantErr error
declareErr error
}
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
for _, s := range sending {
r.did = append(r.did, "grant "+s.node)
}
return r.grantErr
}
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
if r.declareErr != nil {
return "", r.declareErr
}
r.did = append(r.did, "declare "+s.node)
return "digest-" + s.node, nil
}
func ready(names ...string) []readyNode {
var out []readyNode
for _, n := range names {
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
}
return out
}
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
// carries the controller's own right to issue them, and goes first.
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
d := &recordingDelivery{}
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
if err != nil {
t.Fatal(err)
}
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
if !reflect.DeepEqual(d.did, want) {
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
}
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
t.Fatalf("the digests sent were not answered: %v", digests)
}
held := &recordingDelivery{}
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
t.Fatal(err)
}
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
if !reflect.DeepEqual(held.did, want) {
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
}
}
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
// grant that would let the controller issue memberships is never held behind them.
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
// A failure naming no machine (the buckets): everything but the bus's machine.
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
!strings.Contains(err.Error(), "anchor, laptop not sent") {
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
}
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
}
// A membership that failed for one machine: that machine alone.
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
t.Fatalf("one machine's refused membership was not said: %v", err)
}
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
}
// The announced upgrade that hit it is asked again.
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
t.Fatal("an announcement whose send stopped at its grants is not asked again")
}
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
t.Fatal("any failure is asked again, not only a grant's")
}
// Nothing to send is nothing granted either.
none := &recordingDelivery{grantErr: errors.New("never asked")}
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
}
}
// Whether the bus's machine goes first is read from the user list alone, by its digest.
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
list := "users: [a, b]"
if userListBehind(list, digestOf([]byte(list))) {
t.Fatal("the list it was sent reads as behind")
}
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
t.Fatal("a changed or never-sent list reads as current")
}
if userListBehind("", "") {
t.Fatal("a machine sent no list reads as behind")
}
// A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353).
if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) {
t.Fatal("a list nobody could compose reads as current")
}
if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") {
t.Fatal("listBehind does not read the digest as userListBehind does")
}
}
// The machine holding the bus goes first: its declaration carries the user list the new grants are
// checked against. Among the machines it is moved to the front; not among them it is added only
// when it is behind.
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
for _, c := range []struct {
what string
names []string
holder string
behind bool
want []string
}{
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
} {
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
}
}
}
cmd/mesh-controller/delivery_stalled_severity_test.go
-37
View File
@@ -1,37 +0,0 @@
package main
import (
"encoding/json"
"testing"
"github.com/novox/mesh-controller/internal/conditions"
)
func severityRaisedFor(t *testing.T, line string) conditions.Severity {
t.Helper()
var l stalledLine
if err := json.Unmarshal([]byte(line), &l); err != nil {
t.Fatal(err)
}
got := stalledObservations([]stalledLine{l})
if len(got) != 1 {
t.Fatalf("%d conditions", len(got))
}
return got[0].Severity
}
func TestAnUrgentStalledLineRaisesAnUrgentCondition(t *testing.T) { // 1278/AC1
line := `{"id":"novox/mesh-delivery#5","state":"held","for":"3m0s","bound":"0s","h2":"close","says":"held","severity":"urgent"}`
if got := severityRaisedFor(t, line); got != conditions.Urgent {
t.Errorf("an urgent line raised %q", got)
}
}
func TestAStalledLineWithoutOrWithAnUnknownSeverityRaisesAWarning(t *testing.T) { // 1278/AC2
for _, extra := range []string{``, `,"severity":""`, `,"severity":"critical"`, `,"severity":"warning"`} {
line := `{"id":"novox/a#1","state":"unanswered","for":"3m0s","bound":"2m0s","h2":"close","says":"x"` + extra + `}`
if got := severityRaisedFor(t, line); got != conditions.Warning {
t.Errorf("the line %s raised %q", extra, got)
}
}
}
cmd/mesh-controller/delivery_test.go
-155
View File
@@ -1,155 +0,0 @@
package main
import (
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq ADR 0419: a group's order, from the module graph and the manifests' change alone, the same on every
// reading — what a module stands on before it, the controller before a manifest that needs it, the node-engine
// before the controller, a verb's server before its caller — and no `after:` line read.
func TestAGroupIsOrderedByTheGraphAndTheManifestsChange(t *testing.T) {
members := []orderMember{
{ID: "cat", Repository: "novox/mesh-catalog"},
{ID: "ctl", Repository: "novox/mesh-controller"},
{ID: "host", Repository: "novox/mesh-host"},
{ID: "base", Repository: "novox/base"},
{ID: "app", Repository: "novox/app"},
{ID: "lab", Repository: "novox/lab"},
}
reach := map[string]orderReach{
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
"base": {Moved: []string{"base"}},
"app": {Moved: []string{"app"}},
"lab": {Moved: []string{"lab"}},
}
edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeStandsOn},
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
changes := map[string]moduleChange{"gitea": {Manifest: true}, "mesh-controller": {Manifest: true},
"mesh-host": {Manifest: true}, "lab": {Serves: []string{"seat.v"}}, "app": {Invokes: []string{"seat.v"}}}
got := orderOf(members, reach, edges, changes)
if len(got.Cycle) > 0 {
t.Fatalf("a cycle where there is none: %v", got.Cycle)
}
// By repository among those with nothing before them: app waits for base and lab; gitea's catalogue for the controller.
want := []string{"base", "lab", "app", "host", "ctl", "cat"}
if !reflect.DeepEqual(got.Order, want) {
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
}
why := map[string]string{}
for _, p := range got.Pairs {
why[p.Before+">"+p.After] = p.Why
}
for pair, reason := range map[string]string{"host>ctl": ruleEngineFirst, "ctl>cat": ruleVersionSkew,
"ctl>host": "", "base>app": orderBuiltBy, "lab>app": ruleVerbBeforeCaller + ": seat.v"} {
if why[pair] != reason {
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
}
}
// The same members in another order read the same.
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
if again := orderOf(shuffled, reach, edges, changes); !reflect.DeepEqual(again.Order, want) {
t.Fatalf("another reading ordered %v", again.Order)
}
}
// A cycle is the one refusal, and it names every module and edge. A description's `after:` line changes nothing.
func TestAGroupInACycleIsNamedAndNothingElseIsRefused(t *testing.T) {
members := []orderMember{
{ID: "x", Repository: "novox/x", Number: 7},
{ID: "y", Repository: "novox/y", Number: 9},
}
reach := map[string]orderReach{"x": {Moved: []string{"mx"}}, "y": {Moved: []string{"my"}}}
changes := map[string]moduleChange{
"mx": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}},
"my": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}},
}
got := orderOf(members, reach, nil, changes)
if !reflect.DeepEqual(got.Cycle, []string{"x", "y"}) || len(got.Order) > 0 {
t.Fatalf("a cycle was ordered: %v, cycle %v", got.Order, got.Cycle)
}
for _, w := range []string{"mx", "my", "s.a", "s.b"} {
if !strings.Contains(got.CycleSaid, w) {
t.Errorf("the cycle does not say %q: %s", w, got.CycleSaid)
}
}
// Without the edges the same members are ordered by repository.
got = orderOf(members, reach, nil, nil)
if !reflect.DeepEqual(got.Order, []string{"x", "y"}) || len(got.Cycle) > 0 || len(got.Pairs) > 0 {
t.Fatalf("members with no edge: %v %v %+v", got.Order, got.Cycle, got.Pairs)
}
}
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
// record, and never one that moves a module on the controller's own path.
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
unassigned := holder
unassigned.On = nil
for _, c := range []struct {
entries []inventory.Entry
moved []string
waits bool
}{
{nil, []string{"gitea"}, false},
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
{[]inventory.Entry{holder}, []string{"nats"}, false},
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
} {
d := awaitsFor(c.entries, c.moved)
if (d != nil) != c.waits {
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
c.moved, d != nil, c.waits)
}
if d != nil && d.Awaits != catalogue.DeliverySeat {
t.Errorf("waits for %q", d.Awaits)
}
}
}
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
// hand when the owner is down, and the owner's stop ends one as stopped.
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want []string
}{
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
} {
got, err := argvFor(c.verb, c.args)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
}
}
if _, err := argvFor("deliver", map[string]any{}); err == nil {
t.Error("deliver without a walk was composed")
}
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
t.Error("a stop without why was composed")
}
}
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
cmd/mesh-controller/doctor.go
-32
View File
@@ -115,26 +115,11 @@ var probeRegistry = []probe{
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
Phase: 2, run: probeData,
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
// The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by
// the controller, and H2 works it through the owner's `close`.
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
// What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a
// node's presence, which the silent check takes.
{ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " +
"not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156",
Kind: kindStuck, Phase: 3, run: probeStuck},
// A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the
// operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8).
{ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " +
"minutes for mesh-delivery's announcement of the operator's Approve of its head", From: "ADR 0432, issue 1146",
Kind: kindFirstBuildUnasked, Phase: 3, run: probeFirstBuilds},
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
// connections, which the bus's own module reads.
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
@@ -168,30 +153,13 @@ var probeRegistry = []probe{
{ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " +
"ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8",
Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth},
{ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " +
"build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth},
{ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus",
From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth},
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " +
"it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4,
run: probeBusHealth},
// The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its
// condition until a newer build passes; each rollback a witness stands by is said.
{ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " +
"a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack,
Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates},
// The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus.
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
// A delivery plan held for a backup that failed or stayed silent (ADR 0404 decision 4): one condition per plan.
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
run: probeBackupHeld},
// A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it.
{ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " +
"bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1,
run: probeSendKeyRefused},
}
// probeVerdict is one probe's outcome in a run.
cmd/mesh-controller/epoch_send_test.go
-109
View File
@@ -1,109 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
type bodiesDelivery struct {
recordedDelivery
bodies map[string][]byte
}
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
b.bodies[s.node] = body
return b.recordedDelivery.declare(ctx, s, body)
}
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
epoch := uint64(57)
was := epochForActs
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
t.Cleanup(func() { epochForActs = was })
anchor, err := inv.NodeByName(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
t.Fatal(err)
}
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open), d, "", nil); err != nil {
t.Fatal(err)
}
carried := func(node string) (epoch float64, has bool) {
var envelope map[string]any
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
t.Fatal(err)
}
epoch, has = envelope["epoch"].(float64)
return epoch, has
}
if e, has := carried("anchor"); !has || e != 57 {
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
}
if _, has := carried("laptop"); has {
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
}
// A new holder of the lease is not a change of the machine: neither reads as behind.
epoch = 58
would, err := wouldSend(ctx, open, mustNodes(t, open))
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
sent, err := inv.Outstanding(ctx, node)
if err != nil {
t.Fatal(err)
}
if would[node] != sent {
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
}
}
}
// A process that may not act composes nothing and sends nothing: its number is not taken.
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
was := epochForActs
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
t.Cleanup(func() { epochForActs = was })
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open), d, "", nil)
if err != nil {
t.Fatal(err)
}
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
}
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
t.Fatalf("a controller without the lease took sequence %d", seq)
}
// And at the send itself: the gate every declaration passes.
gate := link.ActingGate
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
t.Cleanup(func() { link.ActingGate = gate })
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
!strings.Contains(err.Error(), "lost the lease") {
t.Fatalf("a declaration was let through the gate: %v", err)
}
}
cmd/mesh-controller/failed_wait_provider_test.go
-68
View File
@@ -1,68 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450).
//
// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy
// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the
// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never
// listed on the condition that is open.
// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check.
var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"}
// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why.
func judgedRefused() inventory.ResourceHealth {
r := waitingDatabaseFor(refusedWait)
r.State, r.Waits = link.StateUnhealthy, nil
r.Reason = "says it waits for the secret certificate, which db does not declare"
return r
}
// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list
// it at the provider's unhealthy condition, made urgent by who waits on it.
func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) {
k, _ := withConditionsInMemory(t)
stored := waitingDatabaseFor(refusedWait)
open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()},
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}},
shopFailingBeside(stored))
provider := conditionOf(open, moduleUnhealthyKey("db", "anchor"))
if len(open) != 1 || provider == nil {
t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+
"unhealthy alone", openKeysOf(open))
}
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said)
}
if provider.Severity != conditions.Urgent {
t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity)
}
}
// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens:
// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when
// open, is said as unhealthy with who waits on it.
func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) {
given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase)
given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)}
by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
if !held || by.provider != theDatabase || len(by.waits) > 0 {
t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+
"held under db on anchor as unhealthy, no waits", held, by.provider, by.waits)
}
if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered {
t.Fatalf("a provider whose wait failed its check is said as waiting for another part")
}
}
cmd/mesh-controller/first_builds.go
-277
View File
@@ -1,277 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
"strings"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// firstBuilds asks the first build of every module a merge adds to a repository no registered module is built from, when
// the merge was announced as made on the operator's Approve of that head (novox/hq ADR 0432, issue 1146): the modules'
// directories are the ones the announcement says hold a module.json at the merge commit, as touchedBy reads them. The
// merge is kept, so the build's outcome is judged by the rule that registers it (mayRegisterFrom, approvedMove).
//
// **Complete in either order, with no timer** (ADR 0426): when mesh-delivery's word is kept first, it is found here; when
// the forge's merge is heard first, the merge is kept as a first merge waiting for the word, and keepingDeliveryMerges
// runs this again when the word is kept (resumeFirstBuilds). A word that says no (a Decline, another head) asks nothing and
// says why. Answers whether the merge was dealt with here.
func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository, now time.Time) (bool, error) {
inv := f.open.inventory
repository := m.Owner + "/" + m.Repo
added, why, err := firstBuildsOf(ctx, inv, m, entries, read)
switch {
case errors.Is(err, errFirstNotApproved):
fmt.Printf("%s merged into %s (%.8s) adds %s, and no build is asked: %v\n", repository, m.Base, m.Commit,
strings.Join(addedDirs(m, entries, read), ", "), err)
return true, nil
case errors.Is(err, errNotAnnounced):
// Kept, waiting for mesh-delivery's word: a restart keeps it, and the word's arrival asks the builds.
if kerr := keepFirstMerge(ctx, inv, m, now, true); kerr != nil {
return true, notNow(kerr)
}
fmt.Printf("%s merged into %s (%.8s) adds %s; it is kept, and the first build is asked when mesh-delivery "+
"announces the merge on the operator's Approve\n", repository, m.Base, m.Commit,
strings.Join(addedDirs(m, entries, read), ", "))
return true, nil
case err != nil:
return true, notNow(err)
case len(added) == 0:
return false, nil
}
if err := askFirstBuilds(ctx, f.open, m, now, added, why); err != nil {
return true, notNow(err)
}
return true, nil
}
// keepFirstMerge keeps a first merge as the controller heard it, waiting (no plan answers it).
func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, waitingForWord bool) error {
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
if err != nil {
merged = now
}
event, err := json.Marshal(keptMerge{SourceMoved: m, FirstBuild: waitingForWord})
if err != nil {
return err
}
_, err = inv.AddMerge(ctx, inventory.BatchedMerge{Repository: m.Owner + "/" + m.Repo, Branch: m.Base, Commit: m.Commit,
Merged: merged.UTC(), Heard: now, Event: event})
return err
}
// askFirstBuilds keeps the merge as answered and registers its modules from the builder's done keys (novox/hq ADR 0501
// §3, §6): nothing asks their first builds. The merge kept is the registration's warrant (approvedMove reads it by the
// commit the indexer's trunk value names); a module whose key is not done yet is registered when the key's write wakes
// the take-in (new_modules.go).
func askFirstBuilds(ctx context.Context, open *stores, m link.SourceMoved, now time.Time, added []string,
why string) error {
inv := open.inventory
repository := m.Owner + "/" + m.Repo
if err := keepFirstMerge(ctx, inv, m, now, false); err != nil {
return err
}
fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so %s "+
"are registered from the builder's done keys, asking no build\n", repository, m.Base, m.Commit, why,
strings.Join(added, ", "))
// Answered: it no longer waits, so the word's arrival again does nothing twice.
if err := inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false); err != nil {
return err
}
modules, err := newModulesOf(ctx, repository)
switch {
case err != nil:
// Said, not failed: the merge is answered, and the look at every module when a controller starts, or the next
// write of the builder or the indexer for one of them, registers them.
fmt.Printf(" the indexer's modules of %s cannot be read now (%v): they are registered when the builder's next "+
"write of one wakes the take-in\n", repository, err)
case len(modules) > 0:
inBackground(func() { takeNewModules(context.WithoutCancel(ctx), open, modules...) })
}
return nil
}
// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull
// request's head: the other order of the same two events.
func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error {
actingOnMerges.Lock()
defer actingOnMerges.Unlock()
inv := f.open.inventory
waiting, err := inv.WaitingMerges(ctx)
if err != nil {
return notNow(err)
}
for _, w := range waiting {
k := announcedOf(w)
if !k.FirstBuild || len(k.Held) > 0 || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
k.HeadSHA != d.HeadSHA {
continue
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return notNow(err)
}
read, err := readForPlanning(ctx, inv)
if err != nil {
return notNow(err)
}
added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read)
switch {
case errors.Is(err, errFirstNotApproved):
fmt.Printf("%s/%s merged into %s (%.8s) was waiting for mesh-delivery's word, and no build is asked: %v\n",
k.Owner, k.Repo, k.Base, k.Commit, err)
if aerr := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); aerr != nil {
return notNow(aerr)
}
case err != nil:
return notNow(err)
case len(added) > 0:
if err := askFirstBuilds(ctx, f.open, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
return notNow(err)
}
}
}
return nil
}
// errFirstNotApproved says a merge into a repository the mesh builds nothing from is not one the operator approved.
var errFirstNotApproved = errors.New("it is no merge made on the operator's Approve of its head, so a hand `build` " +
"registers a module from a repository the mesh does not build yet")
// addedDirs are the module directories the merge adds to a repository no registered module is built from, none where
// the merge is not into the trunk or some module is built from the repository already.
func addedDirs(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []string {
if m.Base != "main" {
return nil
}
for _, e := range entries {
if !e.Provided && sameRepository(e.Source.Repository, m) {
return nil
}
}
_, added, _ := touchedBy(nil, entries, m, read)
return added
}
// firstBuildsOf is the directories whose first build a merge asks, and the warrant it was made on; none when the merge
// is no first merge into a repository the mesh builds nothing from. errFirstNotApproved when it is and the word on record
// is no Approve of its head; errNotAnnounced when mesh-delivery's word is not kept yet.
func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) ([]string, string, error) {
added := addedDirs(m, entries, read)
if len(added) == 0 {
return nil, "", nil
}
why, err := approvedAnnounced(ctx, inv, m.Owner, m.Repo, keptMerge{SourceMoved: m})
switch {
case err == nil:
return added, why, nil
case errors.Is(err, errWordNotAnApprove):
return nil, "", fmt.Errorf("%w (%v)", errFirstNotApproved, err)
}
// Not announced yet, or the bus or the store could not be asked: the caller waits or tries again, and asks nothing.
return nil, "", err
}
// heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read,
// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173). Read from a merge kept
// before R4e, never written since: nothing asks a first build (ADR 0501 §6), and a controller that starts answers such a
// merge and registers its modules from their keys (answerHeldMerges).
type heldFirstBuild struct {
Module string `json:"module"`
Path string `json:"path"`
Word string `json:"word"`
At time.Time `json:"at"`
}
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
// refusedWord is the word a build's failure says the reader did not know; empty when it says none.
func refusedWord(failed string) string {
if m := unknownWord.FindStringSubmatch(failed); m != nil {
return m[1]
}
return ""
}
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
const probeFirstBuildsID = "D18"
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
const kindFirstBuildUnasked = "first-build-unasked"
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
const firstBuildWaitBound = 10 * time.Minute
// firstBuildObservations are the conditions of the first merges waiting past the bound, one for each; a merge answered is
// no longer listed as waiting, so its condition clears with the next look.
func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []conditions.Observation {
var out []conditions.Observation
for _, w := range waiting {
k := announcedOf(w)
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
continue
}
repository := k.Owner + "/" + k.Repo
for_ := now.Sub(w.Heard).Round(time.Minute)
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge,
ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), Kind: kindFirstBuildUnasked, Severity: conditions.Warning,
Summary: fmt.Sprintf("the merge %.8s of %s adds modules the mesh builds nothing from yet, and has waited %s for "+
"mesh-delivery's announcement of the operator's Approve of its head: no build is asked", k.Commit, repository, for_),
Said: fmt.Sprintf("%s at %s waits for mesh-delivery's word on pull request %d (head %.8s)", repository, k.Commit,
k.Number, k.HeadSHA),
Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository),
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+
"only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+
"announcement has come in %s. The merge is kept, and the modules are registered from the builder's keys the moment "+
"the announcement is kept.",
k.Commit, repository, for_),
Resolved: "mesh-delivery announces the merge, or the operator registers the modules at the controller's terminal",
})
}
return out
}
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
waiting, err := d.open.inventory.WaitingMerges(ctx)
if err != nil {
return nil, err
}
now := time.Now().UTC()
return append(firstBuildObservations(waiting, now), newModuleObservations(now)...), nil
}
// answerHeldMerges answers every merge a controller of before held for a build-agent that reads its new modules'
// manifests (issue 1173): nothing asks their first builds any more, and the look at every module when this controller
// starts registers them from their keys (ADR 0501 §6, §7).
func answerHeldMerges(ctx context.Context, inv *inventory.Inventory) {
waiting, err := inv.WaitingMerges(ctx)
if err != nil {
fmt.Printf("the merges held for a newer build-agent could not be read: %v\n", err)
return
}
for _, w := range waiting {
if k := announcedOf(w); len(k.Held) > 0 {
if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil {
fmt.Printf("the merge %.8s of %s held for a newer build-agent could not be answered: %v\n", w.Commit,
w.Repository, err)
continue
}
fmt.Printf("the merge %.8s of %s was held for a newer build-agent; its modules are registered from the "+
"builder's keys instead\n", w.Commit, w.Repository)
}
}
}
cmd/mesh-controller/first_builds_integration_test.go
-152
View File
@@ -1,152 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// 1146/AC7 on keys (ADR 0501 §3, §6; issue 1851, A5), whole against a real bus and the store, in both orders of the two
// events: mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's
// merge, heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the
// controller consumes with the router's record of the ask in its bucket, come in either order; each way the module the
// merge commit holds is registered from the builder's done key of it, and nothing asks a build. Without the Approve in
// the record it is not registered. Store-backed: skips without one.
func TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey(t *testing.T) {
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
for _, c := range []struct {
what string
wordFirst bool
approved bool // the router's record holds the operator's Approve of the merged head
wantRegister bool
}{
{what: "the forge's merge before mesh-delivery's word", approved: true, wantRegister: true},
{what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegister: true},
{what: "a word whose record holds no Approve", wordFirst: true},
} {
t.Run(c.what, func(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
noBuildAsked(t)
b := aBuilderOnTheBus(t)
js := b.js
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
if err != nil {
t.Fatal(err)
}
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
if c.approved {
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil {
t.Fatal(err)
}
}
// The indexer's value and the builder's done key of the module the merge commit holds.
fp := fpOf('9')
manifest := b.indexed(t, "mesh-delivery-view", "1", "novox/mesh-delivery", ".", commit, fp)
b.built(t, "mesh-delivery-view", "1", fp, manifest, sdkdelivery.BuildDone)
following := following{open: open}
server := link.ConnectNats(js, nil, noListener{})
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims, kept: following.resumeFirstBuilds}); err != nil {
t.Fatal(err)
}
if err := broker.AssertServingConsumers(js); err != nil {
t.Fatal(err)
}
serving, stop := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { defer close(done); _ = server.Serve(serving) }()
t.Cleanup(func() { stop(); <-done })
word := func() {
t.Helper()
body, _ := json.Marshal(a.merge)
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
t.Fatal(err)
}
becomes(t, 15*time.Second, "the word being kept", func() bool {
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, head)
return err == nil && kept
})
time.Sleep(500 * time.Millisecond) // the keeper's hook runs as the word is kept
}
forge := func() {
t.Helper()
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
MergedAt: time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
if err := following.SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
}
if c.wordFirst {
word()
forge()
} else {
forge()
time.Sleep(300 * time.Millisecond)
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known {
t.Fatal("1146/AC7: registered before mesh-delivery's word was kept")
}
word()
}
if !c.wantRegister {
time.Sleep(time.Second)
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known {
t.Fatal("1146/AC7: a merge whose record holds no Approve registered its module")
}
return
}
becomes(t, 10*time.Second, "the module registered", func() bool {
_, known := inventoryHolds(t, inv, "mesh-delivery-view")
return known
})
src, err := inv.SourceOf(ctx, "mesh-delivery-view")
if err != nil || src.Repository != "novox/mesh-delivery" {
t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err)
}
})
}
}
func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) {
t.Helper()
entries, err := inv.Catalogued(t.Context())
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
if e.Manifest.Module == module {
return e, true
}
}
return inventory.Entry{}, false
}
cmd/mesh-controller/first_builds_test.go
-106
View File
@@ -1,106 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// 1146/AC6: the first merge into a repository no registered module is built from asks the first build of every module
// the merge commit holds when it was made on the operator's Approve of that head, and asks nothing without it.
func TestAFirstMergeIntoANewRepositoryAsksItsModulesBuildsOnlyOnTheApprove(t *testing.T) {
const head = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
now := time.Date(2026, 10, 19, 0, 6, 0, 0, time.UTC)
mergedAt := now.Add(-time.Hour).Format(time.RFC3339Nano)
heard := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: "0177ea7a",
Number: 1, MergedAt: mergedAt, ModuleDirsSaid: true, ModuleDirs: []string{"watchman", "view"},
Paths: []string{"view/module.json", "watchman/module.json", "README.md"}}
built := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git"}}
for _, c := range []struct {
what string
change func(*anApprovedMerge, *link.SourceMoved, *[]inventory.Entry)
unkept bool
dirs []string
refused bool
waits bool
}{
{what: "the Approve of the merged head", dirs: []string{"view", "watchman"}},
{what: "a Decline", refused: true, change: func(a *anApprovedMerge, _ *link.SourceMoved, _ *[]inventory.Entry) {
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
}},
{what: "no word of mesh-delivery yet, however long ago the merge was made", unkept: true, waits: true},
{what: "a module already built from the repository", change: func(_ *anApprovedMerge, _ *link.SourceMoved, e *[]inventory.Entry) {
*e = append(*e, inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}})
}},
{what: "a merge into another branch", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { m.Base = "feat" }},
{what: "a merge that holds no module.json", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) {
m.ModuleDirs, m.Paths = nil, []string{"README.md"}
}},
} {
t.Run(c.what, func(t *testing.T) {
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
a.merge.Number = 1
m, entries := heard, []inventory.Entry{built}
if c.change != nil {
c.change(&a, &m, &entries)
}
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
return a.merge, !c.unkept, nil
}
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
return &a.ask, &a.warrant, nil
}
dirs, _, err := firstBuildsOf(t.Context(), nil, m, entries, nil)
switch {
case c.waits && !errors.Is(err, errNotAnnounced):
t.Fatalf("1146/AC6: %s: want a wait for the word, got %v", c.what, err)
case c.refused && !errors.Is(err, errFirstNotApproved):
t.Fatalf("1146/AC6: %s: want the Approve missing, got %v (dirs %v)", c.what, err, dirs)
case !c.refused && !c.waits && err != nil:
t.Fatalf("1146/AC6: %s: %v", c.what, err)
case !slices.Equal(dirs, c.dirs):
t.Fatalf("1146/AC6: %s: first builds asked of %v, want %v", c.what, dirs, c.dirs)
}
})
}
}
// 1146/AC8: a first merge waiting past the bound for mesh-delivery's word is said, naming the repository, the commit and
// the missing word; one inside the bound, one that is no first merge, and one answered (no longer waiting) are not.
func TestAFirstMergeWaitingPastItsBoundIsSaidAndClearsWhenAnswered(t *testing.T) {
now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC)
waiting := func(commit string, first bool, heard time.Time) inventory.BatchedMerge {
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
Commit: commit, Number: 1, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: first})
return inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, Heard: heard, Event: event}
}
got := firstBuildObservations([]inventory.BatchedMerge{
waiting("0177ea7a00", true, now.Add(-firstBuildWaitBound-time.Minute)),
waiting("1111111100", true, now.Add(-time.Minute)),
waiting("2222222200", false, now.Add(-time.Hour)),
}, now)
if len(got) != 1 {
t.Fatalf("1146/AC8: %d conditions raised, want the one merge past the bound: %+v", len(got), got)
}
o := got[0]
if o.Kind != kindFirstBuildUnasked || o.Scope != conditions.ScopeMerge || o.Severity != conditions.Warning ||
!strings.Contains(o.Summary, "novox/mesh-delivery") || !strings.Contains(o.Summary, "0177ea7a") ||
!strings.Contains(o.Summary, "announcement of the operator's Approve") {
t.Fatalf("1146/AC8: the condition is %+v", o)
}
// Answered, the merge is no longer listed as waiting: nothing is observed, and the condition clears.
if cleared := firstBuildObservations(nil, now); len(cleared) != 0 {
t.Fatalf("1146/AC8: an answered merge still raises %+v", cleared)
}
}
cmd/mesh-controller/fixclass_test.go
-55
View File
@@ -1,55 +0,0 @@
package main
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the
// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core,
// leaf or none, so mesh-delivery can pass it on with the fix's delivery.
func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
entry := func(name, path string) inventory.Entry {
e := fromRepo(name, repo, path)
e.Source.BuiltFrom = "old"
e.On = []string{"anchor"}
return e
}
gitea := entry("gitea", "modules/gitea")
resolver := entry("dnsmasq", "modules/dnsmasq")
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
delivery.Manifest.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
webApp := entry("photos", "modules/photos")
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
// photos stands on gitea: a change to gitea moves photos as its dependent.
edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}}
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
plan := func(paths ...string) link.ChangePlan {
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
}
for _, c := range []struct {
why string
paths []string
want string
}{
{"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf},
{"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore},
{"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore},
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
{"nothing any module's build reads", []string{"README.md"}, "none"},
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
{"a new module on the controller's own path, by its own class", []string{"next/mesh-host/module.json"}, inventory.ClassCore},
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
} {
if got := plan(c.paths...).Class; got != c.want {
t.Errorf("%s: class %q, want %q", c.why, got, c.want)
}
}
}
cmd/mesh-controller/delivery_changes.go → cmd/mesh-controller/forge_read.go
+7 -135
View File
@@ -17,9 +17,8 @@ import (
"github.com/novox/mesh-controller/internal/link"
)
// What a change says of the manifests it touches, read from the forge (novox/hq ADR 0419 clause 3, the lead's settlement
// of issue 977 question 1): the controller asks the forge for each changed module.json at the commit and at what it
// replaced, once per commit. The composed check's result is not used: the plan is cut after the merge.
// What the forge says of a change's manifests (novox/hq ADR 0502): read through the forge's module on the bus, once per
// commit. Kept by release 6 for the pull request's declared config; the delivery plan's reads of it are gone.
// changeSource is one change to a repository whose manifests are read: the commit after it and, when known, what it
// replaced (a pull request's base commit or branch). An empty Old is the first parent of New: the merge's base.
@@ -63,24 +62,6 @@ var manifestAt = func(ctx context.Context, owner, repo, file, ref string) (catal
return m, true, nil
}
// parentOf is the first parent of a commit, from the forge. A variable so a test needs no forge.
var parentOf = func(ctx context.Context, owner, repo, commit string) (string, error) {
var found struct {
Result struct {
Parents []struct {
SHA string `json:"sha"`
} `json:"parents"`
} `json:"result"`
}
if _, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(commit), &found); err != nil {
return "", err
}
if len(found.Result.Parents) == 0 {
return "", nil
}
return found.Result.Parents[0].SHA, nil
}
func escapedPath(p string) string {
parts := strings.Split(p, "/")
for i := range parts {
@@ -114,12 +95,6 @@ func askForgeApi(ctx context.Context, apiPath string, into any) (bool, error) {
return false, nil
}
// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit.
var (
manifestsRead = map[string]manifestRead{}
manifestsReadMu sync.Mutex
)
type manifestRead struct {
manifest catalogue.Manifest
exists bool
@@ -147,96 +122,11 @@ func manifestCached(ctx context.Context, owner, repo, file, ref string) (catalog
return m, exists, err
}
// verbsServed are the seat verbs a manifest serves, as "<seat>.<verb>": those of the seats it defines, those of the
// seats it claims, and its own tools as "<module>.<tool>".
func verbsServed(m catalogue.Manifest) []string {
var out []string
for _, s := range m.DefinesSeats {
for _, v := range s.Serves {
out = appendNew(out, s.Name+"."+v.Name)
}
}
for _, c := range m.Claims {
for _, v := range c.ServesFor(m) {
out = appendNew(out, c.Name+"."+v)
}
}
for _, t := range m.Tools {
out = appendNew(out, m.Module+"."+t)
}
return out
}
// verbsInvoked are the verbs a manifest calls, as "<seat>.<verb>" or "<module>.<tool>".
func verbsInvoked(m catalogue.Manifest) []string {
var out []string
for _, i := range m.Invokes {
if i = strings.TrimPrefix(i, "seat:"); i != "*" && i != "" {
out = appendNew(out, i)
}
}
return out
}
func added(now, was []string) []string {
var out []string
for _, v := range now {
if !slices.Contains(was, v) {
out = append(out, v)
}
}
return out
}
// changesOf reads, for each manifest the changes touch, what they add to the verbs its module serves and invokes, by
// module. A change whose paths are unsaid or truncated names no manifest: nothing is read for it. An error is the
// forge's, never read as "no change": the plan is not cut on a guess.
func changesOf(ctx context.Context, sources []changeSource) (map[string]moduleChange, error) {
out := map[string]moduleChange{}
for _, s := range sources {
if s.Truncated {
continue
}
for _, p := range s.Paths {
if path.Base(p) != "module.json" || slices.Contains(s.Removed, p) {
continue
}
now, exists, err := manifestCached(ctx, s.Owner, s.Repo, p, s.New)
if err != nil {
return nil, err
}
if !exists || now.Module == "" {
continue
}
old := s.Old
if old == "" {
first := s.Since
if first == "" {
first = s.New
}
if old, err = parentOf(ctx, s.Owner, s.Repo, first); err != nil {
return nil, err
}
}
var was catalogue.Manifest
if old != "" {
if was, _, err = manifestCached(ctx, s.Owner, s.Repo, p, old); err != nil {
return nil, err
}
}
c := out[now.Module]
c.Manifest = true
for _, v := range added(verbsServed(now), verbsServed(was)) {
c.Serves = appendNew(c.Serves, v)
}
for _, v := range added(verbsInvoked(now), verbsInvoked(was)) {
c.Invokes = appendNew(c.Invokes, v)
}
out[now.Module] = c
}
}
return out, nil
}
// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit.
var (
manifestsRead = map[string]manifestRead{}
manifestsReadMu sync.Mutex
)
// truncatedPaths are the manifests a change that names only a beginning of its files is read as changing: those of the
// modules it moves that are built from its repository. Said, never skipped: no edge may be lost to a long list.
@@ -251,21 +141,3 @@ func truncatedPaths(entries []inventory.Entry, owner, repo string, moved []strin
owner, repo, strings.Join(out, ", "))
return out
}
// changeSourcesOf are a group's members as changes to read: each head over what its pull request merges into.
func changeSourcesOf(members []orderMember, entries []inventory.Entry, reach map[string]orderReach) []changeSource {
var out []changeSource
for _, m := range members {
p := m.pull()
if p.PathsTruncated {
p.Paths, p.PathsTruncated = truncatedPaths(entries, p.Owner, p.Repo, reach[m.ID].Moved), false
}
old := p.BaseSHA
if old == "" {
old = p.Base
}
out = append(out, changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old, Paths: p.Paths,
Truncated: p.PathsTruncated, Removed: p.Removed})
}
return out
}
cmd/mesh-controller/found_wait_integration_test.go
-127
View File
@@ -1,127 +0,0 @@
//go:build integration
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
for _, c := range []struct {
name string
found time.Duration // when the directory was found, against now
passes bool
}{
{"found a day before the send", -24 * time.Hour, true},
{"found by this send", time.Hour, false},
} {
t.Run(c.name, func(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
return map[string]bool{"anchor": true, "laptop": true}, nil
}
backlogFacts := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f, err := backlogFacts(ctx, open, component)
// The used-as-found condition, raised after the send (its second statement): its own kind, never a
// fault the gate reads as the build's.
f.judged, f.openErr = true, nil
f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound,
Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"},
Raised: time.Now()})
f.health = map[string]inventory.NodeHealth{}
for _, n := range []string{"anchor", "laptop"} {
f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
foundDirectory("app", time.Now().Add(c.found)),
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
}
return f, err
}
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
advancePlans(ctx, b.open)
if p := b.release(t); p.State != inventory.PlanRolling {
break
}
time.Sleep(20 * time.Millisecond)
}
p := b.release(t)
v, found, err := inv.GateOf(ctx, "build-app-c2")
if c.passes {
if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
}
if !strings.Contains(v.Why+p.Note, "hand it over") {
t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
}
return
}
if p.State == inventory.PlanDone {
t.Fatalf("a directory this send found let the walk through: %s", p.Note)
}
})
}
}
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
k, _ := withConditionsInMemory(t)
ctx := t.Context()
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
for i := 0; i < 2; i++ {
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil {
t.Fatal(err)
}
}
open, _ := k.Open(ctx)
var got *conditions.Condition
for i, c := range open {
if c.Key == usedAsFoundKey("notes", "laptop") {
got = &open[i]
}
if c.Kind == kindModuleUnhealthy {
t.Fatalf("raised as a fault: %+v", c)
}
}
if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent ||
!strings.Contains(got.Summary, "notes.data") {
t.Fatalf("the condition: %+v", got)
}
// Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait.
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil {
t.Fatal(err)
}
open, _ = k.Open(ctx)
for _, c := range open {
if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent {
t.Fatal("a directory used as found became urgent")
}
}
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
t.Fatal(err)
}
open, _ = k.Open(ctx)
for _, c := range open {
if c.Key == usedAsFoundKey("notes", "laptop") {
t.Fatal("not cleared once handed over")
}
}
}
cmd/mesh-controller/found_wait_test.go
-50
View File
@@ -1,50 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the
// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an
// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds.
func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory,
Target: "/srv/" + module, State: link.StateUnhealthy, Since: since,
Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " +
"not given it — `nox node hand-over laptop <directory>` on the control-node, with its path, hands it to the mesh"}
}
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
now := time.Now()
sent := now.Add(-time.Minute)
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}}
h, why := moduleHealthWord("notes", "laptop", sent, f)
if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") {
t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why)
}
// Found by this very send: the send brought it, and it is not passed.
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}}
if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
t.Fatalf("a directory this send found reads %v %q; want not yet", h, why)
}
// A container down beside the old wait is a fault, as before.
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{
foundDirectory("notes", sent.Add(-time.Hour)),
{Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}}
if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet {
t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why)
}
}
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
cmd/mesh-controller/gate.go
-1651
View File
File diff suppressed because it is too large Load Diff
cmd/mesh-controller/gate_followup_test.go
-468
View File
@@ -1,468 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
// wait, and a provider waiting for a login says who waits on it.
// withGateBounds sets the gate's bounds for one test.
func withGateBounds(t *testing.T, within, every, bound time.Duration) {
t.Helper()
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateWithin, gateEvery, gateBound = within, every, bound
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
}
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
// of its modules' resources.
func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
return func() gateFacts {
now := time.Now()
at := now
return gateFacts{now: now,
reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{},
served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
}
}
}
func healthyContainer(module string) inventory.ResourceHealth {
return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
State: link.StateHealthy}
}
// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
judgeLeavesAsTheCore(t)
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second)
since := time.Now().Add(-time.Second)
base := factsOn(t, "laptop", since, healthyContainer("app"))
wasGather := gatherGateFacts
t.Cleanup(func() { gatherGateFacts = wasGather })
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
f := base()
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
return f, nil
}
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
judgings := 0
for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
t.Fatal(err)
}
judgings++
time.Sleep(30 * time.Millisecond)
}
if g.Verdict != inventory.GateFailed || judgings < gatePasses {
t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
judgings, g)
}
if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
}
if !strings.Contains(g.Why, "witness") {
t.Errorf("the verdict does not say what broke: %q", g.Why)
}
}
// broker_runtime is the node tools' module name, a core component a witness judges.
const broker_runtime = "node-tools"
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
// while a module beside it is not yet healthy counts the healthy one once.
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, time.Hour, time.Hour, time.Hour)
since := time.Now().Add(-time.Minute)
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
wasGather := gatherGateFacts
t.Cleanup(func() { gatherGateFacts = wasGather })
gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
now := time.Now()
for i := 0; i < 3; i++ {
if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
t.Fatal(err)
}
}
if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
}
}
// **A carried build's pass carries its own wait, never another's** (review (e)).
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
since := time.Now().Add(-time.Minute)
g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
app, _, _ := inv.GateOf(ctx, "build-app-c2")
late, _, _ := inv.GateOf(ctx, "build-late-c2")
if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
t.Errorf("app's pass: %+v; want it without late's wait", app)
}
if !strings.Contains(late.Why, "relogin needed on laptop") {
t.Errorf("late's pass: %+v; want its own wait", late)
}
}
// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
for _, b := range []inventory.Build{
{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
b.Manifest = manifest
b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
Asked: b.Asked}); err != nil {
t.Fatal(err)
}
if b.Commit == "l1" {
if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
"late": "l1"}); err != nil {
t.Fatal(err)
}
}
}
wasMoves := machineMoves
t.Cleanup(func() { machineMoves = wasMoves })
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
gather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f, err := gather(ctx, open, component)
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
State: link.StateUnhealthy, Reason: "down"}}}}
return f, err
}
gateEvery, gateBound = 0, 400*time.Millisecond
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
advancePlans(ctx, g.open)
if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
break
}
time.Sleep(30 * time.Millisecond)
}
p := g.plan(t)
if p.State != inventory.PlanFailed {
t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
}
if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
!strings.Contains(v.Why, "on its own") {
t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
}
if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
t.Fatal("late's build is not marked failed at its gate")
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
current["late"].Commit)
}
}
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
k := conditionsFrom
register(t, open, catalogue.Manifest{Module: "db", Version: "1",
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
t.Fatal(err)
}
at := h0
say := func(machine string, rs ...link.ResourceHealth) {
t.Helper()
at = at.Add(time.Second)
for i := range rs {
rs[i].Since = at
}
if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
t.Fatal(err)
}
}
account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
for look := 0; look < 2; look++ {
say("anchor", account, unit)
say("laptop", shop)
}
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
var keys []string
var c conditions.Condition
for _, x := range list {
keys = append(keys, x.Key)
if x.Key == "module.db.anchor.relogin-needed" {
c = x
}
}
if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
}
if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
}
}
// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back
// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node
// tools are marked and put back in the very judging that found them broken — their registered build is the
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
judgeLeavesAsTheCore(t)
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
build := func(module, commit string, asked time.Time) {
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
}
for _, m := range []string{"app", broker_runtime} {
build(m, "c1", old)
if _, err := inv.Assign(ctx, "laptop", m); err != nil {
t.Fatal(err)
}
}
if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil {
t.Fatal(err)
}
build("app", "c2", recent)
build(broker_runtime, "c2", recent)
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
})
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, _ := open.inventory.Assigned(ctx, node)
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil }
var sends []string
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
n++
carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit}
sends = append(sends, node+":"+carried[broker_runtime])
digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n))
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
var seen []string // the node tools' registered build at each judging
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
now := time.Now()
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, _ := open.inventory.LastReports(ctx)
for _, r := range reports {
f.reports[r.Node] = r
}
current, _ := open.inventory.CurrentBuilds(ctx)
seen = append(seen, current[broker_runtime].Commit)
app := healthyContainer("app")
if current[broker_runtime].Commit == "c2" {
// The witness reverted the node tools; app cannot reach them yet.
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}}
app.State, app.Reason = link.StateUnhealthy, "down"
}
f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}}
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
return f, nil
}
withGateBounds(t, time.Hour, 0, 10*time.Second)
release := func() inventory.Plan {
t.Helper()
plans, _ := inv.RecentPlans(ctx, 10)
for _, p := range plans {
if p.Release != nil {
return p
}
}
t.Fatal("no walk")
return inventory.Plan{}
}
// Judged until the first judging has found the node tools broken, and one more.
for i := 0; i < 20 && len(seen) < 2; i++ {
advancePlans(ctx, open)
}
if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" {
t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen)
}
if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed {
t.Fatal("the node tools' build is not marked failed at once")
}
if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" {
t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate)
}
if !slices.Contains(sends, "laptop:c1") {
t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends)
}
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
advancePlans(ctx, open)
if release().State != inventory.PlanRolling {
break
}
time.Sleep(20 * time.Millisecond)
}
p := release()
if p.State != inventory.PlanFailed {
t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note)
}
if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") {
t.Fatalf("app's verdict: %+v; want its own pass", v)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit)
}
}
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
record := func(commit string, m catalogue.Manifest) {
raw, _ := json.Marshal(m)
at := time.Now().Add(-time.Hour)
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-lights-" + commit, Module: "lights", Commit: commit,
Repository: "novox/mesh-catalog", Path: "modules/lights", Manifest: raw, Asked: at, At: at}); err != nil {
t.Fatal(err)
}
}
record("c1", catalogue.Manifest{Module: "lights"})
record("c2", catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
"name": "operator", "groups": []any{"lights"}}}})
pairs := []judged{{module: "lights", node: "laptop"}}
shelf := map[string]catalogue.Manifest{}
for _, c := range []struct {
from string
want bool
}{{"c1", true}, {"", false}, {"c0-never-built", false}} {
g := &inventory.PlanGate{From: c.from, To: "c2"}
if got := movesAddingGroups(ctx, inv, g, pairs, shelf)["lights"]; got != c.want {
t.Errorf("a move from %q adds a group: %v; want %v", c.from, got, c.want)
}
}
}
cmd/mesh-controller/gate_integration_test.go
-486
View File
@@ -1,486 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
func aGateMesh(t *testing.T) *gateMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
g := &gateMesh{open: open, health: map[string]health{}}
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
wasGated := gateFirstMachine
gateFirstMachine = true
t.Cleanup(func() { gateFirstMachine = wasGated })
g.keeper, _ = withConditionsInMemory(t)
g.told = &conditions.Told{}
was := doctorFrom
doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told}
t.Cleanup(func() { doctorFrom = was })
for _, b := range []inventory.Build{
{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
b.Manifest = manifest
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
register := func(commit string, asked time.Time) {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
Head: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
}
register("c1", time.Now().Add(-2*time.Hour))
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, n, "app"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
}
register("c2", time.Now().Add(-time.Minute))
// Every send: recorded with the build the module is at, applied and reported by the machine.
n := 0
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
g.sent = append(g.sent, append([]string(nil), names...))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
// What a judging reads: the store's reports, and a health the test says per machine.
wasGather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
if g.health[r.Node] == healthBroken {
r.Outcome = inventory.OutcomeFailed
}
f.reports[r.Node] = r
}
for node, h := range g.health {
if h == healthNotYet {
// Not reported on the send: neither the send made last, nor the gate's own (issue 352).
f.rolledBack[node] = nil
r := f.reports[node]
r.Current, r.Declared, r.ReportedSequence = false, "", 0
f.reports[node] = r
}
}
return f, nil
}
t.Cleanup(func() { gatherGateFacts = wasGather })
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateWithin, gateEvery = time.Hour, time.Hour
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
built := time.Now().UTC()
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
Build: "build-2"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
return g
}
func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
advancePlans(ctx, g.open) // the first machine is sent the new build
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v, not the first machine alone", g.sent)
}
if p := g.plan(t); p.Modules["app"].Previous != "c1" {
t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"])
}
g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging
gateEvery = 0
advancePlans(ctx, g.open)
p := g.plan(t)
gate := p.Modules["app"].Gate
if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed ||
gate.Rollback != inventory.RolledBack {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
}
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) {
t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
}
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" {
t.Fatalf("anchor was last sent %v, not the previous build", sent)
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" {
t.Fatalf("laptop was sent the failed build: %v", sent)
}
if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed {
t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err)
}
// Said: a condition for the operator, and the event.
open, err := g.keeper.Open(ctx)
if err != nil {
t.Fatal(err)
}
var key string
for _, c := range open {
if c.Kind == kindRolledBack {
key = c.Key
}
}
if key != "build.app.anchor.rolled-back" {
t.Fatalf("no rolled-back condition for app on anchor: %+v", open)
}
saidIt := false
for _, e := range g.told.Said() {
saidIt = saidIt || e.Event == link.KeyRolledBack
}
if !saidIt {
t.Fatalf("the rollback was not said as an event: %+v", g.told.Said())
}
// Never again: more passes send nothing, a second judging rolls nothing back, and the failed build
// heard again is not registered.
advancePlans(ctx, g.open)
state := p.Modules["app"]
gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again")
if len(g.sent) != 2 {
t.Fatalf("sent again after the rollback: %v", g.sent)
}
_, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app",
Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}),
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}})
if err == nil || !strings.Contains(err.Error(), "failed its gate") {
t.Fatalf("the failed build was registered again: %v", err)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
t.Fatalf("the module moved to %s", current["app"].Commit)
}
if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") {
t.Fatalf("a plan stopped at a failed gate was retried: %v", err)
}
// The probe keeps the condition while the newest verdict is the failure.
obs, err := probeGates(ctx, doctorFrom)
if err != nil || len(obs) != 1 || obs[0].Key() != key {
t.Fatalf("the gate's probe found %+v %v", obs, err)
}
}
func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
gateEvery = 0
for i := 0; i < 6; i++ {
advancePlans(ctx, g.open)
}
p := g.plan(t)
if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v", g.sent)
}
gate := p.Modules["app"].Gate
if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) {
t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate)
}
if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err)
}
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
t.Fatalf("a passing build left a condition: %+v %v", obs, err)
}
}
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
g := aGateMesh(t)
gateFirstMachine = false
ctx := t.Context()
judged := false
wasGather := gatherGateFacts
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
judged = true
return wasGather(ctx, open, component)
}
t.Cleanup(func() { gatherGateFacts = wasGather })
for i := 0; i < 6; i++ {
advancePlans(ctx, g.open)
}
p := g.plan(t)
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
t.Fatalf("sent %v, not both machines in one send", g.sent)
}
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
}
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
}
}
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
gateWithin = time.Hour // a core module could not pass on fewer readings for an hour
advancePlans(ctx, g.open) // the first machine is sent the new build
advancePlans(ctx, g.open) // one healthy judging
p := g.plan(t)
gate := p.Modules["app"].Gate
if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 {
t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note)
}
if !strings.Contains(gate.Why, "healthy 1 times") {
t.Errorf("the verdict says %q", gate.Why)
}
}
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
advancePlans(ctx, g.open)
g.health["anchor"] = healthNotYet
gateEvery = 0
advancePlans(ctx, g.open)
if p := g.plan(t); !p.Open() || len(g.sent) != 1 {
t.Fatalf("judged before the bound: %s %v", p.State, g.sent)
}
gateBound = -time.Second
advancePlans(ctx, g.open)
p := g.plan(t)
if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed ||
!strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack {
t.Fatalf("the plan is %s, its gate %+v", p.State, gate)
}
}
func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}},
Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}}
if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil {
t.Fatal(err)
}
u, err := inv.UpgradeOf(ctx, "nats")
if err != nil || u.RollOut || u.From != catalogue.FromBus {
t.Fatalf("the bus's policy is %+v %v", u, err)
}
if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) {
t.Fatalf("a person rolled the bus out: %v", err)
}
var sent [][]string
was := sendRollout
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
sent = append(sent, names)
return names, nil
}
t.Cleanup(func() { sendRollout = was })
now := time.Now().UTC()
plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now,
State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}},
Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
advancePlans(ctx, open)
if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 {
t.Fatalf("a plan sent the bus: %s %v", p.State, sent)
}
held, err := busHeld(ctx, inv, []string{"anchor", "laptop"})
if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" {
t.Fatalf("a push may send the bus's machine: %v %v", held, err)
}
// Nor may any other send — a plan's for another module on that machine carried the bus with it.
if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) {
t.Fatalf("a send to the bus's machine was not refused: %v", err)
}
// A rebuild that made the same artifacts is no move.
for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} {
b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}}
b.Asked, b.At = time.Now(), time.Now()
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 {
t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second),
At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image",
Reference: "registry/nats@sha256:new"}}}); err != nil {
t.Fatal(err)
}
// The planned step refuses to start without its word on reversibility, and without a snapshot taken
// first by the bus machine's backup holder.
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") {
t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err)
}
wasSnapshot := takeBusSnapshot
t.Cleanup(func() { takeBusSnapshot = wasSnapshot })
takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") }
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil ||
!strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 {
t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent)
}
takeBusSnapshot = func(_ context.Context, module, node string) (string, error) {
return node + "'s restore point of " + module, nil
}
if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) {
t.Fatalf("the step sent %v, not the bus's machine", sent)
}
s, found, err := inv.LatestBusStep(ctx)
if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil ||
!reflect.DeepEqual(s.Machines, []string{"anchor"}) {
t.Fatalf("the step is %+v %v %v", s, found, err)
}
}
func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := asksRecorded(t)
for _, name := range []string{"gone", "kept"} {
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
}
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef",
Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"},
Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}}
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
t.Fatal(err)
}
if len(*asked) != 0 {
t.Fatalf("a deleted module was asked to build: %v", *asked)
}
if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 {
t.Fatalf("a merge that only deleted a module made a plan: %+v", plans)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
if _, still := shelf["gone"]; still {
t.Fatal("a deleted module nothing holds was not forgotten")
}
// Without the announcer saying what went: the build finds no manifest, and the plan goes on.
asked0 := time.Now().UTC().Add(-time.Minute)
plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0,
State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}},
Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-k", Module: "kept", Repository: "novox/mesh-catalog",
At: time.Now().UTC(), Asked: asked0, Failed: "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so " +
"there is nothing saying what it is: open …/module.json: no such file or directory"}); err != nil {
t.Fatal(err)
}
advancePlans(ctx, open)
p, _ := inv.PlanByID(ctx, "plan-deleted")
if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted {
t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"])
}
}
func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) {
open := aMesh(t)
d := &doctor{open: open}
at := time.Now().UTC()
witnessed.heard("control", []lease.Rollback{
{Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack,
Why: "the new controller did not take the lease within 60s", At: at},
{Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at},
}, at)
t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) })
obs, err := probeGates(t.Context(), d)
if err != nil {
t.Fatal(err)
}
got := map[string]conditions.Severity{}
for _, o := range obs {
got[o.Key()] = o.Severity
}
want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent,
"core.node-tools.control.unwitnessed": conditions.Warning}
if !reflect.DeepEqual(got, want) {
t.Fatalf("the witness's verdicts are %v", got)
}
witnessed.heard("control", nil, time.Now())
if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 {
t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err)
}
}
cmd/mesh-controller/gate_rule_test.go
-347
View File
@@ -1,347 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
func withTheGatesTimes(t *testing.T) {
t.Helper()
wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
t.Cleanup(func() {
gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
})
}
func TestEachModuleHasItsGateRule(t *testing.T) {
withTheGatesTimes(t)
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
within time.Duration
witnessed bool
said string
}{
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
{catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"},
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"app", nil, 1, 0, false, "a leaf module's rule"},
} {
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
c.witnessed, c.said)
}
}
}
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
func TestTheCoreStaysTheCore(t *testing.T) {
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
// The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 §3, issue 1210).
claims := []catalogue.Claim(nil)
if m == catalogue.DeliverySeat {
claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
}
if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) {
t.Errorf("%s is no longer judged as the core for its class", m)
}
}
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
t.Error("the resolver's holder is no longer judged as the core for its class")
}
}
// gateOn judges one module on one machine, healthy at every reading.
type gateOn struct {
module, node string
g *inventory.PlanGate
holder *lease.Holder
}
func aGateOn(module, node string, since time.Time) *gateOn {
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
Applied: map[string]time.Time{}}}
}
func (o *gateOn) read(at time.Time, h health, why string) {
m := catalogue.Manifest{Module: o.module}
if o.module == catalogue.DeliverySeat { // today's module, which claims the seat
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
}
r := gateRuleOf(o.module, m)
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
}
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(15 * time.Second)
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
// Ready, but inside the witness window, which opened before it took the lease: not counted.
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
o.read(sent.Add(after), healthGood, "")
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
}
}
o.read(sent.Add(80*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
}
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
!strings.Contains(o.g.Why, "ADR 0351") {
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
}
}
// A controller that starts again takes the lease again: its witness window starts again, and so does the
// gate's wait.
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(10 * time.Second)
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
o.read(sent.Add(90*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
}
o.read(sent.Add(130*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed {
t.Fatalf("not passed after the new window: %+v", o.g)
}
}
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
// and from the send at the least.
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
}
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
}
}
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
if o.g.Verdict != inventory.GateFailed {
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
}
o = aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != "" {
t.Fatalf("failed before the bound: %+v", o.g)
}
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
}
}
// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
t.Helper()
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
return g, func(after time.Duration, h health) {
why := ""
if h != healthGood {
why = "the node-engine reports another build"
}
tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
}
}
// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
// minutes (novox/hq ADR 0352).
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
withTheGatesTimes(t)
for _, module := range []string{"mesh-host", "dnsmasq"} {
g, judge := threeReadings(t, module, time.Now())
judge(15*time.Second, healthGood)
judge(55*time.Second, healthGood)
if g.Verdict != "" || g.Passes != 2 {
t.Fatalf("%s: after two readings %+v; want still judging", module, g)
}
judge(95*time.Second, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
}
}
}
// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
// two minutes comes sooner than the spacing after the last pass.
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
last := sent.Add(100 * time.Second)
held := &inventory.PlanGate{Since: &sent, LastPass: &last}
if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
t.Fatal("the judging at two minutes is not due, or is due before")
}
held.Faulted = true
if dueAtTheMost(held, sent.Add(2*time.Minute)) {
t.Fatal("a send that was not healthy at a reading is judged at two minutes")
}
g, judge := threeReadings(t, "mesh-host", time.Now())
judge(100*time.Second, healthGood)
judge(2*time.Minute, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 2 {
t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
}
g, judge = threeReadings(t, "mesh-host", time.Now())
judge(2*time.Minute, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 1 {
t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
}
}
// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
withTheGatesTimes(t)
g, judge := threeReadings(t, "mesh-host", time.Now())
judge(10*time.Second, healthNotYet)
judge(100*time.Second, healthGood)
judge(2*time.Minute, healthGood)
if g.Verdict != inventory.GatePassed || g.Faulted {
t.Fatalf("%+v; want passed at two minutes", g)
}
}
// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
// never healthy fails at the bound and is put back; one found broken fails.
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
withTheGatesTimes(t)
g, judge := threeReadings(t, "mesh-host", time.Now())
judge(20*time.Second, healthNotYet)
judge(60*time.Second, healthGood)
judge(100*time.Second, healthGood)
judge(2*time.Minute+10*time.Second, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 3 {
t.Fatalf("%+v; want passed only on its third healthy reading", g)
}
g, judge = threeReadings(t, "mesh-host", time.Now())
judge(60*time.Second, healthGood)
judge(100*time.Second, healthNotYet)
judge(2*time.Minute, healthGood)
if g.Verdict != "" || !g.Faulted {
t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
}
judge(10*time.Minute+time.Second, healthNotYet)
if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
t.Fatalf("never healthy again: %+v; want failed at the bound", g)
}
g, judge = threeReadings(t, "mesh-host", time.Now())
judge(60*time.Second, healthGood)
judge(100*time.Second, healthBroken)
if g.Verdict != inventory.GateFailed {
t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
}
}
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("node-tools", "ace", sent)
o.g.Applied["ace"] = sent.Add(10 * time.Second)
o.read(sent.Add(30*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
}
o.read(sent.Add(70*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
}
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
o.read(sent.Add(5*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
}
}
// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
// two minutes (novox/hq ADR 0352 decision 3).
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
modules := []string{"mesh-controller", "mesh-host"}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
}
floors["mesh-host"] = sent
floors["mesh-controller"] = sent.Add(90 * time.Second)
judge := func(after time.Duration, h health) {
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
}
judge(30*time.Second, healthGood)
if g.Passes != 0 {
t.Fatalf("a reading before the controller's floor was counted: %+v", g)
}
judge(70*time.Second, healthNotYet)
judge(2*time.Minute, healthGood)
if !g.Faulted || g.Verdict != "" {
t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
}
}
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
modules := []string{"mesh-controller", "mesh-host"}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
}
for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
}
if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
}
}
cmd/mesh-controller/gate_test.go
-205
View File
@@ -1,205 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8).
// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered,
// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is
// sent and reports it — with the gate's bounds shortened to judge in three steps.
type gateMesh struct {
open *stores
sent [][]string
health map[string]health // per machine, what a judging finds; healthy when unsaid
keeper *conditions.Keeper
told *conditions.Told
}
func (g *gateMesh) plan(t *testing.T) inventory.Plan {
t.Helper()
p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate")
if err != nil {
t.Fatal(err)
}
return p
}
// A module's build that fails its gate on the first machine is put back there — the previous build
// registered and sent to it again — and never reaches the second machine; it is marked, said as a
// condition and an event, never registered or rolled back again.
// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then
// the rest are sent, the verdict kept, and the plan done.
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
// The health a judging finds, per core component and for a module.
func TestTheHealthDefinitions(t *testing.T) {
now := time.Now()
since := now.Add(-time.Minute)
applied := func(node string) gateFacts {
at := now
return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node,
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}}
}
m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}}
f := applied("anchor")
f.served["anchor"] = served{runtime: true, tools: map[string]bool{}}
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") {
t.Errorf("a module whose tools are not served is %v (%s)", h, why)
}
f.served["anchor"].tools["app"] = true
if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
t.Errorf("a module applied with its tools served is %v (%s)", h, why)
}
// A condition raised about it on that machine since it was sent: not yet healthy.
f.judged = true
f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}}
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet {
t.Errorf("a module with a new condition about it is %v", h)
}
f.open[0].Raised = since.Add(-time.Hour)
if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood {
t.Errorf("a condition older than the send counted against it: %v", h)
}
// A witness that put it back: broken.
f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb",
Outcome: lease.OutcomeRolledBack, Why: "x", At: now}}
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
t.Errorf("a component a witness put back is %v", h)
}
// The node tools answer, or not.
f = applied("anchor")
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet {
t.Errorf("node tools not answering are %v", h)
}
f.served["anchor"] = served{runtime: true}
if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood {
t.Errorf("node tools answering are %v", h)
}
// The controller: the lease held since the send, by a controller that says it is ready.
f = applied("control")
f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}}
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet {
t.Errorf("a lease held by a controller older than the build is %v", h)
}
f.holder.Taken = now
if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood {
t.Errorf("a new controller holding the lease and ready is %v", h)
}
f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"}
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet ||
!strings.Contains(why, "first run") {
t.Errorf("a controller not ready is %v (%s)", h, why)
}
// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
f = applied("anchor")
stated := func(state, reason string, heard time.Time) {
f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
Resources: []inventory.ResourceHealth{
{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
}
plain := catalogue.Manifest{Module: "app"}
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
}
stated(link.StateStarting, "", now)
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
}
stated(link.StateUnhealthy, "restarting", now)
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
}
stated(link.StateHealthy, "", since.Add(-time.Minute))
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
}
stated(link.StateHealthy, "", now)
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
}
f.healthErr = errors.New("the store is away")
if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
t.Errorf("health that cannot be read is never read as healthy: %v", h)
}
// And the condition it raises after the send holds it, as every condition about it does.
f.healthErr = nil
f.judged = true
f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
}
// A machine that refused what it was sent: broken.
f = applied("anchor")
r := f.reports["anchor"]
r.Outcome = inventory.OutcomeRefused
f.reports["anchor"] = r
if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken {
t.Errorf("a refusal is %v", h)
}
}
// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused,
// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits.
// A merge that deletes a module's directory builds nothing for it: the module is forgotten where
// nothing holds it, and a plan whose build finds no manifest goes on instead of failing.
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return b
}
func nodeID(t *testing.T, open *stores, name string) string {
t.Helper()
n, err := open.inventory.NodeByName(context.Background(), name)
if err != nil {
t.Fatal(err)
}
return n.ID
}
// What a machine's witness says in its reports is a condition while it says it, by the host's words:
// `core.<component>.<node>.<outcome>`, urgent for a rollback, a warning when it could not judge — and
// gone with the first report that no longer carries it.
// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle
// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still
// open after a healthy reading.
func judgeLeavesAsTheCore(t *testing.T) {
t.Helper()
wasPasses, wasSettles := gateLeafPasses, gateLeafSettles
gateLeafPasses, gateLeafSettles = gatePasses, true
t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles })
}
cmd/mesh-controller/generation.go
-40
View File
@@ -7,10 +7,8 @@ import (
"io"
"os"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -162,44 +160,6 @@ func counterBehind(r link.GenerationRefusal, now int64) bool {
return r.Generation >= now && r.Applied > r.Generation
}
// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
seen := map[string]bool{}
var out []conditions.Observation
for _, s := range f.refusedSends {
if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
continue
}
seen[s.NodeName] = true
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
Machine: s.NodeName, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
modulesWords(s.Modules)),
Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
if s.CounterRaisedTo > 0 {
// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
// was raised; nothing has sent the machine its declaration since, so a person is asked to.
o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
s.NodeName)
o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
"update and kept what it had. The record is repaired; a push sends the update again.",
s.NodeName, s.NodeName, s.NodeName)
}
out = append(out, o)
}
return out
}
// modulesWords is a send's modules as a sentence says them.
func modulesWords(modules []string) string {
if len(modules) == 0 {
cmd/mesh-controller/generation_test.go
-148
View File
@@ -1,148 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The assignment generation a declaration was composed from (novox/hq issue 234).
func bodyKeys(t *testing.T, s sendable) map[string]any {
t.Helper()
raw, err := s.Body()
if err != nil {
t.Fatal(err)
}
var keys map[string]any
if err := json.Unmarshal(raw, &keys); err != nil {
t.Fatal(err)
}
return keys
}
// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate
// composes its put-back afresh, with the generation as it stands.
func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57}
var told sendable
told.Resources = resources
reads.stamp(&told)
keys := bodyKeys(t, told)
if keys["generation"] != float64(40) || keys["sequence"] != float64(12) {
t.Fatalf("a machine that reads a generation was sent %v", keys)
}
if _, there := keys["put_back"]; there {
t.Fatalf("a put-back mark was sent: %v", keys)
}
if told.composedFrom != 40 || told.actingEpoch != 57 {
t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
}
older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57}
var untold sendable
untold.Resources = resources
older.stamp(&untold)
if _, there := bodyKeys(t, untold)["generation"]; there {
t.Fatal("a machine whose node-engine never said it reads a generation was sent one")
}
if untold.composedFrom != 40 {
t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
}
}
// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
// process and build that composed it.
func TestASendNamesItsSender(t *testing.T) {
said := senderOf("g14.node-tools, through the mesh-controller seat")
if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") {
t.Fatalf("the sender reads %q", said)
}
}
// refusedSend is a send a machine refused for its generation at a moment.
func refusedSend(at time.Time) inventory.Send {
return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true}
}
// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
// against which the machine applied, and its sequence — the facts that took a morning to look for.
func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
f := calm(now)
f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))}
got := watchGenerationRefusals(f)
if len(got) != 1 {
t.Fatalf("%+v", got)
}
o := got[0]
if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" {
t.Errorf("raised as %s about %q", o.Key(), o.Machine)
}
for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} {
if !strings.Contains(o.Summary, want) {
t.Errorf("the summary does not say %q: %s", want, o.Summary)
}
}
if o.Headline == "" || o.Explanation == "" || o.Resolved == "" {
t.Errorf("the condition is not worded for the operator: %+v", o)
}
}
// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never
// only a line on stderr.
func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) {
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
f := calm(now)
s := refusedSend(now.Add(-time.Minute))
s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil
f.refusedSends = []inventory.Send{s}
got := watchGenerationRefusals(f)
if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") {
t.Fatalf("an unattributed refusal was not raised as one: %+v", got)
}
}
// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was
// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do.
func TestACounterBehindTheMachineAsksForAPush(t *testing.T) {
now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
f := calm(now)
s := refusedSend(now.Add(-time.Minute))
s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41
f.refusedSends = []inventory.Send{s}
got := watchGenerationRefusals(f)
if len(got) != 1 {
t.Fatalf("%+v", got)
}
if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") ||
strings.Contains(got[0].Explanation, "Nothing for you to do") {
t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation)
}
}
// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below
// it, and the counter is left alone for that.
func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) {
for _, c := range []struct {
refused, applied, now int64
behind bool
}{
{refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40
{refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past
{refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is
} {
r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied}
if got := counterBehind(r, c.now); got != c.behind {
t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind)
}
}
}
cmd/mesh-controller/grants_step_integration_test.go
-27
View File
@@ -1,27 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The store test's manifests are manifests the controller takes in: checked here, where no store is needed,
// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2).
func TestTheGrantsStepFixturesAreManifests(t *testing.T) {
for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false),
"restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} {
m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1"
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
if _, err := catalogue.ParseManifest(raw); err != nil {
t.Errorf("%s: %v", name, err)
}
}
}
cmd/mesh-controller/grants_step_store_test.go
-196
View File
@@ -1,196 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/identity"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list,
// and nothing else — not the new build of a module it runs, not a module newly assigned there.
//
// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a
// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants
// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor
// makes the step fail, said, rather than install it unjudged.
func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
start := time.Now().Add(-time.Hour)
build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult {
manifest["module"], manifest["version"] = module, "1"
raw, _ := json.Marshal(manifest)
return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module,
On: "anchor", Module: module, Commit: commit, Manifest: raw,
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
}
for _, b := range []link.BuildResult{
build("nats", "n1", start, natsFixture()),
build("restic", "c1", start.Add(time.Second), resticFixture(false)),
build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()),
} {
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
t.Fatal(err)
}
}
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} {
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
t.Fatal(err)
}
}
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil {
t.Fatal(err)
}
}
wasEpoch := epochForActs
epochForActs = func(context.Context) (uint64, error) { return 7, nil }
t.Cleanup(func() { epochForActs = wasEpoch })
// A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped.
compose := func(under context.Context) (catalogue.Resolution, sendable) {
t.Helper()
numbered, err := allot(under, inv, "anchor")
if err != nil {
t.Fatal(err)
}
plan, _, err := resolveNode(under, open, "anchor")
if err != nil {
t.Fatal(err)
}
declared, err := composeSend(under, open, "anchor")
if err != nil {
t.Fatal(err)
}
numbered.stamp(&declared)
return plan, declared
}
record := func(declared sendable) {
t.Helper()
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch,
sentRecordOf(ctx, declared)); err != nil {
t.Fatal(err)
}
}
// Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop
// too (who is on the private network), and a step that kept every machine's builds refused anchor's
// composition for want of laptop's last send (repo-check of #230 at 0f853e93).
kept := keepingEveryBuild(keepingRecorded(ctx), "anchor")
// What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it.
_, before := compose(keepingRecorded(ctx))
record(before)
plan, declared := compose(kept)
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
t.Fatalf("with nothing changed, the step reads %q, %v", only, err)
}
// The merge: restic's c2 gives it a health tool.
if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute),
resticFixture(true))); err != nil {
t.Fatal(err)
}
// The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool.
generation, err := inv.AssignmentGeneration(ctx)
if err != nil {
t.Fatal(err)
}
plan, declared = compose(kept)
for _, m := range plan.Modules {
if m.Module == "restic" && len(m.Tools) > 0 {
t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools)
}
}
if declared.Builds["restic"] != "c1" {
t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"])
}
if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") {
t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers)
}
// The generation it carries is the current one (novox/hq issue 234), and the step does not raise it.
if declared.composedFrom != generation {
t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation)
}
// The guard, on the very declaration the send sends: only the user list differs from the last send.
if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" {
t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err)
}
// And the gate's refusal still reads the step's send: it moves nothing there.
if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" {
t.Fatalf("the step's send is refused as a move: %v, %v", names, err)
}
// Recorded as the send records it: anchor still runs restic c1, its gate still to come there.
record(declared)
if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" {
t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err)
}
if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation {
t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err)
}
// A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent.
if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil {
t.Fatal(err)
}
plan, declared = compose(kept)
only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") {
t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only)
}
// And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled.
identity.ForTest(t)
if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) ||
!strings.Contains(err.Error(), "wallpaper") {
t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err)
}
}
// fixtureImage is the image every fixture container of the grants step's tests runs.
var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64)
// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by
// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone
// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither.
func resticFixture(withTool bool) map[string]any {
backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage}
measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage}
m := map[string]any{"resources": []any{backup, measure}}
if withTool {
backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"}
measure["health"] = map[string]any{"kind": "runtime"}
m["tools"] = []string{"backup_health"}
}
return m
}
// natsFixture is the bus's module: it holds mesh-broker and is sent the user list.
func natsFixture() map[string]any {
return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf",
"claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}}
}
// wallpaperFixture is a module the bus's machine is newly assigned.
func wallpaperFixture() map[string]any {
return map[string]any{"resources": []any{
map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}}
}
cmd/mesh-controller/grants_step_test.go
-176
View File
@@ -1,176 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
)
// The grants reach the bus before the gate (novox/hq issue 490).
//
// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to
// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused
// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and
// that machine runs restic too, so it was left out of the send while restic's new build waited there for
// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged.
// aSend is one send the walk made: to which machines, and whether every build there was kept.
type aSend struct {
names []string
keepsAll bool
judged []string
}
// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine
// holding the bus is novox, and its list is behind as behind says.
func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend {
t.Helper()
var sends []aSend
wasSend, wasBehind := sendRollout, brokerBehindOf
t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind })
brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) {
if slices.Contains(names, holder) {
return holder, false, nil
}
return holder, behind, nil
}
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])}
for n := range scopeOf(ctx).judged {
s.judged = append(s.judged, n)
}
sends = append(sends, s)
if refuse != nil && s.keepsAll {
return nil, refuse
}
return names, nil
}
return &sends
}
// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox.
func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) {
sends := sendsRecorded(t, "novox", true, nil)
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if len(*sends) != 2 {
t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends)
}
first, then := (*sends)[0], (*sends)[1]
if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 {
t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first)
}
if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" {
t.Errorf("the second send is %+v, want ace, judged", then)
}
if strings.Join(sent, ",") != "ace" {
t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent)
}
if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" {
t.Fatalf("the gate keeps %+v as its grants step", grants)
}
line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants})
if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") {
t.Errorf("plans says the gate as %q", line)
}
}
// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus.
func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) {
for _, c := range []struct {
name string
node string
behind bool
}{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} {
t.Run(c.name, func(t *testing.T) {
sends := sendsRecorded(t, "novox", c.behind, nil)
_, grants, err := sendJudged(t.Context(), nil, c.node)
if err != nil {
t.Fatal(err)
}
if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil {
t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants)
}
})
}
}
// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before.
func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) {
sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person"))
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
t.Errorf("the walk made %+v", *sends)
}
if grants == nil || grants.Failed == "" || grants.At != nil {
t.Fatalf("the gate keeps %+v", grants)
}
if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line,
"grants step to novox FAILED, sent without it: a bus upgrade waits for a person") {
t.Errorf("plans says the gate as %q", line)
}
}
// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the
// gate says it FAILED with what differed, and the walk goes on.
func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) {
refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly)
sends := sendsRecorded(t, "novox", true, refusal)
sent, grants, err := sendJudged(t.Context(), nil, "ace")
if err != nil {
t.Fatal(err)
}
if strings.Join(sent, ",") != "ace" || len(*sends) != 2 {
t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends)
}
if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") {
t.Fatalf("the gate keeps %+v", grants)
}
}
// The step's exemption is the list's content alone: its path or mode moving is a change like any other.
func TestOnlyTheUserListsContentIsExempt(t *testing.T) {
list := "nats.bus-users"
file := func(fields map[string]any) sentResource {
r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"}
for k, v := range fields {
r[k] = v
}
body, _ := json.Marshal(map[string]any{"resources": []any{r}})
s, err := summarize(body)
if err != nil {
t.Fatal(err)
}
return s[0]
}
was := file(nil)
if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 {
t.Errorf("the list's new content reads as more: %v", other)
}
for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} {
other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list)
if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) {
t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other)
}
}
}
// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed
// as any send composes them.
func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) {
ctx := keepingEveryBuild(t.Context(), "novox")
if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") {
t.Error("the grants step keeps the builds of a machine it does not send")
}
}
cmd/mesh-controller/group_one_batch_test.go
-139
View File
@@ -1,139 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// A group is one batch and one plan (novox/hq ADR 0418 clauses 3 and 6): the batch holding a merge of a group with a
// member still to merge does not close by the merge window, however long that member takes, and a group with a
// member of each kind is one batch whatever order its members are heard in.
func TestAGroupsBatchWaitsForItsLastMemberHoweverLongItTakes(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
states := map[string]string{"novox/one": "ready", "novox/two": "ready"}
groupStates(t, "feat/g", states)
first := fromHead(repoMerge("one", "c1", t0), "feat/g")
hear(t, open, first, t0)
// Past the window and past its maximum, with the second member still to merge: one batch, no walk.
for _, after := range []time.Duration{90 * time.Second, 11 * time.Minute, 30 * time.Minute} {
cutAt(t, open, t0.Add(after))
ws, bs := walks(t, open)
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 1 {
t.Fatalf("%s after the first member: %d walk(s), %d batch(es) %+v", after, len(ws), len(bs), bs)
}
}
// The last member merges: the window runs again from that merge.
at := t0.Add(30 * time.Minute)
states["novox/two"] = "merged"
hear(t, open, fromHead(repoMerge("two", "c2", at), "feat/g"), at.Add(time.Second))
cutAt(t, open, at.Add(time.Second+89*time.Second))
if ws, _ := walks(t, open); len(ws) != 0 {
t.Fatalf("cut before the window ran again: %+v", ws)
}
cutAt(t, open, at.Add(time.Second+90*time.Second))
ws, bs := walks(t, open)
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" {
t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
}
}
// A group with a member of each kind is one batch and one walk, in both orders of hearing, started without
// mesh-delivery's word.
func TestAGroupWithAMemberOfEachKindIsOneBatchAndOneWalk(t *testing.T) {
for name, ownFirst := range map[string]bool{"controller member first": true, "catalogue member first": false} {
t.Run(strings.ReplaceAll(name, " ", "_"), func(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"}
groupStates(t, "feat/both-kinds", states)
own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds")
cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds")
first, second := cat, own
if ownFirst {
first, second = own, cat
}
second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano)
hear(t, open, first, t0)
states["novox/"+second.Repo] = "merged"
hear(t, open, second, t0.Add(10*time.Second))
ws, bs := walks(t, open)
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
t.Fatalf("both members heard: %d walk(s), %d batch(es) %+v; want one batch of both", len(ws), len(bs), bs)
}
cutAt(t, open, t0.Add(10*time.Second+90*time.Second))
ws, bs = walks(t, open)
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/mesh-controller") != "k1" ||
ws[0].CommitOf("novox/mesh-catalog") != cat.Commit {
t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
}
if ws[0].Delivery != nil && ws[0].Delivery.Awaits != "" {
t.Fatalf("a plan carrying the controller waits for %s's word", ws[0].Delivery.Awaits)
}
})
}
}
// A group whose last member has not merged ten minutes after its first raises a condition naming it, and the held
// batch is not the batch S18 reads as a cut the controller failed to make.
func TestAHeldBatchRaisesTheGroupConditionAndNotTheBatchNotCutOne(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"})
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0)
now := t0.Add(12 * time.Minute)
batches, err := gatherBatches(t.Context(), open.inventory, now)
if err != nil || len(batches) != 1 {
t.Fatalf("batches %+v: %v", batches, err)
}
f := &signalFacts{now: now, batches: batches}
if got := watchBatchesNotCut(f); len(got) != 0 {
t.Fatalf("a held batch is read as not cut: %+v", got)
}
got := watchGroupsNotMerged(f)
if len(got) != 1 || got[0].Kind != kindGroupNotMerged || !strings.Contains(got[0].Summary, "novox/two") ||
!strings.Contains(got[0].Summary, "feat/g") {
t.Fatalf("the group's condition: %+v", got)
}
if early := watchGroupsNotMerged(&signalFacts{now: t0.Add(5 * time.Minute), batches: batches}); len(early) != 0 {
t.Fatalf("raised before the maximum: %+v", early)
}
}
// The batch held for a group survives a restart: the hold is in the record.
func TestAHeldBatchSurvivesARestart(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"})
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0)
reopened, err := open.inventory.Batches(t.Context())
if err != nil || len(reopened) != 1 || len(reopened[0].Delivery.Batch.Holds) != 1 {
t.Fatalf("the record of the held batch: %+v %v", reopened, err)
}
cutAt(t, open, t0.Add(time.Hour))
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 || bs[0].State != inventory.PlanAssembling {
t.Fatalf("after an hour: %d walk(s), batches %+v", len(ws), bs)
}
}
// A mesh-delivery that does not yet say the group's gate is composed merges member two only after member one is
// delivered, so a batch held for it would wait for ever: the merge waits for the window, as before.
func TestAnOlderMeshDeliverysGroupIsNotHeldOpen(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
groupStatesFrom(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}, false)
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0)
if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 0 {
t.Fatalf("held for an older mesh-delivery's group: %+v", bs)
}
cutAt(t, open, t0.Add(90*time.Second))
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
t.Fatalf("at its window: %d walk(s), %d batch(es)", len(ws), len(bs))
}
}
cmd/mesh-controller/group_wait.go
-228
View File
@@ -1,228 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **A delivery group still assembles** (novox/hq ADR 0306 decision 2): a merge whose delivery group has a member
// not yet merged waits for the merge window, whether or not a batch is assembling, so the group's members walk in
// one batch, in their order (ADR 0276 decision 5). Which groups exist and where each member stands is
// mesh-delivery's record (ADR 0239): a group is the pull requests of two repositories or more sharing a head
// branch name, which is its id. The controller asks it, through its `groups` verb, when a merge from a head is
// heard; a member whose pull request is open (proposed, checked, ready or rejected) is still to merge. When
// mesh-delivery does not answer within groupAskWithin, or answers something unreadable, the merge waits: the safe
// error is to wait, and its batch says why.
// groupAskWithin is how long mesh-delivery is given to say a merge's group, while the merge is being heard.
var groupAskWithin = 3 * time.Second
// openPullStates are the states of a delivery whose pull request is open and not merged (mesh-delivery's table):
// a member in one of them is still to merge. Closed unmerged is stopped, final, and waits for nothing.
var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready": true, "rejected": true}
// groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `<owner/repo>@<commit> (state)`.
type groupMember struct {
Repository, State string
// Composed says the group's line says its gate is the composed check's (a mesh-delivery that merges a group in
// one pass): only such a group is held open for its last member, since an older mesh-delivery merges member two
// only after member one is delivered, and a held batch would wait for ever.
Composed bool
}
// groupsLine is one group as mesh-delivery's `groups` says it: its id (the head branch name), its members as text and,
// from a mesh-delivery that knows novox/hq ADR 0382, each member again as an object (`member_details`) naming its pull
// request's number, whole head, base branch and whether it is open.
type groupsLine struct {
ID string `json:"id"`
Members []string `json:"members"`
Details []memberLine `json:"member_details,omitempty"`
// ComposedGate says the group's gate is the composed check's (novox/hq ADR 0403): a member's own check runs the
// repository's check alone. Said by a mesh-delivery that merges a group in one pass; absent from an older one, whose
// members keep their own gate.
ComposedGate bool `json:"composed_gate,omitempty"`
}
// memberLine is one member of a group, as `groups` says it beside its text.
type memberLine struct {
ID string `json:"id"`
Repository string `json:"repository"`
State string `json:"state"`
Number int `json:"number"`
Head string `json:"head"`
Base string `json:"base"`
Open bool `json:"open"`
}
// groupsAsked is mesh-delivery's `groups`: every group it keeps, or an error when it cannot be asked or did not
// answer within groupAskWithin. A seam a test replaces.
var groupsAsked = func(ctx context.Context, held bool) ([]groupsLine, error) {
if !held {
return nil, fmt.Errorf("no holder of %s is on record to say it", catalogue.DeliverySeat)
}
// On the bus as this process is: the serving controller's own connection, lent, or — in a verb that runs as a command
// of its own, such as a recheck's `delivery check` — one dialled for the moment (onTheBus).
var raw json.RawMessage
err := onTheBus(func(conn *nats.Conn) error {
asking, cancel := context.WithTimeout(ctx, groupAskWithin)
defer cancel()
var err error
raw, err = askDeliveryOwner(asking, conn, "groups", map[string]any{})
return err
})
if err != nil {
if errors.Is(err, link.ErrNothingServes) {
return nil, fmt.Errorf("nothing serves %s.groups", catalogue.DeliverySeat)
}
return nil, err
}
var lines []groupsLine
if err := json.Unmarshal(raw, &lines); err != nil {
return nil, fmt.Errorf("%s.groups answered something unreadable: %w", catalogue.DeliverySeat, err)
}
return lines, nil
}
// deliveryGroupOf is what mesh-delivery says of the group a head names: its members, and whether the head names a
// group at all; an error when it cannot be asked or did not answer. A seam a test replaces.
var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]groupMember, bool, error) {
lines, err := groupsAsked(ctx, held)
if err != nil {
return nil, false, err
}
for _, l := range lines {
if l.ID != head {
continue
}
var members []groupMember
for _, s := range l.Members {
m := memberOf(s)
m.Composed = l.ComposedGate
members = append(members, m)
}
return members, true, nil
}
return nil, false, nil
}
// memberOf reads one member as `groups` says it: `novox/mesh-sdk@d41d255f28ad (held)`.
func memberOf(s string) groupMember {
id, state := strings.TrimSpace(s), ""
if i := strings.LastIndex(id, " ("); i >= 0 && strings.HasSuffix(id, ")") {
id, state = id[:i], id[i+2:len(id)-1]
}
repository, _, _ := strings.Cut(id, "@")
return groupMember{Repository: repository, State: state}
}
// groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing
// when it names no head, its head names no group, or every other member is merged or closed.
//
// A member of another repository whose merge this controller already keeps in a batch not yet cut, of either
// kind, is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins
// that batch by their head. **Kinds do not part a group** (novox/hq ADR 0418 clause 6): a controller member and a
// catalogue member share one batch.
//
// Asked before the plans are held: a slow answer holds up no walk.
func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) {
why, _, err := groupWaitHolding(ctx, inv, m, held)
return why, err
}
// groupWaitHolding is groupWait, and whether the wait is for a member still to merge: such a batch is held open
// however long it takes (ADR 0418 clause 3). A wait only because mesh-delivery did not say is no hold.
func groupWaitHolding(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held bool) (string, bool, error) {
if m.Head == "" {
return "", false, nil
}
open, isGroup, composed, err := openMembersOf(ctx, inv, held, m.Head, m.Owner+"/"+m.Repo)
if err != nil {
return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+
"window, the safe side", catalogue.DeliverySeat, m.Head, err), false, nil
}
if !isGroup || len(open) == 0 {
return "", false, nil
}
return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), composed, nil
}
// openMembersOf are the members of the group a head names that are still to merge, in words: those whose pull
// request is open and whose merge no batch not yet cut keeps. self is a repository to leave out, the merge being heard.
func openMembersOf(ctx context.Context, inv *inventory.Inventory, held bool, head, self string) ([]string, bool, bool, error) {
kept, err := keptFromHead(ctx, inv, head)
if err != nil {
return nil, false, false, err
}
members, isGroup, err := deliveryGroupOf(ctx, held, head)
if err != nil || !isGroup {
return nil, isGroup, false, err
}
var open []string
composed := false
for _, g := range members {
composed = composed || g.Composed
if strings.EqualFold(g.Repository, self) || kept[strings.ToLower(g.Repository)] || !openPullStates[g.State] {
continue
}
open = append(open, g.Repository+" ("+g.State+")")
}
sort.Strings(open)
return open, true, composed, nil
}
// keptFromHead are the repositories (lower case) of the merges from a head that batches not yet cut hold, of
// either kind.
func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string) (map[string]bool, error) {
batches, err := inv.Batches(ctx)
if err != nil {
return nil, err
}
kept := map[string]bool{}
for _, b := range batches {
merges, err := inv.MergesOf(ctx, b.ID)
if err != nil {
return nil, err
}
for _, k := range merges {
if announcedOf(k).Head == head {
kept[strings.ToLower(k.Repository)] = true
}
}
}
return kept, nil
}
// batchOfGroup is the batch not yet cut that holds a merge of another repository from the same head: the group's
// batch, whatever kind it is.
func batchOfGroup(ctx context.Context, inv *inventory.Inventory, head, repository string) (inventory.Plan, bool, error) {
if head == "" {
return inventory.Plan{}, false, nil
}
batches, err := inv.Batches(ctx)
if err != nil {
return inventory.Plan{}, false, err
}
for _, b := range batches {
merges, err := inv.MergesOf(ctx, b.ID)
if err != nil {
return inventory.Plan{}, false, err
}
for _, k := range merges {
if announcedOf(k).Head == head && !strings.EqualFold(k.Repository, repository) {
return b, true, nil
}
}
}
return inventory.Plan{}, false, nil
}
cmd/mesh-controller/handacts.go
-34
View File
@@ -55,23 +55,9 @@ const causeDrill = "drill"
// repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or
// listed without a decision, counts, so a new verb is a repair until its entry says otherwise.
var handActVerbs = []handActVerb{
// Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by
// default (ADR 0236) exists to end — except a push that only moved builds a `record` policy held for
// a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go).
{Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " +
"upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded},
// Stopping or starting a walk the operator chose on a warrant (novox/hq ADR 0259) is their decision.
{Verb: "plans stop", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
DecidedFor: []string{conditions.CauseOperatorAnswer}},
{Verb: "plans close"},
// A module registered at references by hand, because the builder had no key for it (ADR 0526 clause 3): a repair
// the builder should have made, so it counts.
{Verb: "register"},
// A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or
// not trusted with it — either is a repair the owner should have made. Unless the operator chose it on
// a warrant (ADR 0259).
{Verb: "plans go", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)",
DecidedFor: []string{conditions.CauseOperatorAnswer}},
// An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a
// channel that proved who answered, performed by the controller as itself.
{Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"},
@@ -200,19 +186,6 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) {
}
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
// A push naming one machine says whether it only moves recorded builds (novox/hq issue 301):
// read from what it carries, before it is sent.
if verb == "push" && len(args) == 1 && !strings.HasPrefix(args[0], "-") {
switch carried, why, err := recordedPushOf(ctx, args[0]); {
case err != nil:
fmt.Fprintf(os.Stderr, "whether this push only moves recorded builds could not be read, so it is "+
"recorded as a push by hand: %v\n", err)
case len(carried) > 0:
act.Kind, act.Carried = link.KindRecordedBuilds, carried
default:
fmt.Printf("a push by hand, not of recorded builds only: %s\n", why)
}
}
err := onTheBus(func(conn *nats.Conn) error {
written, err := link.RecordHandAct(ctx, conn, act)
act = written
@@ -319,13 +292,6 @@ func listHandActs(ctx context.Context, args []string, w io.Writer) error {
fmt.Fprintf(w, ", condition %s", a.Condition)
}
fmt.Fprintln(w, ")")
if pushedRecorded(a) {
carried := strings.Join(a.Carried, "; ")
if carried == "" {
carried = recordedBefore[a.ID]
}
fmt.Fprintf(w, " a push of recorded builds, no repair: %s\n", carried)
}
}
if len(repeated) > 0 {
causes := make([]string, 0, len(repeated))
cmd/mesh-controller/handacts_test.go
-43
View File
@@ -9,44 +9,6 @@ import (
"github.com/novox/mesh-controller/internal/inventory"
)
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
// the verb.
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
}{
{"push", map[string]any{"node": "anchor"}},
{"push", map[string]any{}},
{"plans", map[string]any{"close": "plan-1"}},
{"plans", map[string]any{"stop": "plan-1"}},
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
} {
argv, err := argvFor(c.verb, c.args)
if c.verb == "plans" && err == nil {
// The seat composes the command line; the command refuses it, before opening anything.
err = plansCommand(context.Background(), argv[1:])
}
if err == nil || !strings.Contains(err.Error(), "why") {
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
}
}
for _, args := range [][]string{{"EVENTS", "controller"}} {
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
t.Errorf("consumer-reset without why: %v", err)
}
}
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
!strings.Contains(err.Error(), "--why") {
t.Errorf("hand-act record without why: %v", err)
}
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
!strings.Contains(err.Error(), "--cause") {
t.Errorf("hand-act record without a cause: %v", err)
}
}
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
func TestARepairByHandCarriesItsReason(t *testing.T) {
for _, c := range []struct {
@@ -54,11 +16,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) {
args map[string]any
want string
}{
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
"push anchor --wait 0s --why stuck --cause sent-not-reported"},
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
"plans close plan-1 --why the report will not come"},
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
cmd/mesh-controller/handover_test.go
-25
View File
@@ -5,10 +5,7 @@ import (
"errors"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -69,28 +66,6 @@ func TestAHandOverIsRefusedThroughEveryVerb(t *testing.T) {
}
}
// The condition's words are plain and name no machine's binary; the operator's line, with the node and the path
// (novox/hq ADR 0272), is in the summary the module's health gives (moduleHealthWord) and in the evidence.
func TestTheUsedAsFoundConditionNamesTheOperatorsLine(t *testing.T) {
rs := []inventory.ResourceHealth{foundDirectory("notes", time.Now().Add(-24*time.Hour))}
o := usedAsFoundObservation("notes", "laptop", "notes on laptop uses notes.data as found", rs)
if strings.Contains(o.Needs, "mesh-host") || strings.Contains(o.Explanation, "mesh-host") ||
!strings.Contains(o.Needs, "control-node") {
t.Fatalf("the condition's words: %q %q", o.Needs, o.Explanation)
}
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs,
Resolved: o.Resolved}, "laptop"); !ok {
t.Fatalf("not plain: %s", why)
}
f := gateFacts{now: time.Now(), health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
Resources: rs}}}
h, why := moduleHealthWord("notes", "laptop", time.Now().Add(-time.Hour), f)
if h != healthPerson || !strings.Contains(why, "`nox node hand-over laptop <directory>` on the control-node") ||
strings.Contains(why, "mesh-host") || strings.Contains(why, "/srv/") {
t.Fatalf("the module's health reads %v %q; want the operator's line", h, why)
}
}
// **Nothing is asked of a node the mesh does not know, and the engine's refusal is the command's failure**
// (review of issue 356): a refused hand-over never exits as a success.
func TestAHandOverAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
cmd/mesh-controller/healers.go
+1 -238
View File
@@ -12,8 +12,6 @@ import (
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
@@ -100,25 +98,6 @@ const actsInController = "controller"
// test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its
// event (healers_test.go).
var healerRegistry = []healerRow{
{ID: "H1", Kinds: []string{"sent-not-reported"},
Condition: "a machine was sent a declaration and has not reported it (S2)",
Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " +
"comes back is not the declaration it was sent, or nothing comes, send it its current declaration " +
"again — what a push of that one machine does, never moving a build a policy or a plan holds back",
Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)",
Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAMachine, repair: repairReport},
{ID: "H2", Kinds: []string{"stalled"},
Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " +
"exists — or already finished — every module of every tier built or failed, and every one that " +
"rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " +
"take a transition (D14, novox/hq ADR 0239)",
Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " +
"done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " +
"reads its walk again and takes only the transition its table names",
Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)",
Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed,
applies: appliesToAStalePlan, repair: repairPlan},
{ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"},
Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " +
"bus does not hold (D6, S9)",
@@ -181,12 +160,8 @@ type healing struct {
// The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the
// bus's objects, resetting a consumer.
askReport func(ctx context.Context, node string) error
sendAgain func(ctx context.Context, node string) error
assertObjects func(ctx context.Context) error
resetConsumer func(stream, name string) (string, error)
// reportWait is how long H1 waits for the machine's report after asking.
reportWait time.Duration
mu sync.Mutex
// declined is why each condition was last passed over, so it is said once and `healers` can show it.
@@ -199,26 +174,7 @@ func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Telle
h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding,
epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) },
now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) },
reportWait: 45 * time.Second, declined: map[string]string{}}
h.askReport = func(ctx context.Context, node string) error {
if h.js == nil {
return errors.New("this controller is not on the bus")
}
return askToReport(ctx, h.js.Conn(), node)
}
h.sendAgain = func(ctx context.Context, node string) error {
// **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push
// that did not name the machine — a person's word sends a held build, a healer's does not.
held, err := heldMachines(ctx, open, []string{node})
if err != nil {
return err
}
if why := held[node]; len(why) > 0 {
return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+
"`push %s` sends it, on a person's word", strings.Join(why, "; "), node)
}
return sendTo(ctx, open, []string{node})
}
declined: map[string]string{}}
h.assertObjects = func(ctx context.Context) error {
if h.js == nil {
return errors.New("this controller is not on the bus")
@@ -239,21 +195,6 @@ func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Telle
return h
}
// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core
// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store.
func askToReport(ctx context.Context, conn *nats.Conn, node string) error {
body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"})
if err != nil {
return err
}
if err := conn.Publish(broker.AskReportSubject(node), body); err != nil {
return err
}
flushing, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
return conn.FlushWithContext(flushing)
}
// keep runs the healers until ctx ends.
func (h *healing) keep(ctx context.Context) {
tick := time.NewTicker(healEvery)
@@ -537,184 +478,6 @@ func brakeObservation(acts []inventory.Heal, said string) conditions.Observation
Said: strings.Join(most, ", ")}
}
// --- H1 ----------------------------------------------------------------------------------------------
// appliesToAMachine is H1's: a machine named, its budget counted against the condition.
func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) {
if c.Subject.Machine == "" {
return "", false, "it names no machine", nil
}
return c.Key, true, "", nil
}
// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing
// comes, send it again.
func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
node := c.Subject.Machine
inv := h.open.inventory
// The store's clock, as the report's time is: not the runner's, which a test moves by hand.
asked := time.Now()
askErr := h.askReport(ctx, node)
current, heard := false, false
if askErr == nil {
deadline := time.Now().Add(h.reportWait)
for {
r, found, err := lastReportOf(ctx, inv, node)
if err != nil {
return "", "", err
}
if found && r.At != nil && r.At.After(asked) {
heard, current = true, r.Current
if current {
break
}
}
if time.Now().After(deadline) {
break
}
select {
case <-ctx.Done():
return "", "", ctx.Err()
case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)):
}
}
}
if current {
return "asked " + node + "'s node-engine to say again what it last applied",
"it reported the declaration it was sent: its report had not reached the mesh", nil
}
why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb"
switch {
case askErr != nil:
why = "it could not be asked: " + askErr.Error()
case heard:
why = "it reported a declaration other than the one it was sent"
}
if err := h.sendAgain(ctx, node); err != nil {
return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed",
err
}
return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil
}
// lastReportOf is one machine's last report beside its last send.
func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) {
reports, err := inv.LastReports(ctx)
if err != nil {
return inventory.Reported{}, false, err
}
for _, r := range reports {
if r.Node == node {
return r, true, nil
}
}
return inventory.Reported{}, false, nil
}
// --- H2 ----------------------------------------------------------------------------------------------
// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty
// when it is neither, which is not H2's to repair.
func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) {
if p.Release != nil {
return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236)
}
recent, err := inv.RecentPlans(ctx, 50)
if err != nil {
return "", "", err
}
for _, newer := range recent {
if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) ||
newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded || newer.Batch() {
continue
}
return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+
"and branch", newer.ID, newer.Repository, short(newer.Commit)), nil
}
for _, tier := range p.Tiers {
for _, m := range tier {
s := p.Modules[m]
if s == nil || (s.State != "built" && s.State != "failed") {
return "", "", nil
}
if s.State == "built" && s.SentAt == nil {
u, err := inv.UpgradeOf(ctx, m)
if err != nil {
return "", "", err
}
// A recorded module whose move goes out on a machine is still the walk's to send (novox/hq ADR 0377).
if u.RollOut || len(wentOut(s.Recorded)) > 0 {
return "", "", nil
}
}
}
}
return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " +
"out was sent — nothing is left to wait on", nil
}
// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished.
func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
if c.Subject.Scope == conditions.ScopeDelivery {
return appliesToAStalledDelivery(ctx, h, c)
}
p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID)
if err != nil {
return "", false, "", err
}
if !p.Open() {
return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil
}
state, _, err := planStale(ctx, h.open.inventory, p)
if err != nil {
return "", false, "", err
}
if state == "" {
return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " +
"which its own signal says", nil
}
return c.Key, true, "", nil
}
// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set.
func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
if c.Subject.Scope == conditions.ScopeDelivery {
return repairDelivery(ctx, h, c)
}
inv := h.open.inventory
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return "", "", err
}
defer release()
p, err := inv.PlanByID(ctx, c.Subject.ID)
if err != nil {
return "", "", err
}
if !p.Open() {
return "closed nothing", "the plan is " + p.State + " already", nil
}
state, why, err := planStale(ctx, inv, p)
if err != nil {
return "", "", err
}
if state == "" {
return "closed nothing", "its wait is no longer superseded or finished", nil
}
p.State = state
p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why)
if state != inventory.PlanDone {
sayUnsent(&p, func(m string) bool {
u, err := inv.UpgradeOf(ctx, m)
return err == nil && u.RollOut
})
}
if err := inv.SavePlan(ctx, &p); err != nil {
return "", "", err
}
return "closed the plan " + p.ID + " (" + state + ")", why, nil
}
// --- H3 ----------------------------------------------------------------------------------------------
// appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object.
cmd/mesh-controller/healers_integration_test.go
-201
View File
@@ -5,7 +5,6 @@ package main
import (
"context"
"errors"
"slices"
"strconv"
"strings"
"testing"
@@ -18,94 +17,6 @@ import (
"github.com/novox/mesh-controller/internal/testbus"
)
func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, clock := healingOn(t, open)
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil {
t.Fatal(err)
}
key := sentNotReported(t, "laptop")
// First: the report had not reached the mesh, and asking for it brings it.
asked := 0
h.askReport = func(ctx context.Context, node string) error {
asked++
if node != "laptop" {
t.Errorf("asked %s", node)
}
_, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied,
At: time.Now().Add(time.Second), Declared: "d2"})
return err
}
h.tick(ctx)
c, found, err := conditionsFrom.Get(ctx, key)
if err != nil || !found {
t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err)
}
if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") ||
c.Resolver != "healer:H1" {
t.Fatalf("after the first act: asked %d, %+v", asked, c)
}
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed ||
acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 {
t.Fatalf("the act was not said as healer-acted: %+v", acts)
}
// Within its settle, nothing more: the observation has its turn.
clock.pass(time.Minute)
h.tick(ctx)
if asked != 1 {
t.Fatalf("acted again inside the settle: asked %d", asked)
}
// Second: the machine says nothing, so it is sent again.
clock.pass(3 * time.Minute)
sent := 0
h.askReport = func(context.Context, string) error { asked++; return nil }
h.sendAgain = func(_ context.Context, node string) error { sent++; return nil }
h.tick(ctx)
if asked != 2 || sent != 1 {
t.Fatalf("the second act: asked %d, sent %d", asked, sent)
}
c, _, _ = conditionsFrom.Get(ctx, key)
if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") {
t.Fatalf("tried %+v", c.Tried)
}
// The budget is spent: the operator's, urgent, said; and no healer touches it again.
clock.pass(4 * time.Minute)
h.tick(ctx)
c, _, _ = conditionsFrom.Get(ctx, key)
if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 ||
!strings.Contains(c.Tried[2].Outcome, "budget of 2") {
t.Fatalf("not handed to the operator: %+v", c)
}
if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated {
t.Fatalf("the escalation was not said: %+v", acts)
}
clock.pass(time.Hour)
h.tick(ctx)
if asked != 2 || sent != 1 || len(told.said()) != 3 {
t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said()))
}
heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour))
if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed ||
heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 {
t.Fatalf("the heals kept: %+v %v", heals, err)
}
// And a heal is not a hand act: what S15 counts never sees it.
for _, x := range heals {
if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") {
t.Errorf("a heal reads as a hand act: %+v", x)
}
}
}
func TestNoHealerActsWithoutTheLease(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
@@ -123,93 +34,6 @@ func TestNoHealerActsWithoutTheLease(t *testing.T) {
}
}
func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, clock := healingOn(t, open)
for i := 0; i < healBrakeLimit; i++ {
if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent",
Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed,
At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil {
t.Fatal(err)
}
}
sentNotReported(t, "laptop")
h.tick(ctx) // the fakes fail the test if anything acts
braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked")
if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") {
t.Fatalf("the brake was not said: %+v %v", braked, err)
}
if len(told.said()) != 0 {
t.Fatalf("a healer acted under the brake: %+v", told.said())
}
// Past the hour of the first act, still held: it lets go an hour after the last.
clock.pass(30 * time.Minute)
h.tick(ctx)
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held {
t.Fatal("the brake let go before an hour had passed since the last act")
}
clock.pass(31 * time.Minute)
asked := 0
h.askReport = func(context.Context, string) error { asked++; return nil }
h.sendAgain = func(context.Context, string) error { return nil }
h.tick(ctx)
if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held {
t.Fatal("the brake held an hour after the last act")
}
if asked != 1 {
t.Fatalf("the healers did not act again after the brake let go: asked %d", asked)
}
}
func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
h, told, _ := healingOn(t, open)
created := time.Now().UTC().Add(-time.Hour)
older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created,
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}}
waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created,
State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}}
newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute),
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}
for _, p := range []*inventory.Plan{&older, &waiting, &newer} {
if err := open.inventory.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
}
for _, id := range []string{"plan-old", "plan-other"} {
if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled",
Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil {
t.Fatal(err)
}
}
h.tick(ctx)
closed, err := open.inventory.PlanByID(ctx, "plan-old")
if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") ||
!strings.Contains(closed.Note, "plan-new") {
t.Fatalf("the superseded plan: %+v %v", closed, err)
}
if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding {
t.Fatalf("a plan still waiting was closed: %+v", other)
}
if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf {
t.Fatalf("a plan H2 does not repair was touched: %+v", c)
}
if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" {
t.Fatalf("said %+v", acts)
}
// Finished: every module built, none rolled out unsent — closed as done.
done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created,
State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}}
if err := open.inventory.SavePlan(ctx, &done); err != nil {
t.Fatal(err)
}
if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") {
t.Fatalf("a finished plan reads %q %q %v", state, why, err)
}
}
func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
@@ -422,28 +246,3 @@ func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T
}
}
}
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop"))
if err != nil {
t.Fatal(err)
}
if err := askToReport(t.Context(), conn, "laptop"); err != nil {
t.Fatal(err)
}
msg, err := asked.NextMsg(5 * time.Second)
if err != nil || !strings.Contains(string(msg.Data), "healer H1") {
t.Fatalf("the machine heard %v, %v", msg, err)
}
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") ||
slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") {
t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err)
}
}
cmd/mesh-controller/healers_test.go
+1 -5
View File
@@ -42,8 +42,6 @@ func raisedKinds() map[string]string {
// inducedFailures are the healers seen acting in this file, by id: a row without one fails.
var inducedFailures = map[string]string{
"H1": "TestH1AsksAMachineToReportAndSendsItAgain",
"H2": "TestH2ClosesAPlanAnotherHasTakenOver",
"H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget",
"H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers",
}
@@ -170,10 +168,8 @@ func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) {
open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil })
h := &healing{open: open, keeper: conditionsFrom, teller: told,
epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now,
say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond,
say: func(f string, a ...any) { t.Logf(f, a...) },
declined: map[string]string{}}
h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil }
h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil }
h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil }
h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil }
return h, told, clock
cmd/mesh-controller/health_probes.go
-91
View File
@@ -10,10 +10,8 @@ import (
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
@@ -60,95 +58,6 @@ func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observa
return nil, nil
}
// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the
// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send.
func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
return machinesHealth(ctx, d, hostModule)
}
// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them
// announced, answering the bus's discovery.
func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
return machinesHealth(ctx, d, broker.RuntimeModule)
}
// machinesHealth judges a component on every machine running it and heard from, by its health
// definition, as the gate does — with no condition taken as the build's doing.
func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) {
inv := d.open.inventory
running, err := inv.Running(ctx, component)
if err != nil {
return nil, err
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
f, err := gatherGateFacts(ctx, d.open, coreComponent(component))
if err != nil {
return nil, err
}
f.judged = false
heard := heardMachines(d)
var out []conditions.Observation
for _, node := range running {
if !heard[node] {
continue // a machine not heard from is S1's
}
if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound {
continue // inside the bound of a send: S2's, and the gate's
}
// What the machine did with what it was sent is not the component's health: the node-engine's is
// that it reported its current declaration at all, the node tools' that they answer.
if r := f.reports[node]; r.Current || component == broker.RuntimeModule {
now := time.Now()
r.Current, r.Outcome = true, inventory.OutcomeApplied
if r.At == nil {
r.At = &now
}
f.reports[node] = r
}
if component == hostModule {
// A node-engine reporting a build the mesh delivered, current or previous, is the version
// split's (D10), not ill health; a build the mesh did not deliver is.
f.engines[node] = deliveredOr(shelf[component], f.engines[node])
}
h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f)
if h == healthGood {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node,
Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning,
Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)})
}
return sortedFound(out), nil
}
// deliveredOr is the reported engine build, or the current delivered one when the report names any
// build at all: what H-engine judges is that it reports, not which.
func deliveredOr(m catalogue.Manifest, reported string) string {
if reported == "" {
return reported
}
if v := deliveredVersions(m); len(v) > 0 {
return v[0]
}
return reported
}
// componentWords is a core component as a sentence names it.
func componentWords(component string) string {
switch component {
case hostModule:
return "node-engine"
case broker.RuntimeModule:
return "node tools"
case catalogue.ControllerSeatName:
return "controller"
}
return component
}
// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a
// request crosses it to every machine's node tools and back.
func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
cmd/mesh-controller/held_back.go
-283
View File
@@ -1,283 +0,0 @@
package main
import (
"context"
"fmt"
"io"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
)
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
// issue 259, ADR 0221).
//
// A named push ends by sending every other machine whose declaration differs from what it was last
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
// out makes every machine running it differ from the merge on, and so did a module whose plan was
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
// everywhere at once.
//
// What tells the two apart is which build of each module the machine was last sent, kept with every
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
// heldBack is why a push that did not name a machine must not send it, empty when it may.
//
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
// machine was last sent — including a module the machine was never sent at all. A move is held when
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
// is not known, so a held upgrade cannot be told from anything else.
func heldBack(node string, modules []string, sent map[string]string, known bool,
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
if !known {
return []string{"which builds it was last sent is not known — it was last sent before the " +
"mesh kept them, or sent a declaration by hand"}
}
var why []string
for _, m := range modules {
now := current[m]
was, carried := sent[m]
if carried && was == now.Commit {
continue
}
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
if !now.RollOut {
why = append(why, move+", which its upgrade policy records rather than rolls out")
continue
}
if id := planStillToSend(plans, m, node); id != "" {
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
}
}
sort.Strings(why)
return why
}
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
// one holding the module that has neither finished sending it nor sent it here first, and has not
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
// per machine.
func planStillToSend(plans []inventory.Plan, module, node string) string {
for _, p := range plans {
s, holds := p.Modules[module]
if !p.Open() || !holds {
continue
}
if s == nil {
return p.ID
}
if s.SentAt != nil || s.State == "failed" {
continue
}
first := false
for _, n := range s.First {
if n == node {
first = true
}
}
if !first {
return p.ID
}
}
return ""
}
func fromBuild(was string, carried bool) string {
if !carried {
return "(never sent it) "
}
return "from " + buildName(was) + " "
}
func buildName(commit string) string {
if commit == "" {
return "a build with no source"
}
return shortCommit(commit)
}
// heldMachines reads, for each machine named, why a push that did not name it must not send it
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
// to the send, which says why.
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
// One reading of the mesh for every machine this looks at.
ctx = withInputCache(ctx)
out := map[string][]string{}
if len(names) == 0 {
return out, nil
}
inv := open.inventory
current, err := inv.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, err
}
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, err
}
for _, node := range names {
plan, _, err := resolveNode(ctx, open, node)
if err != nil {
continue
}
modules := make([]string, 0, len(plan.Modules))
for _, m := range plan.Modules {
modules = append(modules, m.Module)
}
sent, known, err := inv.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
// A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it
// runs.
same := map[string]string{}
for m, was := range sent {
same[m] = was
if f.identical(m, was, current[m].Commit) {
same[m] = current[m].Commit
}
}
// A move under a module's own `record` that recreates nothing and moves no data goes as a module that rolls
// out goes (novox/hq ADR 0377): a walk's to send, not a person's.
here, copied := current, false
for m, was := range same {
if now, held := current[m]; held && now.OwnRecord() && !f.identical(m, was, now.Commit) && f.rolls(m, was) {
if !copied {
copied = true
here = make(map[string]inventory.CurrentBuild, len(current))
for k, v := range current {
here[k] = v
}
}
now.RollOut = true
here[m] = now
}
}
why := heldBack(node, modules, same, known, here, plans)
// And a build no gate has seen is not carried by a send that does not judge it (ADR 0236).
for _, mv := range f.moves(node, modules, same, known, false) {
if planStillToSend(plans, mv.Module, node) == "" {
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
}
}
if len(why) > 0 {
sort.Strings(why)
out[node] = why
}
}
return out, nil
}
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
func sayHeld(w io.Writer, node string, why []string) {
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
"grant from this push among it — waits with it. `push %s` sends it\n",
node, strings.Join(why, "; "), node)
}
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
//
// `handled` is every machine already sent or already said; it is not considered again. Answers the
// machines that could not be composed, as refusals.
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
keep keeping, w io.Writer) ([]string, error) {
inv := open.inventory
var refusals []string
// Bounded by the node count: a node is marked handled the round it is considered and is never
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
// cascade legitimately still converging, so it is said rather than passed over in silence.
rounds := 0
for {
would, err := wouldSend(ctx, open, nodes)
if err != nil {
return refusals, err
}
behind, err := inv.Waiting(ctx, would)
if err != nil {
return refusals, err
}
var also []string
for _, m := range behind {
if !handled[m.Node] {
also = append(also, m.Node)
}
}
if len(also) == 0 {
return refusals, nil
}
if rounds++; rounds > len(nodes) {
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
"should not happen; run `push --behind` to finish\n",
rounds-1, strings.Join(also, ", "))
return refusals, nil
}
sort.Strings(also)
held, err := heldMachines(ctx, open, also)
if err != nil {
return refusals, err
}
var sending []string
for _, name := range also {
// Every candidate this round is marked handled — the sent ones so they are not
// re-listed, the held ones because they stay held, and the refused ones so a machine
// that cannot be composed does not make the loop spin on it for ever.
handled[name] = true
if why, isHeld := held[name]; isHeld {
sayHeld(w, name, why)
continue
}
sending = append(sending, name)
}
if len(sending) == 0 {
continue
}
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
// Held for this round only, and after the last round's were given back, so two pushes
// cascading into each other's machines never each wait on the other.
refused, err := sendRound(ctx, open, sending, compose, d, holder, keep)
refusals = append(refusals, refused...)
if err != nil {
return refusals, err
}
}
}
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
// is left out of it said, and its declaration allocated.
func composeForPush(open *stores) func(held context.Context, node string) (sendable, error) {
return func(held context.Context, node string) (sendable, error) {
// Read afresh for each machine: the send before it made and recorded things.
held = freshInput(held)
plan, _, err := resolveNode(held, open, node)
if err != nil {
return sendable{}, err
}
reportUnhostable(node, plan)
reportKept(held, plan)
declared, err := composeSend(held, open, node)
if err == nil {
reportLeftOut(node, declared)
}
return declared, err
}
}
cmd/mesh-controller/held_back_integration_test.go
-174
View File
@@ -1,174 +0,0 @@
//go:build integration
package main
import (
"bytes"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
asked := time.Now().Add(-time.Hour)
aResolver(t, open, "c1c1c1c1c1", asked)
for _, node := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
t.Fatal(err)
}
}
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
t.Fatal(err)
}
compose := composeForPush(open)
d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
t.Fatal(err)
}
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
}
digestOfLaptop := func() string {
sent, err := inv.Outstanding(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
return sent
}
before := digestOfLaptop()
// The change merges; the policy is record. `push anchor` sends the anchor...
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
d.declared = nil
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
t.Fatal(err)
}
// ...and its cascade leaves the laptop, saying so.
var said bytes.Buffer
d.declared = nil
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
if err != nil || len(refused) != 0 {
t.Fatalf("the cascade failed: %v %v", refused, err)
}
if len(d.declared) != 0 {
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
}
if digestOfLaptop() != before {
t.Fatal("the laptop's last send moved: it was sent the held build")
}
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
"upgrade policy records", "`push laptop` sends it"} {
if !strings.Contains(said.String(), want) {
t.Errorf("the push did not say %q:\n%s", want, said.String())
}
}
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
// is held, and that what else it is owed waits with it.
aThirdMachine(t, open)
d.declared = nil
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
t.Fatal(err)
}
d.declared = nil
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", nil, &said); err != nil {
t.Fatal(err)
}
if len(d.declared) != 0 || digestOfLaptop() != before {
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
}
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
t.Fatalf("the push did not say both:\n%s", said.String())
}
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
// (novox/hq ADR 0236).
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
t.Fatal(err)
}
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", nil, &said); err != nil {
t.Fatal(err)
}
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
}
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
compose, d, "", nil, &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
}
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
t.Fatalf("the new send did not record the new build: %v", builds)
}
}
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
t.Fatal(err)
}
compose := composeForPush(open)
d := &recordedDelivery{inv: inv}
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
t.Fatal(err)
}
// `push spare`, the machine just placed: the others' peers change with it.
aThirdMachine(t, open)
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
t.Fatal(err)
}
d.declared = nil
var said bytes.Buffer
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
}
if strings.Contains(said.String(), "was not sent") {
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
}
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
record, err := inv.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
t.Fatal(err)
}
d.declared = nil
said.Reset()
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
t.Fatal(err)
}
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
// question.
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
}
}
cmd/mesh-controller/held_back_test.go
-168
View File
@@ -1,168 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
// else that moved is still a consequence the push sends (ADR 0083).
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
current := map[string]inventory.CurrentBuild{
"resolver": {Commit: "c2c2c2c2c2"},
"agent": {Commit: "a2", RollOut: true},
"network": {},
}
modules := []string{"network", "resolver", "agent"}
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
// A module whose policy records moved: held, naming it, both builds and why.
why := heldBack("laptop", modules, sent, true, current, nil)
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
!strings.Contains(why[0], "upgrade policy records") {
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
}
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
sent["resolver"] = "c2c2c2c2c2"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
t.Fatalf("a machine whose builds are all current was held: %v", why)
}
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
sent["agent"] = "a1"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
}
// The last send's builds are not known: held whole.
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
!strings.Contains(why[0], "not known") {
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
}
// A module the machine was never sent, under a recording policy: held, and said so.
delete(sent, "resolver")
sent["agent"] = "a2"
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
t.Fatalf("a module never sent under a recording policy: %v", why)
}
}
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
// naming some other machine must not send them for it.
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
at := time.Now()
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
sent := map[string]string{"agent": "a1"}
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
t.Fatalf("a machine the plan has not reached was not held: %v", why)
}
// The first machine itself was sent by the plan: not held by it.
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
t.Fatalf("the plan's first machine was held: %v", why)
}
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
Modules: map[string]*inventory.PlanModule{"agent": s}}}
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
t.Errorf("%s: not held: %v", what, why)
}
}
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
for what, plans := range map[string][]inventory.Plan{
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"agent": {State: "failed"}}}},
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
"agent": {State: "built"}}}},
} {
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
t.Errorf("%s: held: %v", what, why)
}
}
}
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
// reads the machine as current — and writes down which machines it declared.
type recordedDelivery struct {
inv *inventory.Inventory
declared []string
}
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
r.declared = append(r.declared, s.node)
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
}
// aResolver is a module built from a repository, at a commit, with something on the machine that
// says which build it is.
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
t.Helper()
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
}}
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
}
// A third machine on the private network: once it is sent, every other machine's peers change with
// it, which is a consequence a push must still send — no build moved.
func aThirdMachine(t *testing.T, open *stores) {
t.Helper()
ctx := t.Context()
record, err := open.inventory.AddNode(ctx, "spare")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
t.Fatal(err)
}
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
{"name": "systemd", "present": true}}})
if err != nil {
t.Fatal(err)
}
var profile map[string]any
if err := json.Unmarshal(reported, &profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
t.Fatal(err)
}
}
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
// for another reason is sent by a push that names someone else.
cmd/mesh-controller/held_moves.go
-198
View File
@@ -1,198 +0,0 @@
package main
import (
"context"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
//
// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that
// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and
// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on
// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way.
// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the
// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends
// when the machine no longer waits, whoever pushed it.
//
// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has
// its own row (S17).
// kindMovesHeld is S24's kind: machine.<node>.moves-held.
const kindMovesHeld = "moves-held"
// heldFacts is one machine and the moves of recorded modules that wait there for a person.
type heldFacts struct {
node string
moves []heldMove
}
// gatherHeldMoves is, per machine that holds at least one move no person chose to hold, all its held moves, by machine.
func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) {
nodes, err := inv.Nodes(ctx)
if err != nil {
return nil, fmt.Errorf("the machines cannot be read: %w", err)
}
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, err
}
var out []heldFacts
for _, n := range nodes {
moves, err := heldMovesOn(ctx, inv, f, n.Name)
if err != nil {
return nil, err
}
h := heldFacts{node: n.Name, moves: moves}
if len(h.asked()) > 0 {
out = append(out, h)
}
}
return out, nil
}
// asked are the moves no person chose to hold, and chosen the ones a person did (a module set to record with a why, the bus).
func (h heldFacts) asked() []heldMove { return h.byChoice(false) }
func (h heldFacts) chosen() []heldMove { return h.byChoice(true) }
func (h heldFacts) byChoice(chosen bool) []heldMove {
var out []heldMove
for _, m := range h.moves {
if m.Chosen == chosen {
out = append(out, m)
}
}
return out
}
// modules are the held moves' modules, in order, of all of them.
func (h heldFacts) modules() []string { return moduleNames(h.moves) }
func moduleNames(moves []heldMove) []string {
out := make([]string, 0, len(moves))
for _, m := range moves {
out = append(out, m.Module)
}
return out
}
// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push
// names every module whose move waits there as one that may move its data: the operator's Approve is the word
// ADR 0217 asks for, and the ask says so. Offered only where no person's own hold is on the machine too: a push
// composes every recorded module at its current build, so it would send that move as well, which the ask never named
// (ADR 0259 §6), and hold the whole machine if that module's data would move.
func (h heldFacts) releaseAction() conditions.Action {
return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove,
Arguments: map[string]string{"node": h.node, "move": strings.Join(moduleNames(h.asked()), ","), "why": "",
"cause": conditions.CauseOperatorAnswer}}
}
// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does.
func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) {
names := namesWords(moduleNames(h.asked()), 3)
headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax)
recreates, data := false, false
for _, m := range h.asked() {
switch {
case strings.Contains(m.Why, "recreates"):
recreates = true
default:
data = true
}
}
why := "would change what it runs"
switch {
case recreates && data:
why = "would restart a container or touch its data"
case recreates:
why = "would restart one of its containers"
case data:
why = "would touch its files or data"
}
explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+
"it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+
"has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax)
needs = "release them " + FromMeshMCPServer
if chosen := h.chosen(); len(chosen) > 0 {
// A push of the machine would send the move a person chose to hold as well: no answer is offered here.
explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+
"it waits for you. %s is also held there by your own choice, and a push of %s would send that too, so "+
"nothing is offered here: the push is yours at the controller's terminal.",
names, why, h.node, conditions.Capital(namesWords(moduleNames(chosen), 3)), h.node), conditions.ExplanationMax)
needs = fmt.Sprintf("push %s at the controller's terminal, once you have decided about %s.", h.node,
namesWords(moduleNames(chosen), 3))
}
resolved = "Resolved: " + names + " is no longer waiting on " + h.node
return
}
// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the
// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts.
func watchMovesHeld(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, h := range f.held {
var said []string
for _, m := range h.moves {
said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why))
}
headline, explanation, needs, resolved := heldWords(h)
if c := h.chosen(); len(c) > 0 {
said = append(said, "held by a person's own choice: "+strings.Join(moduleNames(c), ", "))
}
var actions []conditions.Action
if len(h.chosen()) == 0 {
actions = []conditions.Action{h.releaseAction()}
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld,
Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them",
h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node),
Said: strings.Join(said, "; "),
Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved,
Actions: actions})
}
return out
}
// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a
// person: "module on machine", sorted. The plan's own record says the move waited when it was judged
// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older
// than the build now held is not named.
func planHeld(p inventory.Plan, held []heldFacts) []string {
var out []string
for _, h := range held {
for _, m := range h.moves {
s := p.Modules[m.Module]
if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) {
continue
}
out = append(out, m.Module+" on "+h.node)
}
}
sort.Strings(out)
return out
}
// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a
// plan whose module waits for a push is not plainly "done".
func heldSaid(p inventory.Plan, held []heldFacts) string {
waiting := planHeld(p, held)
if len(waiting) == 0 {
return ""
}
return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", "))
}
// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before.
func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts {
held, err := gatherHeldMoves(ctx, inv)
if err != nil {
return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read
}
return held
}
cmd/mesh-controller/held_moves_integration_test.go
-92
View File
@@ -1,92 +0,0 @@
//go:build integration
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
func inventoryUpgrade(why string) inventory.Upgrade { return inventory.Upgrade{Why: why, By: "jochen"} }
func mustMoveFacts(t *testing.T, rm *recordedMesh) moveFacts {
t.Helper()
f, err := readMoveFacts(t.Context(), rm.open.inventory)
if err != nil {
t.Fatal(err)
}
return f
}
// 1732/AC8 (integration): a mesh running mailu at c1 on two machines, whose registered c2 recreates a container, is read
// as two machines each holding mailu's move for a person; a bundle-only c3 on top changes nothing of it; after a machine
// is sent c2 (as a push records it) it is no longer among them, and plans that carried c2 say nothing more of it.
func TestMachinesHoldingAMoveForAPersonAreReadFromTheRecords(t *testing.T) {
rm := aRecordedMesh(t, aMailu(true))
c2 := aMailu(true)
resourceOf(&c2, "imap")["image"] = "registry.invalid/imap@sha256:2"
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
ctx := t.Context()
held, err := gatherHeldMoves(ctx, rm.open.inventory)
if err != nil {
t.Fatal(err)
}
if len(held) != 2 || held[0].node != "anchor" || held[1].node != "laptop" || len(held[0].moves) != 1 ||
held[0].moves[0].Module != "mailu" || held[0].moves[0].To != "c2" || !strings.Contains(held[0].moves[0].Why, "recreates") {
t.Fatalf("held %+v", held)
}
if err := rm.open.inventory.RecordSent(ctx, nodeID(t, rm.open, "anchor"), "d-anchor-c2", map[string]string{"mailu": "c2"}); err != nil {
t.Fatal(err)
}
held, err = gatherHeldMoves(ctx, rm.open.inventory)
if err != nil || len(held) != 1 || held[0].node != "laptop" {
t.Fatalf("after anchor was sent c2: %+v (%v)", held, err)
}
}
// 1732/AC9 (integration): a module a person set to record is no ask: they chose the hold.
func TestAHoldAPersonChoseIsNotAsked(t *testing.T) {
rm := aRecordedMesh(t, aMailu(false))
c2 := aMailu(false)
c2.Bundles[0].Digest = "sha256:bbb"
rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventoryUpgrade("the migration week")); err != nil {
t.Fatal(err)
}
held, err := gatherHeldMoves(t.Context(), rm.open.inventory)
if err != nil || len(held) != 0 {
t.Fatalf("a machine holding only a person's own hold is asked: %+v (%v)", held, err)
}
if moves, _ := heldMovesOn(t.Context(), rm.open.inventory, mustMoveFacts(t, rm), "anchor"); len(moves) != 1 || !moves[0].Chosen {
t.Fatalf("status still names it, marked as chosen: %+v", moves)
}
}
// 1742/AC4 (integration): a registered build naming an image of the store that the store lacks is read as a missing
// image on each machine that runs the module; one the store holds is not; and what the store holds is not asked twice.
func TestARegisteredBuildNamingAnImageTheStoreLacksIsRead(t *testing.T) {
m := aMailu(true)
resourceOf(&m, "imap")["image"] = "artifact-store://mailu/imap@sha256:" + strings.Repeat("e", 64)
rm := aRecordedMesh(t, m)
asked := 0
lacking := func(_ context.Context, reference string) (bool, error) { asked++; return false, nil }
imagesHeld.Lock()
imagesHeld.held = map[string]bool{}
imagesHeld.Unlock()
got, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking)
if err != nil || len(got) != 2 || got[0].Module != "mailu" || got[0].Node != "anchor" || got[1].Node != "laptop" {
t.Fatalf("missing %+v (%v)", got, err)
}
holding := func(_ context.Context, reference string) (bool, error) { asked++; return true, nil }
if got, err := gatherMissingImages(t.Context(), rm.open.inventory, holding); err != nil || len(got) != 0 {
t.Fatalf("an image the store holds is missing: %+v (%v)", got, err)
}
before := asked
if _, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking); err != nil || asked != before {
t.Fatalf("an image the store was found to hold was asked about again (%d, %v)", asked-before, err)
}
}
cmd/mesh-controller/held_moves_test.go
-237
View File
@@ -1,237 +0,0 @@
package main
import (
"context"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words.
func aHeldMailu() heldFacts {
return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb",
Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}}
}
func heldConditionOn(f *signalFacts) conditions.Condition {
o := watchMovesHeld(f)[0]
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
}
// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with
// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there.
func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) {
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
f.held = []heldFacts{aHeldMailu()}
got := watchMovesHeld(f)
if len(got) != 1 {
t.Fatalf("%+v", got)
}
o := got[0]
if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning ||
o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" {
t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key())
}
if len(o.Actions) != 1 {
t.Fatalf("actions: %+v", o.Actions)
}
a := o.Actions[0]
if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove ||
a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
t.Fatalf("the action is not the push of novox: %+v", a)
}
if _, takes := a.Arguments["why"]; !takes {
t.Fatal("the push takes a why, which the asker fills in with the operator's answer")
}
for _, want := range []string{"mailu", "novox"} {
if !strings.Contains(o.Headline+o.Explanation, want) {
t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation)
}
}
if !strings.Contains(o.Said, "recreates 1 of mailu's 12") {
t.Errorf("the evidence does not carry the walk's own words: %q", o.Said)
}
if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax {
t.Errorf("headline: %q", o.Headline)
}
}
// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent.
func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) {
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
if got := watchMovesHeld(f); len(got) != 0 {
t.Fatalf("%+v", got)
}
}
// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the
// operator's warrant calls the controller's own push with the machine and the modules, once.
func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) {
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
f.held = []heldFacts{aHeldMailu()}
c := heldConditionOn(f)
r := newAskerRig(t)
r.open = []conditions.Condition{c}
if err := r.a.reconcile(context.Background()); err != nil {
t.Fatal(err)
}
sent := r.asksSent(t)
if len(sent) != 1 || len(sent[0].Options) != 1 {
t.Fatalf("asked %+v", sent)
}
o := sent[0].Options[0]
if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" {
t.Errorf("the option is %+v", o)
}
if err := sent[0].Check(r.now); err != nil {
t.Errorf("the ask is refused as made: %v", err)
}
w := r.warrantFor(t, c.Key, "Release")
answerWith(t, r, w)
answerWith(t, r, w)
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") ||
!strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) ||
!strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) ||
!strings.Contains(r.called[0], "chose Release") {
t.Fatalf("called %v", r.called)
}
}
// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them.
func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) {
for _, verb := range []string{"push", "upgrade"} {
granted := false
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb)
}
if !granted {
t.Errorf("the controller may not act on a warrant for %s", verb)
}
}
}
func aDoneMailuPlan() inventory.Plan {
return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone,
Tiers: [][]string{{"a"}, {"b"}, {"mailu"}},
Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc",
Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}}
}
// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine
// no longer holds it, or holds another build of the module, the line says nothing more.
func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) {
p := aDoneMailuPlan()
line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()})
if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") {
t.Errorf("the line: %q", line)
}
if said := heldSaid(p, nil); said != "" {
t.Errorf("a plan whose moves were sent says %q", said)
}
other := aHeldMailu()
other.moves[0].To = "9999999ffff"
if said := heldSaid(p, []heldFacts{other}); said != "" {
t.Errorf("a plan older than the build now held says %q", said)
}
p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy}
if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" {
t.Errorf("a move the plan sent says %q", said)
}
}
// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers
// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant.
func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) {
saved := backlogNow
t.Cleanup(func() { backlogNow = saved })
backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person"
backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}}
got := backlogObservation()
if len(got) != 1 || len(got[0].Actions) != 1 {
t.Fatalf("raised %+v", got)
}
a := got[0].Actions[0]
if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove ||
a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
t.Fatalf("the action: %+v", a)
}
o := got[0]
c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
r := newAskerRig(t)
r.open = []conditions.Condition{c}
if err := r.a.reconcile(context.Background()); err != nil {
t.Fatal(err)
}
sent := r.asksSent(t)
if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" {
t.Fatalf("asked %+v", sent)
}
w := r.warrantFor(t, c.Key, "Release")
answerWith(t, r, w)
answerWith(t, r, w)
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") ||
!strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") {
t.Fatalf("called %v", r.called)
}
}
// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for
// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is.
func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) {
saved := backlogNow
t.Cleanup(func() { backlogNow = saved })
now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC)
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding,
Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute),
Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}},
Note: refusalNote(0, errors.New("a build waits there for its gate: novox"))}
backlogNow.held = ""
if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") {
t.Errorf("the wait is not said: %q", line)
}
backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person"
line := planLine(p, now)
if strings.Contains(line, "LATE") || strings.Contains(line, "building") ||
!strings.Contains(line, "until a person releases it") {
t.Errorf("a plan held behind a failed release reads %q", line)
}
}
// 1732/AC10: a machine that also holds a move a person chose to hold gets no Release, since a push would send that move
// too; the condition names both and says the push is the controller's terminal. A machine with only asked moves keeps it.
func TestAMachineAlsoHoldingAPersonsOwnHoldIsOfferedNoRelease(t *testing.T) {
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
h := aHeldMailu()
h.moves = append(h.moves, heldMove{Module: "nftables", From: "n1aaaa", To: "n2bbbb",
Why: "waits: a person set it to record, and holds every move of it until they push", Chosen: true})
f.held = []heldFacts{h}
got := watchMovesHeld(f)
if len(got) != 1 || len(got[0].Actions) != 0 {
t.Fatalf("%+v", got)
}
o := got[0]
text := o.Headline + o.Explanation + o.Needs + o.Summary + o.Said
for _, want := range []string{"mailu", "nftables", "terminal", "push novox"} {
if !strings.Contains(text, want) {
t.Errorf("does not say %q: %q", want, text)
}
}
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
if why, ok := conditions.PlainWords(w, "novox"); !ok {
t.Errorf("not plain: %s", why)
}
f.held = []heldFacts{aHeldMailu()}
if got := watchMovesHeld(f); len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["move"] != "mailu" {
t.Fatalf("a machine with only asked moves: %+v", got)
}
}
cmd/mesh-controller/helpers_kept_integration_test.go
+80
View File
@@ -0,0 +1,80 @@
//go:build integration
package main
import (
"context"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/builderkeys"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// Helpers the integration tests of the deleted plan machinery shared with tests that remain.
// noListener hears no node: these tests send merges only.
type noListener struct{ link.Listener }
func becomes(t *testing.T, within time.Duration, what string, is func() bool) {
t.Helper()
deadline := time.Now().Add(within)
for time.Now().Before(deadline) {
if is() {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatalf("%s did not happen within %s", what, within)
}
// noBuildAsked once failed a test that asked build-agent for a build; nothing can now, so it says nothing.
func noBuildAsked(t *testing.T) { t.Helper() }
func twoTiers(t *testing.T, open *stores) {
t.Helper()
for _, name := range []string{"a", "b"} {
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
t.Fatal(err)
}
}
}
func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) {
t.Helper()
entries, err := inv.Catalogued(t.Context())
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
if e.Manifest.Module == module {
return e, true
}
}
return inventory.Entry{}, false
}
// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by
// module.
type planKeysOf struct {
values map[string]sdkdelivery.Module // by module + "@" + commit
keys map[string]builderkeys.Key
}
func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) {
if v, ok := k.values[module+"@"+commit]; ok {
return v, nil
}
return sdkdelivery.Module{}, builderkeys.ErrNoValue
}
func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
if key, ok := k.keys[module]; ok && key.Fingerprint == fp {
return key, nil
}
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
}
cmd/mesh-controller/helpers_kept_test.go
+45
View File
@@ -0,0 +1,45 @@
package main
import (
"context"
"encoding/json"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// Helpers the tests of the deleted plan machinery shared with tests that remain.
func nodeID(t *testing.T, open *stores, name string) string {
t.Helper()
n, err := open.inventory.NodeByName(context.Background(), name)
if err != nil {
t.Fatal(err)
}
return n.ID
}
func bodyKeys(t *testing.T, s sendable) map[string]any {
t.Helper()
raw, err := s.Body()
if err != nil {
t.Fatal(err)
}
var keys map[string]any
if err := json.Unmarshal(raw, &keys); err != nil {
t.Fatal(err)
}
return keys
}
// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that
// announce a merge from it.
func theDeliveryClaims(context.Context) ([]inventory.Entry, error) {
return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil
}
func claimingTheSeat(module string) inventory.Entry {
return inventory.Entry{Manifest: catalogue.Manifest{Module: module,
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
}
cmd/mesh-controller/hold_test.go
-47
View File
@@ -1,47 +0,0 @@
//go:build integration
package main
import (
"context"
"errors"
"testing"
"time"
)
// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled
// and a send that fails leave nobody waiting for the node.
func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
unmarshallable := func(context.Context, string) (sendable, error) {
return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil
}
plain := func(context.Context, string) (sendable, error) {
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
}
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
fine := &recordingDelivery{}
for name, round := range map[string]func() error{
"a body that cannot be marshalled": func() error {
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "", nil)
return err
},
"a send that fails": func() error {
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "", nil)
return err
},
} {
if err := round(); err == nil {
t.Fatalf("%s was not an error", name)
}
waiting, cancel := context.WithTimeout(ctx, 2*time.Second)
release, err := open.inventory.HoldNodes(waiting, []string{"anchor"})
cancel()
if err != nil {
t.Fatalf("after %s the node is still held: %v", name, err)
}
release()
}
}
cmd/mesh-controller/issue201_integration_test.go
-181
View File
@@ -1,181 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
func core201(t *testing.T) *stores {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
record := func(commit string, asked time.Time) inventory.Build {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
return b
}
before := record("c1aaaaaaaaaa", old)
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
record("c2bbbbbbbbbb", now)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
t.Fatal(err)
}
if err := inv.RestoreModule(ctx, before); err != nil {
t.Fatal(err)
}
wasMoves := machineMoves
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
t.Cleanup(func() { machineMoves = wasMoves })
return open
}
func TestReplay201(t *testing.T) {
open := core201(t)
ctx := t.Context()
k, _ := withConditionsInMemory(t)
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
}
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
}
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
if err := applyKept(keepingRecorded(judged), open, "anchor", keeps, shelf); err != nil || shelf["mesh-controller"].Version != "1" {
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
}
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
}
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
}
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
if _, found, _ := k.Get(ctx, key); !found {
t.Fatal("a send that kept the newer controller cleared the condition")
}
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
if _, found, _ := k.Get(ctx, key); found {
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
}
}
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
open := core201(t)
ctx := t.Context()
k, _ := withConditionsInMemory(t)
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
t.Fatal(err)
}
healer := withScope(ctx, sendScope{})
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
t.Fatalf("the healer's send is made: %v %v", kept, err)
}
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
}
if open, _ := k.Open(ctx); len(open) != 0 {
t.Fatalf("a completed put-back raised a condition: %+v", open)
}
}
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
open := core201(t)
ctx := t.Context()
withConditionsInMemory(t)
inv := open.inventory
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
for _, b := range []struct {
commit string
at time.Time
}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
Asked: b.at}); err != nil {
t.Fatal(err)
}
}
if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
t.Fatal(err)
}
// The send as it is made: the controller kept at c2, the app moved to the record.
wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
}
t.Cleanup(func() { sendRollout = wasSend })
moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
if err != nil || len(sent) != 1 {
t.Fatalf("the gated send is made: %v %v", sent, err)
}
if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
}
builds, _, err := inv.SentBuilds(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
g := &inventory.PlanGate{Carried: moves}
f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
for _, m := range []string{"app", "mesh-controller"} {
if why, superseded := f.supersededOn(m, "anchor"); superseded {
t.Fatalf("the gate is superseded by its own send: %s", why)
}
}
}
cmd/mesh-controller/issue201_test.go
-93
View File
@@ -1,93 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/lease"
)
// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
// behind the build its machine was last sent: the send keeps it there, and carries the rest.
// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
// their gate.
func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
return func(module, commit string) (askedBuild, bool, error) {
at, known := table[module+"@"+commit]
b := askedBuild{asked: at}
for _, f := range failed {
if f == module+"@"+commit {
b.failedGate = true
}
}
return b, known, nil
}
}
func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
old, now := t0, t0.Add(time.Minute)
table := map[string]time.Time{
"mesh-controller@c1": old, "mesh-controller@c2": now,
"mesh-host@h1": old, "mesh-host@h2": now,
"app@a1": old, "app@a2": now,
}
sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
if err != nil {
t.Fatal(err)
}
if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
}
if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
}
// The record forward, the same build, or a build the records do not hold: nothing kept.
for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
if err != nil || len(found) != 0 {
t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
}
}
// A sent build that failed its gate is never ahead: the record put back before it is sent.
if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
found[0].module != "mesh-host" {
t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
}
// A gate's own put-back carries its own modules back, and nothing else.
putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
}
if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
askedIn(table)); len(found) != 2 {
t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
}
}
// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
// controller is the build before it, c1, which passed its gate when it rolled out.
// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
// as a person's push does, clears the condition.
// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
// failed its gate is never kept.
// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
cmd/mesh-controller/issue348_integration_test.go
-199
View File
@@ -1,199 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
"github.com/novox/mesh-controller/internal/link"
)
func TestReplay348(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv, k := open.inventory, conditionsFrom
say := func(at time.Time, n *link.NetworkHealth) {
t.Helper()
if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
t.Fatal(err)
}
}
network := func() (conditions.Condition, bool) {
t.Helper()
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
for _, c := range list {
if c.Key == "machine.anchor.network" {
return c, true
}
}
return conditions.Condition{}, false
}
// 10:58:45 — the second failing look: raised.
say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2)))
raised, ok := network()
if !ok {
t.Fatal("the resolver refusing on the control node raised nothing")
}
time.Sleep(5 * time.Millisecond)
sent := time.Now().UTC()
time.Sleep(5 * time.Millisecond)
// 10:59:42 — the restarted engine's first statement: one look failed, a second decides.
say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1)))
if _, ok := network(); !ok {
t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " +
"said the fault was gone while it still refused")
}
// 11:00:23 — its second look: unhealthy again, the same raising.
say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2)))
again, ok := network()
if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 {
t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count)
}
// The gate on the control node, for a build sent after the fault began.
open2, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
f := gateFacts{judged: true, open: open2}
if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 {
t.Fatalf("a fault from before the send held the build: %+v", w)
}
// Decided healthy: cleared.
say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0}))
if c, ok := network(); ok {
t.Fatalf("a statement that decides the names healthy left %s open", c.Key)
}
}
// namesRefused is the control node's names part as its node-engine said it in the outage: its own
// resolver, at its own address, refusing.
func namesRefused(state string, streak int) link.NetworkPart {
p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak}
if state == link.StateUnhealthy {
p.Reason = "1 of its 2 resolvers do not answer as the mesh's do"
p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused"
p.Toward = []string{"10.77.0.1"}
}
return p
}
func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth {
state := link.StateHealthy
for _, p := range parts {
switch {
case p.State == link.StateUnhealthy:
state = link.StateUnhealthy
case p.State == link.StateUnknown && state == link.StateHealthy:
state = link.StateUnknown
}
}
return &link.NetworkHealth{State: state, Since: h0, Parts: parts}
}
// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared
// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review
// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings.
func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) {
since := h0
network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition {
raised := since.Add(time.Minute)
if len(gaps) > 0 {
raised = gaps[len(gaps)-1].Reopened
}
return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"},
Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised}
}
held := func(c conditions.Condition) bool {
return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true,
open: []conditions.Condition{c}}).whole != ""
}
// The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it.
flapped := network(since.Add(-49*time.Second),
conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)})
if held(flapped) {
t.Fatal("a fault there at the send, flapping after it, held the machine")
}
// Recovered before the send, broken again after it: the send's.
recovered := network(since.Add(-time.Hour),
conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)})
if !held(recovered) {
t.Fatal("a machine recovered at the send and broken after it passed the gate")
}
// An older gap, before the send, and the fault there at the send: not the send's.
twice := network(since.Add(-time.Hour),
conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)},
conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)})
if held(twice) {
t.Fatal("a fault there at the send, with an older gap, held the machine")
}
// Raised after the send, never cleared: the send's.
if !held(network(time.Time{})) {
t.Fatal("a fault raised after the send held nothing")
}
// And a module's own, through judgeHealth.
at := since.Add(2 * time.Minute)
g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor",
Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{},
served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{
Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing",
First: since.Add(-time.Hour), Raised: since.Add(time.Minute),
Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}}
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") {
t.Fatalf("a module's own fault there at the send: %s", why)
}
g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second)
if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") {
t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why)
}
}
// Only an undecided part holds a condition that names it; a condition about another part clears, and a
// statement unknown as a whole holds every part (review of PR 179, A4). Pure.
func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) {
f := netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1},
inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}),
"laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}),
"spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}),
"other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}),
})
u := undecidedParts(f)
if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] ||
!u["other"][link.PartTunnel] || u["other"]["*"] {
t.Fatalf("undecided: %v", u)
}
about := func(machine, said string, also ...string) conditions.Condition {
return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine,
Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}}
}
for _, c := range []struct {
c conditions.Condition
held bool
}{
{about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true},
{about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false},
{about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false},
{about("spare", "route since …: no default route"), true},
{about("hub", "anchor: names: refused", "anchor"), true},
{about("hub", "anchor: tunnel: no handshake", "anchor"), false},
} {
if got := heldUndecided(c.c, u); got != c.held {
t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held)
}
}
}
cmd/mesh-controller/issue352_integration_test.go
-227
View File
@@ -1,227 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
func TestReplay352(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
judgeLeavesAsTheCore(t)
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
}
func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied
// 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on.
carried, _, _ := inv.SentBuilds(ctx, "anchor")
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil {
t.Fatal(err)
}
reports, _ := inv.LastReports(ctx)
for _, r := range reports {
if r.Node == "anchor" && r.Current {
t.Fatal("the fixture's newer send reads as reported")
}
}
gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here
for i := 0; i < 4; i++ {
advancePlans(ctx, b.open)
}
p := b.release(t)
if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") {
t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note)
}
if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") {
t.Fatalf("the gate does not keep what it sent: %+v", g)
}
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
t.Fatalf("app was put back to %s", current["app"].Commit)
}
}
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
advancePlans(ctx, b.open) // the release judges app c2 on anchor
_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
}
if len(b.sent) != 1 {
t.Fatalf("sent %v", b.sent)
}
}
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
advancePlans(ctx, g.open) // anchor is sent app c2 first
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil {
t.Fatal(err)
}
gateEvery = 0
advancePlans(ctx, g.open)
p := g.plan(t)
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
t.Fatalf("the plan is %s: %s", p.State, p.Note)
}
// Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no
// rollback (a release may walk what the other send left waiting on anchor; that is not a put-back).
if r := p.Modules["app"].Gate.Rollback; r != "" {
t.Fatalf("a superseded judging made a rollback: %q", r)
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
t.Fatalf("app was put back to %s", current["app"].Commit)
}
if failed, _ := inv.GateFailed(ctx, "build-2"); failed {
t.Fatal("a superseded build was marked failed")
}
}
func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
advancePlans(ctx, b.open)
// Another send moves app on anchor to a build this gate does not judge.
carried, _, _ := inv.SentBuilds(ctx, "anchor")
carried["app"] = "c3"
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil {
t.Fatal(err)
}
gateEvery = 0
advancePlans(ctx, b.open)
p := b.release(t)
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
t.Fatalf("the release is %s: %s", p.State, p.Note)
}
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent)
}
if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found {
t.Fatal("a superseded judging kept a verdict")
}
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
t.Fatalf("app was put back to %s", current["app"].Commit)
}
}
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
keeper, _ := withConditionsInMemory(t)
told := &conditions.Told{}
was := doctorFrom
doctorFrom = &doctor{open: open, keeper: keeper, teller: told}
t.Cleanup(func() { doctorFrom = was })
wasSend := sendRollout
var sent [][]string
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
sent = append(sent, names)
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
build := func(id, commit, digest string, asked time.Time) inventory.Build {
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit})
b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".",
Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle,
Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit},
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil {
t.Fatal(err)
}
return b
}
previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour))
failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute))
if versionOfBuild(previous) != strings.Repeat("1", 12) {
t.Fatalf("the build's version is %q", versionOfBuild(previous))
}
applied, err := inv.SchemaApplied(ctx)
if err != nil || applied < 87 {
t.Fatalf("the store's schema reaches %d (%v)", applied, err)
}
// The build before never recorded what it reads: not proved, refused.
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") {
t.Fatalf("an unknown reach: %v %q", ok, why)
}
// It reads less than the store has: refused, naming both.
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil {
t.Fatal(err)
}
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") {
t.Fatalf("a reach behind the store: %v %q", ok, why)
}
// Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent.
at := time.Now().Add(-5 * time.Minute)
state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at}
p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at,
State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply")
if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") {
t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note)
}
if len(sent) != 0 {
t.Fatalf("sent %v: nothing is put back", sent)
}
if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit {
t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit)
}
if marked, _ := inv.GateFailed(ctx, failed.ID); !marked {
t.Fatal("the failed build is not marked failed at its gate")
}
open2, _ := keeper.Open(ctx)
var found bool
for _, c := range open2 {
if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") {
found = true
}
}
if !found {
t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2)
}
// Reaching the store: allowed.
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil {
t.Fatal(err)
}
if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
t.Fatalf("a build that reads the whole schema was refused: %q", why)
}
// A build with no bundle to know it by: refused.
if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") {
t.Fatalf("a build without a bundle: %v %q", ok, why)
}
}
cmd/mesh-controller/issue352_test.go
-156
View File
@@ -1,156 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// novox/hq issue 352: on 2026-10-09 a release's gate on the control node read the machine's report against a
// newer send another plan had just made there — not against its own send — and failed three builds the
// machine had reported healthy ("has not reported on what it was sent"), put them back on every machine,
// to a controller older than the store's schema, and that controller then judged the newer plan's
// controller passed from the put-back build's health.
// TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports;
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
// to another build supersedes the judging: no verdict, nothing put back.
// A plan's own first send waits while a release judges the same module on that machine with another build.
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
// without its send reads the report against the send made last, as before. Pure.
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
for _, c := range []struct {
r inventory.Reported
want bool
}{
{inventory.Reported{Declared: "d-490", Current: false}, true},
{inventory.Reported{Declared: "d-491", ReportedSequence: 491, Current: true}, true},
{inventory.Reported{Declared: "d-489", ReportedSequence: 489, Current: false}, false},
{inventory.Reported{Declared: "other", ReportedSequence: 490}, true}, // the same sequence, said by another digest
{inventory.Reported{Declared: "d-495", ReportedSequence: 495, Current: true}, true}, // the last send: this one or a later one
{inventory.Reported{Declared: "", Current: false}, false},
} {
if got := sent.ReportsOn(c.r); got != c.want {
t.Errorf("%+v on %+v: %v", c.r, sent, got)
}
}
byDigest := inventory.SentDeclaration{Digest: "d-1"}
if !byDigest.ReportsOn(inventory.Reported{Declared: "d-1"}) || byDigest.ReportsOn(inventory.Reported{ReportedSequence: 5}) ||
!byDigest.ReportsOn(inventory.Reported{Current: true}) {
t.Fatal("a send kept without a sequence is matched by its digest and by the last send alone")
}
f := gateFacts{sent: map[string]inventory.SentDeclaration{"anchor": sent}}
if !f.reportedOn("anchor", inventory.Reported{Declared: "d-490"}) || f.reportedOn("anchor", inventory.Reported{Declared: "d-1"}) {
t.Fatal("a gate that kept its send read the report against something other than it")
}
if !f.reportedOn("laptop", inventory.Reported{Current: true}) || f.reportedOn("laptop", inventory.Reported{Current: false}) {
t.Fatal("a gate that did not keep its send does not read the report against the send made last")
}
// Through the merge plan's first-machine wait too.
at := time.Now()
state := inventory.PlanModule{First: []string{"anchor"}, FirstAt: &at,
Gate: &inventory.PlanGate{Machines: []string{"anchor"}, Sent: map[string]inventory.SentDeclaration{"anchor": sent}}}
reports := []inventory.Reported{{Node: "anchor", At: &at, Outcome: inventory.OutcomeApplied, Current: false, Declared: "d-490"}}
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting != "" || step.failed != "" {
t.Fatalf("the first machine's report on the plan's own send read as none: %+v", step)
}
reports[0].Declared = "d-480"
if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting == "" {
t.Fatalf("a report on an older send read as the plan's: %+v", step)
}
}
// A gate judges only the build the machine was last sent: last sent another build of the module, the
// judging is superseded, whatever the machine reports. Pure.
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
at := time.Now()
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
commits: map[string]string{"mesh-controller": "e6b00e2e"}, sentBuilds: map[string]map[string]string{"anchor": {"mesh-controller": "ef26d4cb"}}}
taken := at.Add(-30 * time.Second)
f.holder = &lease.Holder{Taken: taken, Health: &lease.Health{Ready: true}}
h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f)
if h != healthSuperseded || !strings.Contains(why, "ef26d4cb") || !strings.Contains(why, "e6b00e2e") {
t.Fatalf("a controller build the machine no longer runs: %v %q", h, why)
}
f.sentBuilds["anchor"]["mesh-controller"] = "e6b00e2e"
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
t.Fatalf("the build sent read as another: %q", why)
}
delete(f.sentBuilds, "anchor")
if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
t.Fatalf("a machine whose send is not known read as superseded: %q", why)
}
g := &inventory.PlanGate{To: "c2", Carried: []inventory.CarriedMove{{Module: "late", Node: "anchor", To: "c5"}}}
if got := judgedCommits(g, []judged{{"app", "anchor"}, {"late", "anchor"}}); got["app"] != "c2" || got["late"] != "c5" {
t.Fatalf("judged commits %v", got)
}
}
// A move of another build of a module to a machine where a release or a plan is judging that module
// waits for that judging; the same build to that machine is already there. Pure.
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
at := time.Now()
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
{Module: "mesh-controller", Node: "novox", From: "ef26d4cb", To: "2913c54c"}}}}}
merge := inventory.Plan{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
"app": {First: []string{"anchor"}, FirstAt: &at, Commit: "c2", Gate: &inventory.PlanGate{Machines: []string{"anchor"}}}}}
f := moveFacts{plans: []inventory.Plan{release, merge}}
for _, c := range []struct {
module, node, to, want string
}{
{"mesh-controller", "novox", "e6b00e2e", "release-1"}, // the day's case: a newer controller to the judged machine
{"mesh-controller", "novox", "2913c54c", ""}, // the same build: already there
{"mesh-controller", "ace", "2913c54c", "release-1"}, // another machine while the first is judged
{"mesh-host", "novox", "x", ""}, // a module the release does not carry
{"app", "anchor", "c2", ""},
{"app", "anchor", "c3", "plan-1"},
{"app", "laptop", "c2", "plan-1"},
} {
if got := f.walkedBy(c.module, c.node, c.to); got != c.want {
t.Errorf("%s %s to %s: walked by %q, want %q", c.module, c.to, c.node, got, c.want)
}
}
release.Release.Gate.Verdict = inventory.GatePassed
merge.Modules["app"].Gate.Verdict = inventory.GatePassed
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" || f.walkedBy("app", "laptop", "c3") != "" {
t.Fatal("a passed judging still holds a move")
}
release.Release.Gate.Verdict = ""
f.plans[0].State = inventory.PlanSuperseded
if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" {
t.Fatal("a closed release still holds a move")
}
}
// The controller is never put back to a build that reaches less of the store's schema than the store
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
if h := holderOf("x"); h.Build != "ad62528c47c7" {
t.Fatalf("the holder's build is %q", h.Build)
}
t.Setenv(RunningBuildVar, "")
if h := holderOf("x"); h.Build != version {
t.Fatalf("without a declared version the holder's build is %q", h.Build)
}
if reach, err := schemaReach(); err != nil || reach < 87 {
t.Fatalf("this build's reach is %d (%v)", reach, err)
}
}
cmd/mesh-controller/issue459_integration_test.go
-293
View File
@@ -1,293 +0,0 @@
//go:build integration
package main
import (
"errors"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/cat1aaaaaaaa")
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
if len(ws) != 1 || !ws[0].Open() {
t.Fatalf("no open walk to stop: %+v", ws)
}
stopped := ws[0]
p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute))
if err != nil {
t.Fatal(err)
}
if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" ||
!reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) {
t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery)
}
if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) {
t.Fatalf("the stop answered %+v", said)
}
again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
if err != nil || len(again) != 1 || again[0].Commit != one.Commit {
t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err)
}
if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit {
t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left)
}
cutAt(t, open, t0.Add(4*time.Minute))
fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
if err != nil || fresh.Batch() || !fresh.Open() {
t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err)
}
if _, in := fresh.Modules["one"]; !in {
t.Fatalf("the fresh walk does not build one: %v", fresh.Modules)
}
if _, in := fresh.Modules["app"]; in {
t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules)
}
if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit {
t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges)
}
// A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one).
cutAt(t, open, t0.Add(5*time.Minute))
if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 {
t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone)
}
}
func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/cat1aaaaaaaa")
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
first := catalogueMerge("cat1aaaaaaaa", "app", t0)
later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second))
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head)
hear(t, open, first, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
hear(t, open, later, t0.Add(11*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x",
[]string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil {
t.Fatal("a stop leaving out a merge the walk does not answer was taken")
}
if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() {
t.Fatalf("a refused stop ended the walk: %s", p.State)
}
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute))
if err != nil {
t.Fatal(err)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit},
{Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}}
if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) {
t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want)
}
if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) {
t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note)
}
again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy)
if len(again) != 1 || again[0].Commit != one.Commit {
t.Fatalf("walked again: %+v; want the other repository's merge alone", again)
}
// Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more.
cutAt(t, open, t0.Add(4*time.Minute))
fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy)
q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too",
[]string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute))
if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" {
t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err)
}
if _, bs := walks(t, open); len(bs) != 0 {
t.Fatalf("a batch of nothing was made: %+v", bs)
}
}
func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/cat1aaaaaaaa")
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one1, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute))
hear(t, open, one2, t0.Add(3*time.Minute))
_, bs := walks(t, open)
if len(bs) != 1 {
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
}
waiting := bs[0]
p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
if err != nil {
t.Fatal(err)
}
if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID {
t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID)
}
if _, bs := walks(t, open); len(bs) != 1 {
t.Fatalf("%d batches; want the open one alone", len(bs))
}
b, _ := open.inventory.PlanByID(ctx, waiting.ID)
if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") {
t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch)
}
// Its window closed by the stop, not by the clock: cut at once.
cutAt(t, open, t0.Add(3*time.Minute+11*time.Second))
fresh, _ := open.inventory.PlanByID(ctx, waiting.ID)
if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit {
t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one"))
}
var answered []string
for _, m := range fresh.Delivery.Merges {
answered = append(answered, m.Commit+">"+m.Carried)
}
if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") {
t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered)
}
}
func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/cat1aaaaaaaa")
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one1, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
sent := t0.Add(150 * time.Second)
for name, m := range walk.Modules {
if name == "app" || name == "one" {
m.First, m.FirstAt = []string{"anchor"}, &sent
}
}
if err := open.inventory.SavePlan(ctx, &walk); err != nil {
t.Fatal(err)
}
cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute))
hear(t, open, cat2, t0.Add(3*time.Minute))
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) {
t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept)
}
p, _ := open.inventory.PlanByID(ctx, walk.ID)
if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") {
t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note)
}
if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) {
t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy)
}
if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") {
t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err)
}
}
func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
_, bs := walks(t, open)
b := bs[0]
r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second))
if err != nil {
t.Fatal(err)
}
if said.WalkedAgainBy != b.ID || said.Stopped == b.ID {
t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said)
}
if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit {
t.Fatalf("the stopped record holds %+v", left)
}
if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed ||
stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID {
t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery)
}
_ = r
cutAt(t, open, t0.Add(3*time.Minute))
w, _ := open.inventory.PlanByID(ctx, b.ID)
if _, in := w.Modules["app"]; in || w.Batch() {
t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules)
}
// A batch whose every merge is left out is stopped itself.
hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
_, bs = walks(t, open)
last := bs[0]
if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x",
[]string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID {
t.Fatalf("a batch left with nothing: %+v %v", said, err)
}
if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() {
t.Fatalf("the emptied batch is %s", p.State)
}
}
func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) {
open := windowed(t)
everyHeadWaits(t)
endsItsLine(t)
ctx := t.Context()
hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second))
_, bs := walks(t, open)
stale := bs[0]
moved := stale
moved.Note = "written by somebody else"
if err := open.inventory.SavePlan(ctx, &moved); err != nil {
t.Fatal(err)
}
fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0,
State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}}
err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale},
[]inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}})
if !errors.Is(err, inventory.ErrPlanMoved) {
t.Fatalf("a stale plan in the act: %v", err)
}
if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil {
t.Fatal("the new plan was written though the act was refused")
}
if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID {
t.Fatalf("the merge moved to %q though the act was refused", m.Plan)
}
}
cmd/mesh-controller/issue459_test.go
-158
View File
@@ -1,158 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one
// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges
// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them
// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it.
// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and
// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS"
// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the
// test never ran.
func endsItsLine(t *testing.T) {
t.Cleanup(func() { fmt.Println() })
}
// leaved is a kept merge of a repository, made at a moment.
func leaved(repository, commit string, made time.Time) inventory.BatchedMerge {
return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made}
}
// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it
// contains; an earlier merge of that repository and every other repository's merge are walked again.
func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) {
c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0)
c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second))
c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second))
o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second))
again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"})
if err != nil {
t.Fatal(err)
}
var walked []string
for _, m := range again {
walked = append(walked, m.Commit)
}
if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) {
t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit},
{Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}}
if !reflect.DeepEqual(out, want) {
t.Fatalf("left out: %+v; want %+v", out, want)
}
// A merge the walk does not answer is refused, naming it: nothing ends.
if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil ||
!strings.Contains(err.Error(), "ffffffffffff") {
t.Fatalf("a merge the walk does not answer was left out: %v", err)
}
if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil {
t.Fatal("a merge without its commit was taken")
}
}
// moving is a kept merge that moved modules when it was heard.
func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge {
m.Event, _ = json.Marshal(keptMerge{Moves: modules})
return m
}
// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules
// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until
// none more is: a merge sharing a module with that one is left out as well.
func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) {
stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app")
reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one")
next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one")
free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three")
again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"})
if err != nil {
t.Fatal(err)
}
if len(again) != 1 || again[0].Commit != free.Commit {
t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again)
}
want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit},
{Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}},
{Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}}
if !reflect.DeepEqual(out, want) {
t.Fatalf("left out %+v; want %+v", out, want)
}
}
// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its
// own batch moves is built, and empty tiers go.
func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) {
plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}},
Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}}
got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"})
if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) {
t.Fatalf("withheld %v, tiers %v", got, plan.Tiers)
}
if _, in := plan.Modules["app"]; in {
t.Fatal("a withheld module is still in the plan")
}
}
// The seat verb passes the merges to leave out to the command.
func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) {
got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen",
"without": "novox/mesh-lab@b1,novox/mesh-lab@b2"})
want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen",
"--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"}
if err != nil || !reflect.DeepEqual(got, want) {
t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want)
}
}
// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped,
// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk
// that builds the other repository's module alone.
// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it
// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only
// ended. A merge the walk does not answer is refused, and nothing ends.
// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge
// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one
// (review of #232): its window closes, and its walk is at the newer commit, answering both.
// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of
// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk
// that walked its merges again is refused a retry, saying why.
// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on
// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped.
// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody
// else since it was read refuses all of it, and no merge moves, no plan is written.
// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent
// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent
// to; a module a merge walked again moves is sent again, and is not named.
func TestKeptNamesEveryMachineASendReached(t *testing.T) {
at := t0
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
"together": {SentAt: &at},
"first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}},
"again": {SentAt: &at},
"unsent": {},
}}
got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}})
want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}},
{Module: "together", Machines: []string{"anchor", "laptop"}}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("kept %+v; want %+v", got, want)
}
}
cmd/mesh-controller/issue529_test.go
-102
View File
@@ -1,102 +0,0 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 529: a walk waiting because its delivery is held for a person said only "its walk waits for
// mesh-delivery's word — `plans go … --why …` starts it by hand", pointing a reader at a hand start instead of at
// the person's decision. The line names what mesh-delivery says holds the walk: the held delivery, its reason and
// `mesh-delivery.release`; or the delivery not at its turn.
func waitingWalk(now time.Time) inventory.Plan {
return inventory.Plan{ID: "plan-1791793277221379309", Repository: "novox/mesh-sdk", Branch: "main",
Commit: "9747f3e1b1ac", State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}},
Created: now.Add(-9 * time.Minute), Modules: map[string]*inventory.PlanModule{},
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
}
func TestAWalkWaitingOnAHeldDeliveryNamesTheHold(t *testing.T) {
now := time.Date(2026, 10, 12, 8, 35, 0, 0, time.UTC)
p := waitingWalk(now)
held := "novox/mesh-sdk@d41d255f28ad held since 2026-10-12T08:23:00Z: its group adr-0307-state-history's composed " +
"check did not pass for the heads that merged — a person decides: `mesh-delivery.release` (id " +
"novox/mesh-sdk@d41d255f28ad, why) lets it go, `mesh-delivery.stop` ends it"
waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@d41d255f28ad", State: "held", Walk: p.ID, Says: held}}}
line := planLineSaid(p, now, tierAtLeast, nil, waits)
for _, want := range []string{"held for a person", "composed check did not pass", "mesh-delivery.release"} {
if !strings.Contains(line, want) {
t.Fatalf("the line does not say %q: %q", want, line)
}
}
if strings.Contains(line, "starts it by hand") {
t.Fatalf("a held walk's line points at a hand start: %q", line)
}
// Told nothing — an older mesh-delivery, or none answering — it reads as before.
if line := planLineSaid(p, now, tierAtLeast, nil, nil); !strings.Contains(line, "starts it by hand") {
t.Fatalf("told nothing, the line reads %q", line)
}
}
func TestAWalkWaitingOnADeliveryNotAtItsTurnSaysSo(t *testing.T) {
now := time.Date(2026, 10, 12, 8, 57, 0, 0, time.UTC)
p := waitingWalk(now)
waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@9747f3e1b1ac", State: "published", Walk: p.ID,
Says: "not let go yet: novox/mesh-tools@3f0c1a6cec18 (published) carried by it not at its turn"}}}
line := planLineSaid(p, now, tierAtLeast, nil, waits)
if !strings.Contains(line, "not at its turn") || !strings.Contains(line, "novox/mesh-tools@3f0c1a6cec18") {
t.Fatalf("the line reads %q", line)
}
// No hand start offered: it would skip the group's order (review of issue 529).
if strings.Contains(line, "plans go") {
t.Fatalf("a walk not at its turn points at a hand start: %q", line)
}
}
// A walk nothing holds — the nzbgeek case, its merges' deliveries ended as nothing-to-move — is said so, and the hand
// start is the way on.
func TestAWalkHeldByNothingSaysWhyAndTheHandStart(t *testing.T) {
now := time.Date(2026, 10, 11, 20, 30, 0, 0, time.UTC)
p := waitingWalk(now)
waits := walkWaits{p.ID: {{ID: "novox/mesh-media-catalog@cdcdcdcd0000", State: "unheld", Walk: p.ID,
Says: "held by nothing: its merges' deliveries ended as nothing-to-move (novox/mesh-media-catalog@abababab0000), " +
"yet the walk moves nzbgeek, nzbget — nothing gives it its word"}}}
line := planLineSaid(p, now, tierAtLeast, nil, waits)
if !strings.Contains(line, "ended as nothing-to-move") || !strings.Contains(line, "plans go "+p.ID) {
t.Fatalf("the line reads %q", line)
}
}
// What holds each waiting walk is asked of mesh-delivery's `stalled` with `walks`, a verb the controller's grant
// already names; only lines naming a walk are kept, by walk.
func TestWhatHoldsTheWaitingWalksIsAskedOfTheDeliveryOwner(t *testing.T) {
was := askDeliveryOwner
defer func() { askDeliveryOwner = was }()
var asked map[string]any
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
if verb != "stalled" {
t.Fatalf("asked %s", verb)
}
asked = args
return json.Marshal([]stalledLine{{ID: "novox/a@1", State: "held", Walk: "plan-1", Says: "held: why"},
{ID: "novox/b@2", State: "held", Says: "a delivery past its bound, no walk"}})
}
got := walksWaiting(t.Context(), nil, true)
if asked["walks"] != true {
t.Fatalf("stalled was asked with %v", asked)
}
if len(got) != 1 || len(got["plan-1"]) != 1 || got["plan-1"][0].Says != "held: why" {
t.Fatalf("the waiting walks are %+v", got)
}
if got := walksWaiting(t.Context(), nil, false); got != nil {
t.Fatalf("with no holder on record it asked: %+v", got)
}
}
cmd/mesh-controller/issue611_integration_test.go
-54
View File
@@ -1,54 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) {
open := windowed(t)
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
inv := open.inventory
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main",
Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour),
Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil {
t.Fatal(err)
}
readme := repoMerge("one", "one1aaaaaaaa", t0)
readme.Paths = []string{"README.md"}
hear(t, open, readme, t0.Add(time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
if len(ws) != 1 {
t.Fatalf("walks: %+v", ws)
}
w := ws[0]
if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 {
t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules)
}
if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") {
t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery)
}
if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") {
t.Errorf("the walk's note does not name it: %q", w.Note)
}
// A merge that changes the build source is walked, and says nothing unchanged.
code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute))
hear(t, open, code, t0.Add(3*time.Minute+time.Second))
cutAt(t, open, t0.Add(6*time.Minute))
ws, _ = walks(t, open)
if len(ws) != 2 {
t.Fatalf("walks: %+v", ws)
}
if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 {
t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery)
}
}
cmd/mesh-controller/issue611_test.go
-102
View File
@@ -1,102 +0,0 @@
package main
import (
"reflect"
"sort"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// novox/hq issue 611: mesh-controller #293 changed one test file. Its delivery plan, made while another walk built
// build-agent and route-proxy (issue 606), read them whole and named them; at the cut their recorded build sources
// held none of its files, so its walk moved nothing and its record named no module. mesh-delivery could not tell
// "matched what runs" from "never built" and stopped the delivery, asking the operator about a change already on
// every machine. The walk now says, per module, that its build matched what runs (delivery.unchanged, ADR 0267 §5).
// The shape of #293: a test file of the controller's repository. The controller, built from the root, and the two
// modules packaging the repository each recorded a build source holding none of it: each is said unchanged, with
// why; a module of another repository is not named.
func TestAMergeSaysTheModulesWhoseBuildSourceItLeftUnchanged(t *testing.T) {
const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git"
const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git"
entries := []inventory.Entry{
fromRepo("mesh-controller", controllerRepo, ""),
fromRepo("build-agent", catalogueRepo, "modules/build-agent"),
fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"),
fromRepo("gitea", catalogueRepo, "modules/gitea"),
}
read := map[string][]inventory.ReadRepository{
"mesh-controller": {{Own: true, Paths: []string{"go.mod", "module.json", "cmd/mesh-controller/", "internal/builder/"}}},
"build-agent": {
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "cmd/mesh-builder/", "internal/builder/"}},
{Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}},
},
"route-proxy": {
{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "modules/route-proxy/cmd/route-proxy/"}},
{Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}},
},
}
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "8f1aec0",
Paths: []string{"internal/builder/replay363_test.go"}}
if moved := reachOfMerge(m, entries, read, sharedRepositoryEdges).Moved(); len(moved) != 0 {
t.Fatalf("the fixture moves %v: a test file is in no build source", moved)
}
got := unchangedOfMerge(m, entries, read)
var names []string
for n, why := range got {
names = append(names, n)
if !strings.Contains(why, "holds none of") || !strings.Contains(why, "internal/builder/replay363_test.go") ||
!strings.Contains(why, "the build the mesh holds is its build") {
t.Errorf("%s is said unchanged as %q", n, why)
}
}
if want := []string{"build-agent", "mesh-controller", "route-proxy"}; !reflect.DeepEqual(sortedOf(names), want) {
t.Fatalf("said unchanged: %v; want %v", sortedOf(names), want)
}
// A module whose build source holds a changed file is moved, never said unchanged.
touched := m
touched.Paths = []string{"internal/builder/builder.go"}
if got := unchangedOfMerge(touched, entries, read); got["mesh-controller"] != "" || got["build-agent"] != "" {
t.Errorf("a module the merge moves is said unchanged: %v", got)
}
if got := unchangedOfMerge(touched, entries, read); got["route-proxy"] == "" {
t.Errorf("the route proxy, whose build source holds none of it, is not said unchanged: %v", got)
}
// With no build source recorded, the module is read whole and moved: nothing says it unchanged.
unsaid := map[string][]inventory.ReadRepository{"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}}}
if got := unchangedOfMerge(m, entries, unsaid); len(got) != 0 {
t.Errorf("modules with no recorded build source are said unchanged: %v", got)
}
// Files not all said: every module is moved, none said unchanged.
truncated := m
truncated.PathsTruncated = true
if got := unchangedOfMerge(truncated, entries, read); len(got) != 0 {
t.Errorf("a merge whose files were not all said says %v unchanged", got)
}
}
// A module the walk builds anyway — another merge's, or a dependent — is not said unchanged: the modules map keeps
// meaning what the walk builds, and the two never name one module.
func TestAWalkSaysUnchangedOnlyWhatItDoesNotBuild(t *testing.T) {
unchanged := map[string]string{"build-agent": "why a", "route-proxy": "why b"}
got := unchangedLeft(unchanged, map[string]*inventory.PlanModule{"build-agent": {}})
if !reflect.DeepEqual(got, map[string]string{"route-proxy": "why b"}) {
t.Fatalf("left %v", got)
}
if got := unchangedLeft(map[string]string{"x": "y"}, map[string]*inventory.PlanModule{"x": {}}); got != nil {
t.Fatalf("left %v; want none", got)
}
}
// Against the store, the path a real merge takes: a merge whose module recorded a build source that holds none
// of its files is cut into a walk done with nothing to walk, whose record names the module unchanged and why.
func sortedOf(names []string) []string {
out := append([]string{}, names...)
sort.Strings(out)
return out
}
cmd/mesh-controller/issue648_test.go
-90
View File
@@ -1,90 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
// send that last carried that build before the walk got there, and nothing will make them report again:
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates
// nothing.
for _, m := range []catalogue.Manifest{
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
{Module: "web", Version: "w1"},
} {
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
t.Fatal(err)
}
}
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
for _, b := range []inventory.Build{
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
} {
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
// Each machine answered that send, applied, before the walk's tier was built.
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
At: time.Now().Add(-10 * time.Second)}); err != nil {
t.Fatal(err)
}
}
built := time.Now().UTC()
p := g.plan(t)
p.Tiers = [][]string{{"app"}, {"web"}}
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if len(g.sent) != 0 {
t.Fatalf("a no-move build was sent: %v", g.sent)
}
p = g.plan(t)
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
t.Fatalf("the build was not marked no move: %+v", s)
}
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
}
// A machine that has not answered its last send is not waited on either: a built-by edge gates no tier
// (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop.
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
p = g.plan(t)
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note)
}
}
cmd/mesh-controller/kept_helpers.go
+252
View File
@@ -0,0 +1,252 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/micro"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"regexp"
"strconv"
"strings"
"time"
)
// Small helpers the plan machinery once held and other code still reads.
// silentCallBound is how long a call to the bus may take before it is called unanswered.
var silentCallBound = 20 * time.Second
// clipWords keeps a plain sentence within a bound, at a word.
func clipWords(s string, n int) string {
if len(s) <= n {
return s
}
cut := strings.LastIndex(s[:n-1], " ")
if cut <= 0 {
cut = n - 1
}
return s[:cut] + "…"
}
// orWords is s, or the words for when it is empty.
func orWords(s, none string) string {
if s == "" {
return none
}
return s
}
func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) {
ctx, cancel := context.WithTimeout(ctx, silentCallBound)
defer cancel()
type answer struct {
v T
err error
}
done := make(chan answer, 1)
go func() { v, err := call(ctx); done <- answer{v, err} }()
select {
case a := <-done:
return a.v, a.err
case <-ctx.Done():
var zero T
return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err())
}
}
func sameCommitID(a, b string) bool {
if a == "" || b == "" {
return false
}
if a == b {
return true
}
n := min(len(a), len(b))
return n >= 7 && a[:n] == b[:n]
}
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
var wordsZone = func() *time.Location { return time.Local }
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
var wordsNow = time.Now
var (
planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`)
walkWd = regexp.MustCompile(`\bwalks\b`)
walkOne = regexp.MustCompile(`\bwalk\b`)
)
// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly.
func plainRefusalOr(refusal, fallback string) string {
r := planIDWord.ReplaceAllString(refusal, "the plan")
r = walkWd.ReplaceAllString(r, "deliveries")
r = walkOne.ReplaceAllString(r, "delivery")
r = strings.TrimSpace(r)
if _, ok := conditions.Plain(r); !ok {
return fallback
}
return r
}
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
// a seam a test replaces. The holder's own refusal is an error naming it.
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
granted := false
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
granted = granted || v.Verb == verb
}
if !granted {
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
}
if conn == nil {
return nil, errors.New("this controller is not on the bus")
}
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
if err != nil {
return nil, err
}
if answer.Error != "" {
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
}
return unwrapToolResult(answer.Result), nil
}
// deliveryOwnerWithin is how long the owner is given to answer.
var deliveryOwnerWithin = 10 * time.Second
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
// protocol's content list holding it as text.
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
var wrapped struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
return json.RawMessage(wrapped.Content[0].Text)
}
return raw
}
// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules'
// tools are served where. A variable so a test needs no runtime.
var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) {
inbox := conn.NewRespInbox()
sub, err := conn.SubscribeSync(inbox)
if err != nil {
return nil, err
}
defer func() { _ = sub.Unsubscribe() }()
if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
return nil, fmt.Errorf("asking the bus who serves what: %w", err)
}
out := map[string]served{}
add := func(node string) served {
s, ok := out[node]
if !ok {
s = served{tools: map[string]bool{}}
}
return s
}
deadline := time.Now().Add(discoveryPatience)
for time.Now().Before(deadline) {
wait, cancel := context.WithTimeout(ctx, discoveryQuiet)
msg, err := sub.NextMsgWithContext(wait)
cancel()
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
break
}
var info micro.Info
if json.Unmarshal(msg.Data, &info) != nil {
continue
}
if info.Name == broker.RuntimeModule {
node := info.Metadata["node"]
if node == "" {
node = info.ID
}
s := add(node)
s.runtime = true
out[node] = s
}
for _, e := range info.Endpoints {
if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" {
continue
}
node := e.Metadata["node"]
if node == "" {
node = info.ID
}
s := add(node)
s.tools[e.Metadata["module"]] = true
out[node] = s
}
}
return out, nil
}
// served is what one machine's node tools answered the bus's discovery with.
type served struct {
runtime bool
tools map[string]bool
}
// theGraph is what the planner reads.
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
[]inventory.Edge, error) {
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, nil, nil, err
}
read, err := readForPlanning(ctx, inv)
if err != nil {
return nil, nil, nil, err
}
edges, err := inv.Dependencies(ctx)
if err != nil {
return nil, nil, nil, err
}
return entries, read, edges, nil
}
// refusedWord is the word a build's failure says the reader did not know; empty when it says none.
func refusedWord(failed string) string {
if m := unknownWord.FindStringSubmatch(failed); m != nil {
return m[1]
}
return ""
}
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
// durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds.
func durationSetting(v any) (time.Duration, bool) {
switch t := v.(type) {
case string:
if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 {
return d, true
}
if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 {
return time.Duration(n * float64(time.Second)), true
}
case float64:
if t >= 0 {
return time.Duration(t * float64(time.Second)), true
}
}
return 0, false
}
cmd/mesh-controller/lean_walk_integration_test.go
-141
View File
@@ -1,141 +0,0 @@
//go:build integration
package main
import (
"context"
"slices"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklogOnThree(t)
ctx := t.Context()
said := events(t)
for i := 0; i < 12; i++ {
advancePlans(ctx, b.open)
}
p := b.release(t)
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
for _, n := range []string{"desk", "laptop", "anchor"} {
if !slices.Contains(*said, "send "+n) {
t.Fatalf("%s was not sent: %v", n, *said)
}
}
if slices.Contains(*said, "judge") {
t.Fatalf("a machine was judged by its health: %v", *said)
}
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
}
}
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklogOnThree(t)
ctx := t.Context()
inv := b.open.inventory
b.broken["desk"] = true // desk fails what it is sent
advancePlans(ctx, b.open)
for i := 0; i < 12 && b.release(t).Open(); i++ {
advancePlans(ctx, b.open)
}
p := b.release(t)
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
}
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
}
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
t.Fatalf("desk was put back: %v", sent)
}
}
func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
p, err := releaseBacklog(ctx, b.open, "")
if err != nil || p == nil {
t.Fatalf("no release opened: %v", err)
}
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
}
if err := inv.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
// The send is made, and the controller is replaced before it keeps it.
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
t.Fatal(err)
}
advancePlans(ctx, b.open)
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
t.Fatalf("anchor was sent again: %v", b.sent)
}
g := b.release(t).Release.Gates["anchor"]
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
t.Fatalf("anchor is not judged on the send made: %+v", g)
}
}
func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
reporting := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
b.sent = append(b.sent, names)
current, _ := open.inventory.CurrentBuilds(ctx)
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
}
return reporting(ctx, open, names)
}
t.Cleanup(func() { sendRollout = reporting })
for i := 0; i < 8; i++ {
advancePlans(ctx, b.open)
}
p := b.release(t)
g := p.Release.Gates["laptop"]
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
}
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
g.Since = &since
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, b.open)
p = b.release(t)
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
t.Fatal("a silent machine's build was marked failed")
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
t.Fatalf("laptop was put back: %v", sent)
}
obs, err := silentObservation(ctx, inv)
if err != nil {
t.Fatal(err)
}
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
t.Fatalf("the silence is not said: %+v", obs)
}
}
cmd/mesh-controller/lean_walk_test.go
-116
View File
@@ -1,116 +0,0 @@
package main
import (
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send,
// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading.
func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
sent := now.Add(-time.Minute)
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
at := now.Add(-10 * time.Second)
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{
"app": {Rest: map[string]inventory.SentDeclaration{
"ace": {Digest: "d-ace", At: sent},
"novox": {Digest: "d-novox", At: sent},
"shanks": {Digest: "d-shanks", At: sent},
"laptop": {Digest: "d-laptop", At: long},
}},
"proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}},
}}
reports := []inventory.Reported{
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
{Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"},
{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"},
}
waiting, failed, silent := walkReports(p, reports, now, nil)
if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) ||
!slices.Equal(silent, []string{"laptop"}) {
t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent)
}
reports[2].Declared = "d-shanks"
if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 {
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
}
}
// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before
// novox/hq ADR 0360 started.
func withTheGate(t *testing.T) {
t.Helper()
was := gateFirstMachine
gateFirstMachine = true
t.Cleanup(func() { gateFirstMachine = was })
}
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7).
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7).
// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once
// the engine and the machine's provisioner client both said so.
func TestAWalkEndWaitsForTheProvisioner(t *testing.T) {
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
at := now.Add(-10 * time.Second)
plan := func(sentAt time.Time) inventory.Plan {
return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
"ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
"g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}},
}}}}
}
reports := []inventory.Reported{
{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"},
{Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"},
}
says := map[string]inventory.ProvReport{}
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at}
// (a) the engine applied, the client has not said: the end waits on both.
waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 {
t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason.
says["ace/app"] = applied
says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at}
waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov)
if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") ||
!strings.Contains(failed[0], "no value") {
t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// (c) no client report past ApplySilentAfter: named silent, never applied.
delete(says, "g14/app")
waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov)
if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) {
t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// both said it: nothing waits.
says["g14/app"] = applied
if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
// (d) a send handing nothing to the provisioner is the engine's report alone, as before.
bare := plan(now.Add(-time.Minute))
bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}}
if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) {
return inventory.ProvReport{}, false
}); len(waiting)+len(failed)+len(silent) != 0 {
t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent)
}
}
cmd/mesh-controller/lone_merge_test.go
-338
View File
@@ -1,338 +0,0 @@
//go:build integration
package main
import (
"context"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// The tests of novox/hq issue 525's window skip: a merge waits for the merge window only when a batch assembling
// holds a merge related to it, or its delivery group may have a member still to merge (ADR 0304 decision 1, ADR
// 0306 decisions 2 and 3). Walks stay one at a time.
// A merge from no head that shares nothing, with no walk open, is cut into its walk when it is heard: no window.
func TestALoneMergeSharingNothingIsCutAtOnce(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
hear(t, open, lone(repoMerge("one", "c1", t0)), t0)
ws, bs := walks(t, open)
if len(ws) != 1 || len(bs) != 0 || !ws[0].Open() || ws[0].Commit != "c1" {
t.Fatalf("a lone merge sharing nothing: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
}
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(t0) {
t.Fatalf("its window closed at %+v; want when it was heard, %s", ws[0].Times, t0)
}
}
// A merge related to the batch assembling joins it and waits for the window, which its coming restarts: by its
// repository, by a module in common (a dependent of what the batch moves), by an edge of the module graph between
// their modules (one that widens nothing), or by its delivery group's head.
func TestARelatedMergeJoinsTheBatchAndWaitsForTheWindow(t *testing.T) {
ctx := t.Context()
for _, c := range []string{"same repository", "a module in common", "an edge of the graph", "the same group head"} {
t.Run(c, func(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/c1")
asksWithPaths(t)
inv := open.inventory
first := repoMerge("one", "c1", t0)
second := lone(repoMerge("two", "c2", t0.Add(10*time.Second)))
why := map[string]string{"same repository": "both change novox/one", "a module in common": "both move two",
"an edge of the graph": "two depends on builder (built-by)",
"the same group head": "both are members of group " + first.Head}[c]
switch c {
case "same repository":
second = lone(repoMerge("one", "c2", t0.Add(10*time.Second)))
case "a module in common":
// two is built on one: a merge of one moves two too.
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "two", Version: "1",
Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "one"}}}},
inventory.Source{Repository: "novox/two", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
case "an edge of the graph":
// two is built by the build machine: an edge that orders and widens nothing.
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1",
Claims: []catalogue.Claim{{Name: "mesh-build-machine", Scope: catalogue.ScopeMesh}}},
inventory.Source{Repository: "novox/builder", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
first = repoMerge("builder", "c1", t0)
case "the same group head":
second = fromHead(repoMerge("two", "c2", t0.Add(10*time.Second)), first.Head)
}
hear(t, open, first, t0)
said := captured(t, func() error {
return (following{open: open}).hearMerge(ctx, second, t0.Add(10*time.Second))
})
if !strings.Contains(said, "("+why+")") {
t.Fatalf("the merge was not said related by %q: %s", why, said)
}
ws, bs := walks(t, open)
if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
t.Fatalf("the related merge: %d walk(s), batches %+v", len(ws), bs)
}
if bs[0].Delivery.Batch.CutAtOnce {
t.Fatal("a batch a related merge joined is cut at once")
}
cutAt(t, open, t0.Add(99*time.Second))
if ws, _ := walks(t, open); len(ws) != 0 {
t.Fatalf("cut before the window the related merge restarted closed: %+v", ws)
}
cutAt(t, open, t0.Add(100*time.Second))
ws, bs = walks(t, open)
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
}
})
}
}
// A merge from a head whose delivery group has a member still to merge waits for the window, even with no batch
// assembling, and says why; its sibling joins the same batch by their head (ADR 0306 decision 2). A merge from a
// head that names no group shares nothing and is cut at once, as one from no head is.
func TestAMergeOfAGroupWaitsEvenAlone(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
states := map[string]string{"novox/one": "ready", "novox/two": "ready", "novox/elsewhere": "ready"}
groupStates(t, "feat/x", states)
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
ws, bs := walks(t, open)
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce || windowClosed(bs[0], t0) {
t.Fatalf("a group's merge alone: %d walk(s), batches %+v", len(ws), bs)
}
if words := batchWords(bs[0], t0); !strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready), novox/two (ready)") {
t.Fatalf("it reads %q", words)
}
// Its members merge: the batch is held for the last, and the window runs again from that merge.
delete(states, "novox/elsewhere")
states["novox/two"] = "merged"
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/x"), t0.Add(5*time.Second))
if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
t.Fatalf("the group's second member did not join the first: %+v", bs)
}
cutAt(t, open, t0.Add(94*time.Second))
if ws, _ := walks(t, open); len(ws) != 0 {
t.Fatalf("cut before its window closed: %+v", ws)
}
cutAt(t, open, t0.Add(95*time.Second))
ws, bs = walks(t, open)
if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" {
t.Fatalf("at its window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs))
}
}
// A merge from a head that names no group, sharing nothing, with no walk open, is cut at once.
func TestAHeadedMergeInNoGroupIsCutAtOnce(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
hear(t, open, fromHead(repoMerge("one", "c1", t0), "fix/alone"), t0)
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 || !ws[0].Open() {
t.Fatalf("a headed merge in no group: %d walk(s), %d batch(es)", len(ws), len(bs))
}
}
// When mesh-delivery cannot say a head's group, the merge waits for the window, the safe side, and its batch says
// why.
func TestAGroupNobodyCanSayWaits(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
was := deliveryGroupOf
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
return nil, false, errors.New("nothing serves mesh-delivery.groups")
}
t.Cleanup(func() { deliveryGroupOf = was })
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0)
ws, bs := walks(t, open)
if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce {
t.Fatalf("a merge whose group nobody could say: %d walk(s), batches %+v", len(ws), bs)
}
if words := batchWords(bs[0], t0); !strings.Contains(words, "mesh-delivery did not say whether group feat/x has a "+
"member still to merge (nothing serves mesh-delivery.groups): it waits for the window, the safe side") {
t.Fatalf("it reads %q", words)
}
}
// A member counts as still to merge only while its pull request is open, and not once this controller keeps its
// merge: a group whose other members are merged or closed makes nobody wait.
func TestAGroupsMergedOrClosedMembersMakeNobodyWait(t *testing.T) {
open := windowed(t)
asksWithPaths(t)
ctx := t.Context()
members := []groupMember{{Repository: "novox/one", State: "ready"}, {Repository: "novox/two", State: "published"},
{Repository: "novox/three", State: "stopped"}}
was := deliveryGroupOf
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return members, true, nil }
t.Cleanup(func() { deliveryGroupOf = was })
m := fromHead(repoMerge("one", "c1", t0), "feat/x")
if why, err := groupWait(ctx, open.inventory, m, true, false); err != nil || why != "" {
t.Fatalf("the group's other members are merged or closed, yet: %q %v", why, err)
}
members = append(members, groupMember{Repository: "novox/four", State: "checked"})
if why, _ := groupWait(ctx, open.inventory, m, true, false); !strings.Contains(why, "novox/four (checked)") {
t.Fatalf("an open member: %q", why)
}
// novox/four's merge is kept in a batch already: merged, whatever mesh-delivery says yet.
hear(t, open, fromHead(repoMerge("two", "c4", t0), "feat/x"), t0)
members[3].Repository = "novox/two"
if why, _ := groupWait(ctx, open.inventory, m, true, false); why != "" {
t.Fatalf("a member this controller keeps the merge of still waits: %q", why)
}
if got := memberOf("novox/mesh-sdk@d41d255f28ad (held)"); got != (groupMember{Repository: "novox/mesh-sdk", State: "held"}) {
t.Fatalf("a member reads %+v", got)
}
}
// A merge related to two batches assembling folds the newer into the older, which answers every merge of both
// and names nothing twice; the folded one names the batch that took it over.
func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "feat/a", "feat/b")
asksWithPaths(t)
ctx := t.Context()
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/a"), t0)
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/b"), t0.Add(5*time.Second))
_, bs := walks(t, open)
if len(bs) != 2 {
t.Fatalf("two merges sharing nothing: %d batch(es)", len(bs))
}
older, newer := bs[1], bs[0] // newest first
// Of one's repository and from two's head: related to both.
hear(t, open, fromHead(repoMerge("one", "c3", t0.Add(10*time.Second)), "feat/b"), t0.Add(10*time.Second))
_, bs = walks(t, open)
if len(bs) != 1 || bs[0].ID != older.ID || len(bs[0].Delivery.Merges) != 3 {
t.Fatalf("after a merge related to both: batches %+v; want %s answering three merges", bs, older.ID)
}
folded, err := open.inventory.PlanByID(ctx, newer.ID)
if err != nil || folded.State != inventory.PlanSuperseded || folded.Delivery == nil ||
folded.Delivery.TakenOverBy != older.ID {
t.Fatalf("the newer batch reads %s %+v (%v)", folded.State, folded.Delivery, err)
}
if left, _ := open.inventory.MergesOf(ctx, newer.ID); len(left) != 0 {
t.Fatalf("the folded batch still answers %+v", left)
}
cutAt(t, open, t0.Add(100*time.Second))
ws, bs := walks(t, open)
var cut []inventory.Plan
for _, w := range ws {
if w.State != inventory.PlanSuperseded {
cut = append(cut, w)
}
}
if len(cut) != 1 || len(bs) != 0 || cut[0].ID != older.ID || cut[0].CommitOf("novox/one") != "c3" ||
cut[0].CommitOf("novox/two") != "c2" {
t.Fatalf("the folded batch's walk: %+v, %d batch(es)", cut, len(bs))
}
}
// With a walk open that it is related to (the same repository, issue 973), a batch cut at once still waits for
// it, said queued behind it (ADR 0276 decision 4, ADR 0407 decision 4); it is cut once that walk ended, with no
// window to wait for.
func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/c1")
asksWithPaths(t)
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(90*time.Second))
ws, _ := walks(t, open)
if len(ws) != 1 || !ws[0].Open() {
t.Fatalf("no walk open: %+v", ws)
}
first := ws[0]
heard := t0.Add(100 * time.Second)
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
ws, bs := walks(t, open)
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID ||
!bs[0].Delivery.Batch.CutAtOnce {
t.Fatalf("a lone merge during a walk: %d walk(s), batches %+v", len(ws), bs)
}
if !strings.HasPrefix(batchWords(bs[0], heard), "queued behind "+first.ID) {
t.Fatalf("it reads %q", batchWords(bs[0], heard))
}
first.State = inventory.PlanDone
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
t.Fatal(err)
}
cutAt(t, open, heard.Add(time.Second))
ws, bs = walks(t, open)
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
t.Fatalf("the queued batch was not cut once the walk ended: %+v %+v", ws, bs)
}
if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(heard) {
t.Fatalf("its window closed at %+v; want when its merge was heard, %s", ws[0].Times, heard)
}
}
// A stopped walk's other merges are walked again in a batch they are related to, and a batch of their own when
// they share nothing with the one assembling (ADR 0299, ADR 0304 decision 1): never in a batch of another
// change, whose window the stop would close.
func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/cat1aaaaaaaa")
endsItsLine(t)
asksRecorded(t)
ctx := t.Context()
cat := catalogueMerge("cat1aaaaaaaa", "app", t0)
one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head)
hear(t, open, cat, t0.Add(time.Second))
hear(t, open, one, t0.Add(6*time.Second))
cutAt(t, open, t0.Add(2*time.Minute))
ws, _ := walks(t, open)
walk := ws[0]
// Related to the walk by its repository (queued behind it), and to nothing the stop walks again.
hear(t, open, catalogueMerge("cat2cccccccc", "notes", t0.Add(3*time.Minute)), t0.Add(3*time.Minute))
_, bs := walks(t, open)
if len(bs) != 1 {
t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs))
}
other := bs[0]
_, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped",
[]string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second))
if err != nil {
t.Fatal(err)
}
if said.WalkedAgainBy == "" || said.WalkedAgainBy == other.ID {
t.Fatalf("walked again by %q; want a batch of its own, not %s", said.WalkedAgainBy, other.ID)
}
b, _ := open.inventory.PlanByID(ctx, other.ID)
if b.Delivery.Batch == nil || b.Delivery.Batch.Closed || len(b.Delivery.Merges) != 1 {
t.Fatalf("the other change's batch reads %+v; want it untouched by the stop", b.Delivery)
}
}
// A catalogue batch's cut folds in every walk waiting for its delivery's word (cutBatch's carry), so what that
// walk moves is the batch's too: a lone merge related to the waiting walk joins the batch assembling and waits for
// its window, rather than being cut at once as sharing nothing with what the batch's own merges move.
func TestALoneMergeRelatedToAWalkTheCutFoldsJoinsTheBatch(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/c3")
asksWithPaths(t)
ctx := t.Context()
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
BuiltFrom: "c0", Head: "c0"}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
t.Fatal(err)
}
hear(t, open, repoMerge("one", "c1", t0), t0)
cutAt(t, open, t0.Add(90*time.Second))
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Waiting() {
t.Fatalf("the walk does not wait for its word: %+v", ws)
}
hear(t, open, repoMerge("three", "c3", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
hear(t, open, lone(repoMerge("one", "c2", t0.Add(2*time.Minute+5*time.Second))), t0.Add(2*time.Minute+5*time.Second))
_, bs := walks(t, open)
if len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 || bs[0].Delivery.Batch.CutAtOnce {
t.Fatalf("a lone merge related to the waiting walk: batches %+v", bs)
}
}
cmd/mesh-controller/machine_network_integration_test.go
-266
View File
@@ -1,266 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"os"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv, k := open.inventory, conditionsFrom
say := func(at time.Time, n *link.NetworkHealth) {
t.Helper()
if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil {
t.Fatal(err)
}
}
openKeys := func() []string {
t.Helper()
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
var keys []string
for _, c := range list {
keys = append(keys, c.Key)
}
return keys
}
healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{
{Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}}
rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{
{Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said,
Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)},
{Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said,
Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}}
say(h0, healthy)
if keys := openKeys(); len(keys) != 0 {
t.Fatalf("a healthy network raised %v", keys)
}
say(h0.Add(time.Minute), rewritten)
if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
t.Fatalf("the rewrite raised %v", keys)
}
kept, had, err := inv.HealthOf(ctx, "laptop")
if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" {
t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err)
}
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") ||
!strings.Contains(lines, "unhealthy resolv-conf") {
t.Fatalf("node show says:\n%s", lines)
}
say(h0.Add(2*time.Minute), healthy)
if keys := openKeys(); len(keys) != 0 {
t.Fatalf("written back, still open: %v", keys)
}
// An engine older than the judging says no network: nothing raised, and node show says it is not known.
say(h0.Add(3*time.Minute), nil)
kept, had, _ = inv.HealthOf(ctx, "laptop")
if keys := openKeys(); len(keys) != 0 || kept.Network != nil {
t.Fatalf("an older engine: %v %+v", keys, kept.Network)
}
if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") {
t.Fatalf("node show says:\n%s", lines)
}
}
func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) {
raw, err := os.ReadFile("testdata/network-drill.json")
if err != nil {
t.Fatal(err)
}
var said []link.Health
if err := json.Unmarshal(raw, &said); err != nil {
t.Fatal(err)
}
open := aMesh(t)
ctx := t.Context()
k := conditionsFrom
var raisedAt []int
for i, h := range said {
if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil {
t.Fatal(err)
}
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
for _, c := range list {
if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") {
t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary)
}
raisedAt = append(raisedAt, i)
}
}
// Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back.
if len(raisedAt) != 1 || raisedAt[0] != 3 {
t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt)
}
}
func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth {
for i := range parts {
if parts[i].State == "" {
parts[i].State = link.StateUnhealthy
}
parts[i].Since = h0
}
return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts}
}
func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) {
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateHealthy),
"laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN),
}))
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" {
t.Fatalf("want one finding, the rewrite, got %v", keys)
}
o := obs[0]
for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve",
"next reconcile"} {
if !strings.Contains(o.Summary, want) {
t.Errorf("the summary does not say %q: %s", want, o.Summary)
}
}
if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") {
t.Errorf("an address or a path reached the summary: %s", o.Summary)
}
if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" {
t.Errorf("the evidence or the severity is wrong: %+v", o)
}
}
func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) {
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateHealthy),
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
"spare": aNetwork(link.StateHealthy),
}))
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
t.Fatalf("want the laptop's own, got %v", keys)
}
if obs[0].Severity != conditions.Warning {
t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity)
}
}
func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) {
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateHealthy),
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
"spare": aNetwork(link.StateUnhealthy, tunnelDown),
}))
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" {
t.Fatalf("want one condition at the hub, got %v", keys)
}
o := obs[0]
if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 {
t.Fatalf("the hub's condition is %+v", o)
}
}
func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) {
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)})
f.silent["anchor"] = true
if obs := networkObservations(f); len(obs) != 0 {
t.Fatalf("the hub's silence says it; got %v", keysOf(obs))
}
}
func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) {
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateUnhealthy, noRoute),
"laptop": aNetwork(link.StateUnhealthy, tunnelDown),
}))
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" {
t.Fatalf("want the hub's own, holding the laptop's, got %v", keys)
}
if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") {
t.Fatalf("the hub's condition is %+v", o)
}
}
func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) {
silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do",
Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}}
obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy),
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
}))
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" {
t.Fatalf("one machine alone: want its own, got %v", keys)
}
obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{
"anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy),
"laptop": aNetwork(link.StateUnhealthy, silentResolver),
}))
if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" {
t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys)
}
if !strings.Contains(obs[0].Summary, "its resolver") {
t.Fatalf("the resolver's machine is said %s", obs[0].Summary)
}
}
func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) {
f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)})
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
t.Fatalf("the control node's rewritten file: %+v", obs)
}
bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"}
f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)})
if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent {
t.Fatalf("the bus unreachable from the laptop: %+v", obs)
}
}
func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) {
f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)})
if obs := networkObservations(f); len(obs) != 0 {
t.Fatalf("got %v", keysOf(obs))
}
}
// The gate: a resolver file another program rewrote waits the judging rather than failing it at the
// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own
// network fault holds the machine as a whole, as before.
func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) {
since := h0
rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"},
Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork}
f := gateFacts{judged: true, open: []conditions.Condition{rewrite}}
if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 {
t.Fatalf("a rewrite the send did not make: %+v", w)
}
if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" ||
w.on["letta"] != "" || w.waiting != "" {
t.Fatalf("a rewrite of the file a moved module owns: %+v", w)
}
own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork}
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" {
t.Fatalf("the machine's own network: %+v", w)
}
unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}},
Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork}
if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true,
open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" {
t.Fatalf("a machine that cannot reach the hub: %+v", w)
}
}
cmd/mesh-controller/machine_units_integration_test.go
-125
View File
@@ -1,125 +0,0 @@
//go:build integration
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv, k := open.inventory, conditionsFrom
openConditions := func() map[string]conditions.Condition {
t.Helper()
list, err := k.Open(ctx)
if err != nil {
t.Fatal(err)
}
out := map[string]conditions.Condition{}
for _, c := range list {
out[c.Key] = c
}
return out
}
say := func(at time.Time, h link.Health) {
t.Helper()
if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil {
t.Fatal(err)
}
}
say(h0, desktop(h0))
got := openConditions()
machine, raised := got["machine.laptop.units"]
if !raised || len(got) != 1 {
t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got)
}
for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} {
if !strings.Contains(machine.Summary, unit) {
t.Errorf("the finding names %s: %s", unit, machine.Summary)
}
}
if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning {
t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine)
}
say(h0.Add(time.Minute), desktop(h0.Add(time.Minute)))
got = openConditions()
module, raised := got["module.openrazer.laptop.unhealthy"]
if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") ||
!strings.Contains(module.Summary, "account's own service manager") {
t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got)
}
kept, had, err := inv.HealthOf(ctx, "laptop")
if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 {
t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err)
}
lines := strings.Join(unitsLines(kept, had, h0), "\n")
for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it",
"openrazer-daemon.service — openrazer's"} {
if !strings.Contains(lines, want) {
t.Errorf("node show does not say %q:\n%s", want, lines)
}
}
// An older engine says nothing of its units: nothing cleared on its word.
older := desktop(h0.Add(2 * time.Minute))
older.Units = nil
say(h0.Add(2*time.Minute), older)
if _, still := openConditions()["machine.laptop.units"]; !still {
t.Fatal("a statement saying nothing of units cleared the machine's finding")
}
// The operator mends the mounts and removes the leftover link; the module's unit still fails.
mended := desktop(h0.Add(3 * time.Minute))
mended.Units.Failed = []link.FailedUnit{}
say(h0.Add(3*time.Minute), mended)
got = openConditions()
if _, still := got["machine.laptop.units"]; still {
t.Fatalf("no failed unit of the machine's own, still open: %v", got)
}
if _, still := got["module.openrazer.laptop.unhealthy"]; !still {
t.Fatalf("the module's unit still fails, and its condition stands: %v", got)
}
// And the module's unit mended: running, nothing open.
well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute),
Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}}
say(h0.Add(4*time.Minute), well)
if got = openConditions(); len(got) != 0 {
t.Fatalf("a running service manager leaves nothing open: %v", got)
}
}
// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged
// user unit, the module's; two mounts of the machine's own table and a unit a removed package left
// behind, the machine's.
func desktop(at time.Time) link.Health {
return link.Health{Contract: link.ReadinessContract, At: at,
Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit,
Target: "openrazer-daemon.service", State: link.StateUnhealthy,
Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}},
Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{
{Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0},
{Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0},
{Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0},
}}}
}
func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) {
since := h0
units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits}
if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true,
open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 {
t.Fatalf("the machine's own failed units held a send: %+v", w)
}
}
cmd/mesh-controller/main.go
+2 -23
View File
@@ -63,15 +63,6 @@ func run() error {
// 45 §6), so the next controller takes it at once rather than after its age.
defer theLease.release()
// Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save
// that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its
// own saves.
if args[0] != "serve" {
said, done := sayingWhatAVerbKept(ctx)
inventory.PlanSaved = said
defer done()
}
switch args[0] {
case "switch":
return switchCommand(ctx, args[1:])
@@ -94,7 +85,8 @@ func run() error {
return rehearseCommand(ctx, args[1:])
case "builds":
return buildsCommand(ctx, args[1:])
case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume":
case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume", "plans", "plan", "push", "deliver",
"delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery", "upgrade":
return removedByName(args[0])
case "try":
return tryCommand(ctx, args[1:])
@@ -113,11 +105,6 @@ func run() error {
return movePlacementCommand(ctx, args[1:])
case "move-config":
return moveConfigCommand(ctx, args[1:])
case "plans":
return plansCommand(ctx, args[1:])
case "delivery":
// The verbs the delivery's owner asks with (novox/hq ADR 0239).
return deliveryCommand(ctx, args[1:])
case "pin":
return pinCommand(ctx, args[1:], true)
case "unpin":
@@ -140,13 +127,9 @@ func run() error {
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9).
case "facts":
return factsCommand(ctx, args[1:])
case "upgrade":
return upgradeCommand(ctx, args[1:])
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
case "bus":
return busCommand(ctx, args[1:])
case "declare":
return declare(ctx, args[1:])
case "overlay":
return overlayCommand(ctx, args[1:])
case "module":
@@ -165,10 +148,6 @@ func run() error {
return secretCommand(ctx, args[1:])
case "operator":
return operatorCommand(ctx, args[1:])
case "plan":
return planCommand(ctx, args[1:])
case "push":
return pushCommand(ctx, args[1:])
case "rollout":
return rolloutCommand(ctx, args[1:])
case "seats":
cmd/mesh-controller/members_file_test.go
-81
View File
@@ -1,81 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"os"
"slices"
"strings"
"testing"
)
// aLongGroup is members as JSON over 200 KB: 1666 paths of the shape of mesh-delivery #5's change (novox/hq issue 1272).
func aLongGroup(t *testing.T) string {
t.Helper()
var paths []string
for i := 0; i < 1666; i++ {
paths = append(paths, fmt.Sprintf("modules/mesh-delivery-forge-bridge/vendor/github.com/nats-io/nats-server/v2/server/certstore/certstore_windows_file_%04d.go", i))
}
body, err := json.Marshal([]map[string]any{{"id": "novox/mesh-delivery#5", "paths": paths}, {"id": "novox/mesh-catalog#374"}})
if err != nil {
t.Fatal(err)
}
if len(body) < 200<<10 {
t.Fatalf("the members are only %d bytes", len(body))
}
return string(body)
}
func readBackFromTheCommand(t *testing.T, argv []string, members string) {
t.Helper()
for _, a := range argv {
if len(a) > 64<<10 {
t.Errorf("an argument of %d bytes is handed to the command", len(a))
}
}
i := slices.Index(argv, "--members-file")
if i < 0 {
t.Fatalf("the command does not name a members file: %.200v", argv)
}
raw, err := os.ReadFile(argv[i+1])
if err != nil || string(raw) != members {
t.Errorf("the file does not hold the whole members list (%d bytes, %v)", len(raw), err)
}
removeMembersFile(argv)
if _, err := os.Stat(argv[i+1]); !os.IsNotExist(err) {
t.Errorf("the file is still there after the command: %v", err)
}
}
func TestDeliveryOrderHandsALongMembersListInAFile(t *testing.T) { // 1272/AC2
members := aLongGroup(t)
argv, err := argvFor("delivery-order", map[string]any{"members": members})
if err != nil {
t.Fatal(err)
}
if argv[0] != "delivery" || argv[1] != "order" {
t.Errorf("the command is %.100v", strings.Join(argv, " "))
}
readBackFromTheCommand(t, argv, members)
}
func TestAShortMembersListIsStillTheArgumentItWas(t *testing.T) { // 1272/AC3
members := `[{"id":"novox/a#1"},{"id":"novox/b#2"}]`
argv, err := argvFor("delivery-order", map[string]any{"members": members})
want := []string{"delivery", "order", "--members", members}
if err != nil || !slices.Equal(argv, want) {
t.Errorf("a short list gives %v, %v, want %v", argv, err, want)
}
}
func TestTheCommandReadsTheMembersFromTheFileAsFromTheArgument(t *testing.T) { // 1272/AC1
members := `[{"id":"novox/a#1"}]`
argv, _ := membersFlags(strings.Repeat(" ", membersInArgument) + members)
defer removeMembersFile(argv)
if len(argv) != 2 || argv[0] != "--members-file" {
t.Fatalf("a long list gives %v", argv)
}
if st, err := os.Stat(argv[1]); err != nil || st.Mode().Perm() != 0o600 {
t.Errorf("the file is %v, %v", st, err)
}
}

Some files were not shown because too many files have changed in this diff Show More