novox/mesh-controller
7
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Assert the backlog walk sends every machine at once and passes on the report (hq ADR 0360)
mesh/merge-gate head 0e974138751e merged into trunk 76304d9db817 — pass: builds mesh-controller → novox; no bus step; every machine composes with the chang…
Details
mesh/repo-check head 0e974138751e merged into trunk 76304d9db817 — fail: its merge-check.sh failed: --- FAIL: TestAReusedCheckBuildIsRegisteredByItsDigestA…
Details
mesh/delivery superseded: a newer head of the same pull request
Details

Browse Source
This commit is contained in:
jschoubben
2026-10-14 19:21:42 +02:00
parent da8a5e9bc0
commit 0e97413875
2 changed files with 28 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/release_test.go
+6 -1
View File
@@ -111,8 +111,13 @@ func aBacklog(t *testing.T) *backlogMesh {
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
// A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict).
outcome := inventory.OutcomeApplied
if b.broken[node] {
outcome = inventory.OutcomeFailed
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
cmd/mesh-controller/rest_at_once_test.go
+22 -127
View File
@@ -59,14 +59,13 @@ func events(t *testing.T) *[]string {
return &said
}
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
// was judged to its verdict before the third was sent.
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklogOnThree(t)
ctx := t.Context()
said := events(t)
gateEvery = 0
for i := 0; i < 12; i++ {
advancePlans(ctx, b.open)
}
@@ -74,27 +73,27 @@ func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
t.Fatalf("sends and judgings: %v", *said)
for _, n := range []string{"desk", "laptop", "anchor"} {
if !slices.Contains(*said, "send "+n) {
t.Fatalf("%s was not sent: %v", n, *said)
}
}
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
if slices.Contains(*said, "judge") {
t.Fatalf("a machine was judged by its health: %v", *said)
}
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
}
}
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklogOnThree(t)
ctx := t.Context()
inv := b.open.inventory
gateEvery = 0
b.broken["desk"] = true // desk fails what it is sent in the rest's step
b.broken["desk"] = true // desk fails what it is sent
advancePlans(ctx, b.open)
for i := 0; i < 12 && b.release(t).Open(); i++ {
advancePlans(ctx, b.open)
@@ -103,97 +102,14 @@ func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
t.Fatalf("laptop was never sent: %v", b.sent)
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
}
if !slices.Contains(p.Release.Done, "laptop") {
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
}
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
}
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
t.Fatal("app's build is not marked failed")
}
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
t.Fatalf("desk was not put back: %v", sent)
}
}
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
release := b.release(t)
judging := release.Release.Gate
if judging == nil || judging.Pending || judging.Passes == 0 {
t.Fatalf("the release is not judging anchor: %+v", judging)
}
built := time.Now().UTC()
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
Build: "build-app-c2"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
advancePlans(ctx, b.open)
if len(b.sent) != 1 {
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
}
p, err := inv.PlanByID(ctx, "plan-app")
if err != nil {
t.Fatal(err)
}
g := p.Modules["app"].Gate
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
}
if p.Modules["app"].Previous != "c1" {
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
}
}
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
// The old controller sends anchor app c2 and is replaced before it keeps the send.
made := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
if _, err := made(ctx, open, names); err != nil {
return nil, err
}
return nil, context.Canceled
}
advancePlans(ctx, g.open)
sendRollout = made
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
}
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if len(g.sent) != 1 {
t.Fatalf("the send was made again: %v", g.sent)
}
s := g.plan(t).Modules["app"]
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
}
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
if s.Previous != "c1" || s.Gate.From != "c1" {
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
t.Fatalf("desk was put back: %v", sent)
}
}
@@ -208,7 +124,7 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
if err != nil || p == nil {
t.Fatalf("no release opened: %v", err)
}
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
}
if err := inv.SavePlan(ctx, p); err != nil {
@@ -219,35 +135,15 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
t.Fatal(err)
}
advancePlans(ctx, b.open)
if len(b.sent) != 1 {
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
t.Fatalf("anchor was sent again: %v", b.sent)
}
g := b.release(t).Release.Gate
g := b.release(t).Release.Gates["anchor"]
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
t.Fatalf("anchor is not judged on the send made: %+v", g)
}
}
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
b := aBacklogOnThree(t)
ctx := t.Context()
gateEvery = 0
b.broken["anchor"] = true
advancePlans(ctx, b.open)
for i := 0; i < 6 && b.release(t).Open(); i++ {
advancePlans(ctx, b.open)
}
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
for _, s := range b.sent {
if !slices.Equal(s, []string{"anchor"}) {
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
}
}
}
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
// ordered after every other.
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
@@ -259,7 +155,6 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
t.Fatal(err)
}
said := events(t)
gateEvery = 0
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
advancePlans(ctx, b.open)
}
@@ -290,7 +185,6 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
return reporting(ctx, open, names)
}
t.Cleanup(func() { sendRollout = reporting })
gateEvery = 0
for i := 0; i < 8; i++ {
advancePlans(ctx, b.open)
}
@@ -306,7 +200,8 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
}
advancePlans(ctx, b.open)
p = b.release(t)
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {