From 0e974138751ef287d096dc73667e41864727f4f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 19:21:42 +0200 Subject: [PATCH] Assert the backlog walk sends every machine at once and passes on the report (hq ADR 0360) --- cmd/mesh-controller/release_test.go | 7 +- cmd/mesh-controller/rest_at_once_test.go | 149 ++++------------------- 2 files changed, 28 insertions(+), 128 deletions(-) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 35b7d35c..b7a6c6c0 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -111,8 +111,13 @@ func aBacklog(t *testing.T) *backlogMesh { if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { return nil, err } + // A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict). + outcome := inventory.OutcomeApplied + if b.broken[node] { + outcome = inventory.OutcomeFailed + } if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil { return nil, err } } diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 223d5e76..39cebc05 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -59,14 +59,13 @@ func events(t *testing.T) *[]string { return &said } -// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second -// was judged to its verdict before the third was sent. -func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { +// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); +// before, the first machine was judged before the rest were sent (ADR 0353 §1). +func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() said := events(t) - gateEvery = 0 for i := 0; i < 12; i++ { advancePlans(ctx, b.open) } @@ -74,27 +73,27 @@ func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } - desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") - if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { - t.Fatalf("sends and judgings: %v", *said) + for _, n := range []string{"desk", "laptop", "anchor"} { + if !slices.Contains(*said, "send "+n) { + t.Fatalf("%s was not sent: %v", n, *said) + } } - if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { - t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) + if slices.Contains(*said, "judge") { + t.Fatalf("a machine was judged by its health: %v", *said) } if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } } -// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own -// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. -func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { +// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other +// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). +func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() inv := b.open.inventory - gateEvery = 0 - b.broken["desk"] = true // desk fails what it is sent in the rest's step + b.broken["desk"] = true // desk fails what it is sent advancePlans(ctx, b.open) for i := 0; i < 12 && b.release(t).Open(); i++ { advancePlans(ctx, b.open) @@ -103,97 +102,14 @@ func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } - if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { - t.Fatalf("laptop was never sent: %v", b.sent) + if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the others have no verdict of their own: %+v", p.Release) } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop has no verdict of its own: %+v", p.Release) - } - // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } - // app failed on desk: marked, and put back wherever it was sent (ADR 0236). - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { - t.Fatalf("desk was not put back: %v", sent) - } -} - -// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that -// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. -func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once - release := b.release(t) - judging := release.Release.Gate - if judging == nil || judging.Pending || judging.Passes == 0 { - t.Fatalf("the release is not judging anchor: %+v", judging) - } - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-app-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again what it runs: %v", b.sent) - } - p, err := inv.PlanByID(ctx, "plan-app") - if err != nil { - t.Fatal(err) - } - g := p.Modules["app"].Gate - if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { - t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) - } - if p.Modules["app"].Previous != "c1" { - t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) - } -} - -// A walk's first send that was made and not kept — the controller replaced between the two — is not made again -// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). -func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - // The old controller sends anchor app c2 and is replaced before it keeps the send. - made := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if _, err := made(ctx, open, names); err != nil { - return nil, err - } - return nil, context.Canceled - } - advancePlans(ctx, g.open) - sendRollout = made - if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { - t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) - } - last, _, err := g.open.inventory.SentTo(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if len(g.sent) != 1 { - t.Fatalf("the send was made again: %v", g.sent) - } - s := g.plan(t).Modules["app"] - if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { - t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) - } - // What anchor ran before was kept before the send, so a failure still puts it back (§5). - if s.Previous != "c1" || s.Gate.From != "c1" { - t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { + t.Fatalf("desk was put back: %v", sent) } } @@ -208,7 +124,7 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { if err != nil || p == nil { t.Fatalf("no release opened: %v", err) } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) } if err := inv.SavePlan(ctx, p); err != nil { @@ -219,35 +135,15 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { t.Fatal(err) } advancePlans(ctx, b.open) - if len(b.sent) != 1 { + if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { t.Fatalf("anchor was sent again: %v", b.sent) } - g := b.release(t).Release.Gate + g := b.release(t).Release.Gates["anchor"] if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { t.Fatalf("anchor is not judged on the send made: %+v", g) } } -// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. -func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { - b := aBacklogOnThree(t) - ctx := t.Context() - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - for i := 0; i < 6 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, s := range b.sent { - if !slices.Equal(s, []string{"anchor"}) { - t.Fatalf("sent %v: the rest after a failed first machine", b.sent) - } - } -} - // The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is // ordered after every other. func TestTheControlNodeIsSentLastInAStep(t *testing.T) { @@ -259,7 +155,6 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) { t.Fatal(err) } said := events(t) - gateEvery = 0 for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { advancePlans(ctx, b.open) } @@ -290,7 +185,6 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { return reporting(ctx, open, names) } t.Cleanup(func() { sendRollout = reporting }) - gateEvery = 0 for i := 0; i < 8; i++ { advancePlans(ctx, b.open) } @@ -306,7 +200,8 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { } advancePlans(ctx, b.open) p = b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). + if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {