diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go new file mode 100644 index 00000000..dc926d98 --- /dev/null +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -0,0 +1,234 @@ +package main + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by +// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them — +// a machine already sent what a send is for is not sent again, and its judging goes on from what was read. +// +// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in +// eleven minutes; the new controller its last send started sent the control node again the controller build it +// had just applied, and judged it from its first reading. + +// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others. +func aBacklogOnThree(t *testing.T) *backlogMesh { + t.Helper() + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + if _, err := inv.AddNode(ctx, "desk"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "desk", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + wasHeard := releaseHeard + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil + } + t.Cleanup(func() { releaseHeard = wasHeard }) + return b +} + +// events records every send and every judging, in order. +func events(t *testing.T) *[]string { + t.Helper() + var said []string + wasSend, wasGather := sendRollout, gatherGateFacts + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + said = append(said, "send "+strings.Join(names, ",")) + return wasSend(ctx, open, names) + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + said = append(said, "judge") + return wasGather(ctx, open, component) + } + t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather }) + return &said +} + +// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second +// was judged to its verdict before the third was sent. +func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + gateEvery = 0 + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") + if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { + t.Fatalf("sends and judgings: %v", *said) + } + if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { + t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own +// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. +func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + gateEvery = 0 + b.broken["desk"] = true // desk fails what it is sent in the rest's step + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { + t.Fatalf("laptop was never sent: %v", b.sent) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop has no verdict of its own: %+v", p.Release) + } + // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + // app failed on desk: marked, and put back wherever it was sent (ADR 0236). + if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { + t.Fatal("app's build is not marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { + t.Fatalf("desk was not put back: %v", sent) + } +} + +// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that +// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. +func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once + release := b.release(t) + judging := release.Release.Gate + if judging == nil || judging.Pending || judging.Passes == 0 { + t.Fatalf("the release is not judging anchor: %+v", judging) + } + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-app-c2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again what it runs: %v", b.sent) + } + p, err := inv.PlanByID(ctx, "plan-app") + if err != nil { + t.Fatal(err) + } + g := p.Modules["app"].Gate + if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { + t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) + } + if p.Modules["app"].Previous != "c1" { + t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) + } +} + +// A walk's first send that was made and not kept — the controller replaced between the two — is not made again +// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). +func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + // The old controller sent anchor app c2 and was replaced before it kept the send. + if _, err := sendRollout(ctx, g.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + last, _, err := g.open.inventory.SentTo(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if len(g.sent) != 1 { + t.Fatalf("the send was made again: %v", g.sent) + } + s := g.plan(t).Modules["app"] + if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { + t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) + } +} + +// A release's send kept before it was made, and made, is judged by the controller that takes over without being +// made again (ADR 0353 §8). +func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gate + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits +// for the next step, and one whose every build passed goes now (ADR 0353 §2). +func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, + } + passed := func(module, commit string) bool { return module == "app" && commit == "c2" } + step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) + } + if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }); !slices.Equal(step, []string{"laptop"}) { + t.Fatalf("a machine whose every build passed waits: %v", step) + } +} diff --git a/testdata/store-tests b/testdata/store-tests index bbfc1467..4d3eaa55 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -61,3 +61,8 @@ ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName ./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne ./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack +./cmd/mesh-controller TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses +./cmd/mesh-controller TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn +./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere +./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain +./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain