jschoubben
b482122d66
mesh/merge-gate head b482122d6646 merged into trunk 1601cb17432e — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head b482122d6646 merged into trunk 1601cb17432e — pass: its merge-check.sh passed
mesh/delivery delivered
An empty answer on a read error would clear release-silent unseen.
373 lines
15 KiB
Go
373 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by
|
|
// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
|
|
// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
|
|
//
|
|
// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
|
|
// eleven minutes; the new controller its last send started sent the control node again the controller build it
|
|
// had just applied, and judged it from its first reading.
|
|
|
|
// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
|
|
func aBacklogOnThree(t *testing.T) *backlogMesh {
|
|
t.Helper()
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
if _, err := inv.AddNode(ctx, "desk"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wasHeard := releaseHeard
|
|
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
|
return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
|
|
}
|
|
t.Cleanup(func() { releaseHeard = wasHeard })
|
|
return b
|
|
}
|
|
|
|
// events records every send and every judging, in order.
|
|
func events(t *testing.T) *[]string {
|
|
t.Helper()
|
|
var said []string
|
|
wasSend, wasGather := sendRollout, gatherGateFacts
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
said = append(said, "send "+strings.Join(names, ","))
|
|
return wasSend(ctx, open, names)
|
|
}
|
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
said = append(said, "judge")
|
|
return wasGather(ctx, open, component)
|
|
}
|
|
t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
|
|
return &said
|
|
}
|
|
|
|
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
|
|
// was judged to its verdict before the third was sent.
|
|
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklogOnThree(t)
|
|
ctx := t.Context()
|
|
said := events(t)
|
|
gateEvery = 0
|
|
for i := 0; i < 12; i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
p := b.release(t)
|
|
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
|
|
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
|
|
t.Fatalf("sends and judgings: %v", *said)
|
|
}
|
|
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
|
|
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
|
|
}
|
|
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
|
}
|
|
}
|
|
|
|
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
|
|
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
|
|
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklogOnThree(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
gateEvery = 0
|
|
b.broken["desk"] = true // desk fails what it is sent in the rest's step
|
|
advancePlans(ctx, b.open)
|
|
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
p := b.release(t)
|
|
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
|
|
t.Fatalf("laptop was never sent: %v", b.sent)
|
|
}
|
|
if !slices.Contains(p.Release.Done, "laptop") {
|
|
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
|
|
}
|
|
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
|
|
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
|
}
|
|
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
|
|
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
|
t.Fatal("app's build is not marked failed")
|
|
}
|
|
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
|
|
t.Fatalf("desk was not put back: %v", sent)
|
|
}
|
|
}
|
|
|
|
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
|
|
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
|
|
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
|
|
release := b.release(t)
|
|
judging := release.Release.Gate
|
|
if judging == nil || judging.Pending || judging.Passes == 0 {
|
|
t.Fatalf("the release is not judging anchor: %+v", judging)
|
|
}
|
|
built := time.Now().UTC()
|
|
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
|
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
|
Build: "build-app-c2"}}}
|
|
if err := inv.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, b.open)
|
|
if len(b.sent) != 1 {
|
|
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
|
|
}
|
|
p, err := inv.PlanByID(ctx, "plan-app")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
g := p.Modules["app"].Gate
|
|
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
|
|
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
|
|
}
|
|
if p.Modules["app"].Previous != "c1" {
|
|
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
|
|
}
|
|
}
|
|
|
|
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
|
|
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
|
|
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
g := aGateMesh(t)
|
|
ctx := t.Context()
|
|
// The old controller sends anchor app c2 and is replaced before it keeps the send.
|
|
made := sendRollout
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
if _, err := made(ctx, open, names); err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, context.Canceled
|
|
}
|
|
advancePlans(ctx, g.open)
|
|
sendRollout = made
|
|
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
|
|
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
|
|
}
|
|
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, g.open)
|
|
if len(g.sent) != 1 {
|
|
t.Fatalf("the send was made again: %v", g.sent)
|
|
}
|
|
s := g.plan(t).Modules["app"]
|
|
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
|
|
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
|
|
}
|
|
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
|
|
if s.Previous != "c1" || s.Gate.From != "c1" {
|
|
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
|
|
}
|
|
}
|
|
|
|
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
|
// made again (ADR 0353 §8).
|
|
func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
p, err := releaseBacklog(ctx, b.open, "")
|
|
if err != nil || p == nil {
|
|
t.Fatalf("no release opened: %v", err)
|
|
}
|
|
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
|
|
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
|
}
|
|
if err := inv.SavePlan(ctx, p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The send is made, and the controller is replaced before it keeps it.
|
|
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, b.open)
|
|
if len(b.sent) != 1 {
|
|
t.Fatalf("anchor was sent again: %v", b.sent)
|
|
}
|
|
g := b.release(t).Release.Gate
|
|
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
|
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
|
}
|
|
}
|
|
|
|
// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
|
|
// for the next step, and one whose every build passed goes now (ADR 0353 §2).
|
|
func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
|
|
waiting := map[string][]inventory.CarriedMove{
|
|
"desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
|
"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
|
"shanks": {{Module: "app", To: "c2"}},
|
|
"novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
|
|
}
|
|
passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
|
|
never := func(string) bool { return false }
|
|
step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
|
|
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
|
t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
|
|
}
|
|
if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
|
|
t.Fatalf("a machine whose every build passed waits: %v", step)
|
|
}
|
|
}
|
|
|
|
// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
|
|
// carries either to one machine only, while what else passed goes everywhere at once.
|
|
func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
|
|
waiting := map[string][]inventory.CarriedMove{
|
|
"desk": {{Module: hostModule, To: "h2"}},
|
|
"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
|
|
"shanks": {{Module: "app", To: "c2"}},
|
|
"novox": {{Module: "dns", To: "d2"}},
|
|
"ace": {{Module: "dns", To: "d2"}},
|
|
}
|
|
resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
|
|
alone := func(m string) bool {
|
|
if m == "dns" {
|
|
return oneAtATime(m, resolver)
|
|
}
|
|
return oneAtATime(m, catalogue.Manifest{Module: m})
|
|
}
|
|
step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
|
|
func(string, string) bool { return true }, alone)
|
|
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
|
t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
|
|
}
|
|
if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
|
|
t.Fatal("a leaf module is held to one machine at a time")
|
|
}
|
|
}
|
|
|
|
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
|
|
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
|
|
b := aBacklogOnThree(t)
|
|
ctx := t.Context()
|
|
gateEvery = 0
|
|
b.broken["anchor"] = true
|
|
advancePlans(ctx, b.open)
|
|
for i := 0; i < 6 && b.release(t).Open(); i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
for _, s := range b.sent {
|
|
if !slices.Equal(s, []string{"anchor"}) {
|
|
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
|
|
// ordered after every other.
|
|
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklogOnThree(t)
|
|
ctx := t.Context()
|
|
register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
|
|
if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
said := events(t)
|
|
gateEvery = 0
|
|
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
|
|
t.Fatalf("the order is %v", p.Release.Order)
|
|
}
|
|
laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
|
|
if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
|
|
t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
|
|
}
|
|
}
|
|
|
|
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
|
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
|
|
func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
reporting := sendRollout
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
|
b.sent = append(b.sent, names)
|
|
current, _ := open.inventory.CurrentBuilds(ctx)
|
|
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
|
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
|
}
|
|
return reporting(ctx, open, names)
|
|
}
|
|
t.Cleanup(func() { sendRollout = reporting })
|
|
gateEvery = 0
|
|
for i := 0; i < 8; i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
p := b.release(t)
|
|
g := p.Release.Gates["laptop"]
|
|
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
|
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
|
}
|
|
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
|
g.Since = &since
|
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, b.open)
|
|
p = b.release(t)
|
|
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
|
t.Fatal("a silent machine's build was marked failed")
|
|
}
|
|
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
|
t.Fatalf("laptop was put back: %v", sent)
|
|
}
|
|
obs, err := silentObservation(ctx, inv)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
|
t.Fatalf("the silence is not said: %+v", obs)
|
|
}
|
|
}
|