From 6a5b3cadbca9b48263ba4832ce571d320e5ea1ae Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 18:41:02 +0000 Subject: [PATCH] Bring F3 up to its release branch, with both sides' store tests (part 3 of 14) Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/release_plan.go | 193 +++++++++++- cmd/mesh-controller/rest_at_once_test.go | 372 +++++++++++++++++++++++ cmd/mesh-controller/seatverbs.go | 28 ++ 3 files changed, 583 insertions(+), 10 deletions(-) create mode 100644 cmd/mesh-controller/rest_at_once_test.go diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 77060d8b..f28516fa 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -251,6 +251,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) return out } +// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine +// running it runs (novox/hq issue 280): nothing is sent for it. +// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader. +const ( + noMovePrefix = "no move" + noMoveWhy = noMovePrefix + ": made from the source every machine running it runs" +) + +// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says, +// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648). +func noMove(s *inventory.PlanModule) bool { + return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix) +} + // applied says whether every machine running the module has reported since the module was built. // appliedEach is applied with a moment of its own for each machine: the reports that count are the ones // after that machine was sent the build (novox/hq issue 256). @@ -595,7 +609,7 @@ func advancePlans(ctx context.Context, open *stores) { return } defer release() - // What waits for a gate, released one machine at a time (ADR 0236). + // What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353). if _, err := releaseBacklog(ctx, open, ""); err != nil { fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) } @@ -805,7 +819,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, } else if same { now := time.Now().UTC() state.SentAt = &now - state.Why = "no move: made from the source every machine running it runs" + state.Why = noMoveWhy fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m) continue } @@ -995,6 +1009,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if state.BuiltAt != nil { since = *state.BuiltAt } + // **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648). + // Nothing was sent for it, so no report follows its marking or its build: a machine that does not + // happen to report again held the walk for ever. Each machine runs it once it answered its last + // send — the one whose builds made it no move — whenever that was. + if noMove(state) { + byNode := map[string]inventory.Reported{} + for _, r := range reports { + byNode[r.Node] = r + } + if ok, on := appliedEach(m, func(node string) time.Time { + if s := byNode[node].Sent; s != nil { + return *s + } + return time.Time{} + }, running, reports); !ok { + waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", "))) + } + continue + } // **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218) // a module is sent twice — the first machine, then the rest — and SentAt is the second. // Asked of every machine, the first machine's report, made between the two sends, read as @@ -1056,10 +1089,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string runningOf map[string][]string) error { inv := open.inventory before, beforeKnown, _ := inv.SentBuilds(ctx, node) + // **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan + // before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from + // the readings of a gate already judging it there. + // Only a send the machine reported it applied is one it has; one it never reported on is sent, as before + // (issue 280: what it ran before is then that build, and a failure leaves it as it was). + if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 && + appliedLastSend(ctx, inv, node) { + if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 { + adoptSend(ctx, open, p, node, modules, before, runningOf) + return nil + } + } + // **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced + // between the send and keeping it reads it here, and a failure still puts the machine back. + kept := false + for _, m := range modules { + s := p.Modules[m] + if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] { + s.Previous, kept = before[m], true + } + } + if kept { + if err := inv.SavePlan(ctx, p); err != nil { + return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err) + } + } owns := make([]inventory.CarriedMove, 0, len(modules)) for _, m := range modules { s := p.Modules[m] - owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build}) + owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build}) } // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges // all of it there. @@ -1074,10 +1133,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string sentWhat := sentNow(ctx, inv, sent) for _, m := range modules { s := p.Modules[m] - // What the first machine ran before: what a failed gate puts back (ADR 0236). - if beforeKnown { - s.Previous = before[m] - } + // What the first machine ran before (ADR 0236), kept above before the send. s.First, s.FirstAt = sent, &now s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]), From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat} @@ -1106,6 +1162,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string return nil } +// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts +// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build +// there — whose carried move also says what the machine ran before, for a rollback. +func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, + before map[string]string, runningOf map[string][]string) { + inv := open.inventory + since := time.Now().UTC() + last, found, err := inv.SentTo(ctx, node) + if err == nil && found && !last.At.IsZero() { + since = last.At + } + lead := modules[0] + other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build) + for _, m := range modules { + s := p.Modules[m] + // What it ran before: kept by this walk before its send was made (§6); else what the other walk's send + // moved it from; else the build itself, so a failure leaves it as it was (issue 280). + switch { + case s.Previous != "": + case m == lead && from != "": + s.Previous = from + default: + s.Previous = before[m] + } + at := since + s.First, s.FirstAt = []string{node}, &at + s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]), + From: s.Previous, To: s.Commit, Since: &at} + if found { + s.Gate.Sent = map[string]inventory.SentDeclaration{node: last} + } + s.GatedBy = "" + if m == lead { + s.Gate.TakeReadings(other, modules, id) + } else { + s.GatedBy = lead + } + } + p.State = inventory.PlanRolling + p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send", + p.Tier, node, strings.Join(modules, ", ")) + if id != "" { + p.Note += ", going on from the readings of " + id + } + fmt.Printf("%s: %s\n", p.ID, p.Note) +} + +// appliedLastSend says a machine reported it applied the declaration it was sent last. +func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool { + reports, err := inv.LastReports(ctx) + if err != nil { + return false + } + for _, r := range reports { + if r.Node == node { + return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied + } + } + return false +} + +// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the +// walk's id, and what the machine ran before; nil when none does. +func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) { + plans, err := inv.OpenPlans(ctx) + if err != nil { + return nil, "", "" + } + // The same build, by its id where both name one, else by its commit. + same := func(b, c string) bool { + if b != "" && build != "" { + return b == build + } + return sameCommit(c, commit) + } + usable := func(g *inventory.PlanGate) bool { + return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) && + slices.Contains(g.Machines, node) + } + for _, q := range plans { + if q.ID == self { + continue + } + if r := q.Release; r != nil { + gates := []*inventory.PlanGate{r.Gate} + for _, g := range r.Gates { + gates = append(gates, g) + } + for _, g := range gates { + if !usable(g) { + continue + } + for _, c := range g.Carried { + if c.Module == module && c.Node == node && same(c.Build, c.To) { + return g, q.ID, c.From + } + } + } + continue + } + if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) { + return s.Gate, q.ID, s.Previous + } + } + return nil, "", "" +} + // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate // on the first of them passed, and its pass was kept for every build it carried (passCarried). func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) { @@ -1594,9 +1757,19 @@ func plansCommand(ctx context.Context, args []string) error { // A release plan's walk (ADR 0236): machines done, the one judged, those to come. fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", "))) - if r.Gate != nil { - fmt.Printf(" %s\n", gateLine(r.Gate)) - for _, c := range r.Gate.Carried { + if len(r.Failed)+len(r.Silent) > 0 { + fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", "))) + } + gates := []*inventory.PlanGate{r.Gate} + for _, n := range r.Order { + gates = append(gates, r.Gates[n]) + } + for _, g := range gates { + if g == nil { + continue + } + fmt.Printf(" %s\n", gateLine(g)) + for _, c := range g.Carried { fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To)) if c.Recreates != "" { fmt.Printf(" %-22s %s\n", "", c.Recreates) diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go new file mode 100644 index 00000000..8b6f7155 --- /dev/null +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -0,0 +1,372 @@ +package main + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by +// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them — +// a machine already sent what a send is for is not sent again, and its judging goes on from what was read. +// +// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in +// eleven minutes; the new controller its last send started sent the control node again the controller build it +// had just applied, and judged it from its first reading. + +// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others. +func aBacklogOnThree(t *testing.T) *backlogMesh { + t.Helper() + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + if _, err := inv.AddNode(ctx, "desk"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "desk", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + wasHeard := releaseHeard + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil + } + t.Cleanup(func() { releaseHeard = wasHeard }) + return b +} + +// events records every send and every judging, in order. +func events(t *testing.T) *[]string { + t.Helper() + var said []string + wasSend, wasGather := sendRollout, gatherGateFacts + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + said = append(said, "send "+strings.Join(names, ",")) + return wasSend(ctx, open, names) + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + said = append(said, "judge") + return wasGather(ctx, open, component) + } + t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather }) + return &said +} + +// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second +// was judged to its verdict before the third was sent. +func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + gateEvery = 0 + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") + if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { + t.Fatalf("sends and judgings: %v", *said) + } + if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { + t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own +// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. +func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + gateEvery = 0 + b.broken["desk"] = true // desk fails what it is sent in the rest's step + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { + t.Fatalf("laptop was never sent: %v", b.sent) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop has no verdict of its own: %+v", p.Release) + } + // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + // app failed on desk: marked, and put back wherever it was sent (ADR 0236). + if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { + t.Fatal("app's build is not marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { + t.Fatalf("desk was not put back: %v", sent) + } +} + +// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that +// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. +func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once + release := b.release(t) + judging := release.Release.Gate + if judging == nil || judging.Pending || judging.Passes == 0 { + t.Fatalf("the release is not judging anchor: %+v", judging) + } + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-app-c2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again what it runs: %v", b.sent) + } + p, err := inv.PlanByID(ctx, "plan-app") + if err != nil { + t.Fatal(err) + } + g := p.Modules["app"].Gate + if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { + t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) + } + if p.Modules["app"].Previous != "c1" { + t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) + } +} + +// A walk's first send that was made and not kept — the controller replaced between the two — is not made again +// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). +func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + // The old controller sends anchor app c2 and is replaced before it keeps the send. + made := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if _, err := made(ctx, open, names); err != nil { + return nil, err + } + return nil, context.Canceled + } + advancePlans(ctx, g.open) + sendRollout = made + if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { + t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) + } + last, _, err := g.open.inventory.SentTo(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if len(g.sent) != 1 { + t.Fatalf("the send was made again: %v", g.sent) + } + s := g.plan(t).Modules["app"] + if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { + t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) + } + // What anchor ran before was kept before the send, so a failure still puts it back (§5). + if s.Previous != "c1" || s.Gate.From != "c1" { + t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) + } +} + +// A release's send kept before it was made, and made, is judged by the controller that takes over without being +// made again (ADR 0353 §8). +func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gate + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits +// for the next step, and one whose every build passed goes now (ADR 0353 §2). +func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, + } + passed := func(module, commit string) bool { return module == "app" && commit == "c2" } + never := func(string) bool { return false } + step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) + } + if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { + t.Fatalf("a machine whose every build passed waits: %v", step) + } +} + +// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step +// carries either to one machine only, while what else passed goes everywhere at once. +func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: hostModule, To: "h2"}}, + "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "dns", To: "d2"}}, + "ace": {{Module: "dns", To: "d2"}}, + } + resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} + alone := func(m string) bool { + if m == "dns" { + return oneAtATime(m, resolver) + } + return oneAtATime(m, catalogue.Manifest{Module: m}) + } + step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, + func(string, string) bool { return true }, alone) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) + } + if oneAtATime("app", catalogue.Manifest{Module: "app"}) { + t.Fatal("a leaf module is held to one machine at a time") + } +} + +// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. +func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { + b := aBacklogOnThree(t) + ctx := t.Context() + gateEvery = 0 + b.broken["anchor"] = true + advancePlans(ctx, b.open) + for i := 0; i < 6 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, s := range b.sent { + if !slices.Equal(s, []string{"anchor"}) { + t.Fatalf("sent %v: the rest after a failed first machine", b.sent) + } + } +} + +// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is +// ordered after every other. +func TestTheControlNodeIsSentLastInAStep(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) + if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { + t.Fatal(err) + } + said := events(t) + gateEvery = 0 + for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { + t.Fatalf("the order is %v", p.Release.Order) + } + laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") + if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { + t.Fatalf("the control node is not sent last in the rest's step: %v", *said) + } +} + +// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put +// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). +func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + gateEvery = 0 + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 3cc62d1a..c0c222e3 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -457,6 +457,28 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil + case "try": + // A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered. + if id := str("id"); id != "" { + return []string{"try", "--id", id}, nil + } + if a.given["repository"] == "" && a.given["ref"] == "" { + // Neither shape named: the command says its usage, which names both, as `build` does. + return []string{"try"}, nil + } + if err := need("repository", "ref"); err != nil { + return nil, err + } + argv := []string{"try", str("repository"), str("ref")} + for _, flag := range []string{"base", "tests"} { + if v := str(flag); v != "" { + argv = append(argv, "--"+flag, v) + } + } + if on("store-only") { + argv = append(argv, "--store-only") + } + return argv, nil case "cancel", "kill": if err := need("id"); err != nil { return nil, err @@ -1018,6 +1040,8 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co "delivery": true, // The build queue, each ask with its position among the waiting (novox/hq issue 455). "queue": true, + // A branch tried before its pull request (novox/hq issue 686). + "try": true, // A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data. "plans": true} @@ -1308,6 +1332,10 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { link.AnswersWithin(ctx, bound+planWaitStart) } } + if verb == "try" && len(argv) > 1 && argv[1] == "--id" { + // Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229). + link.AnswersWithin(ctx, tryWait+planWaitStart) + } if answer, read := readHere(ctx, argv); read { return answer, nil }