From da8a5e9bc0f769e89f3fcac950a1e3ac757d81b7 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 19:19:49 +0200 Subject: [PATCH 1/4] Send a merge's builds to every machine at once, with no first-machine gate (hq ADR 0360) The operator chose the leanest rollout: no gate, no one machine first, failures loud and fixed forward. A walk sends every rolled-out module to every machine running it in one step, built-by orders no tier, and a walk ends on its machines' reports. The backlog walk sends every machine at once and passes each on its applied report. --- cmd/mesh-controller/lean_walk_test.go | 41 +++++ cmd/mesh-controller/planner_rules_test.go | 20 ++- cmd/mesh-controller/release.go | 208 ++-------------------- cmd/mesh-controller/release_plan.go | 136 ++++++++------ cmd/mesh-controller/release_plan_test.go | 26 +-- cmd/mesh-controller/rest_at_once_test.go | 47 ----- cmd/mesh-controller/worker_order_test.go | 5 +- 7 files changed, 168 insertions(+), 315 deletions(-) create mode 100644 cmd/mesh-controller/lean_walk_test.go diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go new file mode 100644 index 00000000..560ba49d --- /dev/null +++ b/cmd/mesh-controller/lean_walk_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "slices" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send, +// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading. +func TestAWalkEndsOnItsMachinesReports(t *testing.T) { + now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + sent := now.Add(-time.Minute) + long := now.Add(-inventory.ApplySilentAfter - time.Minute) + at := now.Add(-10 * time.Second) + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ + "app": {Rest: map[string]inventory.SentDeclaration{ + "ace": {Digest: "d-ace", At: sent}, + "novox": {Digest: "d-novox", At: sent}, + "shanks": {Digest: "d-shanks", At: sent}, + "laptop": {Digest: "d-laptop", At: long}, + }}, + "proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}}, + }} + reports := []inventory.Reported{ + {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, + {Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"}, + {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"}, + } + waiting, failed, silent := walkReports(p, reports, now) + if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) || + !slices.Equal(silent, []string{"laptop"}) { + t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent) + } + reports[2].Declared = "d-shanks" + if waiting, _, _ := walkReports(p, reports, now); len(waiting) != 0 { + t.Fatalf("the walk still waits once every machine reported: %v", waiting) + } +} diff --git a/cmd/mesh-controller/planner_rules_test.go b/cmd/mesh-controller/planner_rules_test.go index 69ed505f..9e422c3a 100644 --- a/cmd/mesh-controller/planner_rules_test.go +++ b/cmd/mesh-controller/planner_rules_test.go @@ -145,8 +145,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) { repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, {what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")}, repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - {what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, + // Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered. + {what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")}, + repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, {what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")}, repo: "one", paths: []string{"modules/a/x"}, want: "a"}, {what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")}, @@ -296,18 +297,18 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { []string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read, "mesh-controller | build-agent", ""}, // A package all three build from: all three; the build seat's holder after the controller whose worker - // it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by). + // it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3). {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"a vendored package all three build from: all three", "mesh-controller", - []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"go.sum: all three", "mesh-controller", - []string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"a file added to the proxy's package: the proxy", "mesh-controller", []string{"examples/route-proxy/new.go"}, read, "route-proxy", ""}, // Before any build source is said: as before. {"no build source said: a README moves all three, as before", "mesh-controller", - []string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""}, + []string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""}, // In the catalogue, where they live, each by its own build source. {"the route proxy's recipe: it alone", "mesh-catalog", []string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""}, @@ -331,7 +332,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { // Files not all said: everything the repository builds, as before. m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head", Paths: []string{"README.md"}, PathsTruncated: true} - if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" { + if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" { t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got) } } @@ -452,8 +453,9 @@ func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *test kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages, inventory.EdgeBuiltBy, inventory.EdgeWorkerOf} widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true} + // Built-by orders nothing since novox/hq ADR 0360 §3. orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, - inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true} + inventory.EdgeWorkerOf: true} // Directory names drawn from one pool, so two repositories hold directories of the same name, and one // is a prefix of another. dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 37d68311..453a4493 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -751,42 +751,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return advanceRest(ctx, open, p, now) } if r.Gate == nil { - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, err - } - for r.Next < len(r.Order) { - node := r.Order[r.Next] - if !heard[node] { - r.Skipped = append(r.Skipped, node) - r.Next++ - continue - } - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - note := fmt.Sprintf("waiting before %s: %v", node, err) - changed := p.Note != note - p.Note = note - return changed, nil - } - if err != nil { - return false, fmt.Errorf("reading what waits on %s: %w", node, err) - } - if len(moves) == 0 { - r.Done = append(r.Done, node) - r.Next++ - continue - } - r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true} - p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves)) - return true, nil - } - p.State, p.Tier = inventory.PlanDone, len(p.Tiers) - p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) - if len(r.Skipped) > 0 { - p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - return true, nil + // **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step. + r.Rest = true + return advanceRest(ctx, open, p, now) } g := r.Gate if g.Pending { @@ -927,24 +894,6 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } } if len(pending) > 0 { - // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on - // a machine since its step was chosen could reach a second machine in it; the step is chosen again. - if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { - return false, err - } else if grew != "" { - for _, n := range pending { - delete(r.Gates, n) - } - opened, waiting, err := openRestStep(ctx, open, p) - if err != nil { - return true, err - } - p.Note = "chosen again: " + grew - if !opened && waiting != "" { - p.Note += "; " + waiting - } - return true, nil - } var sent, already, waiting []string for _, node := range pending { out, err := sendKept(ctx, open, p, node, r.Gates[node], now) @@ -988,7 +937,6 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } changed, noteWas := false, p.Note var notes, judging []string - batched, back := batchingRollbacks(ctx) for _, node := range r.Order { g := r.Gates[node] if g == nil || g.Verdict != "" || g.Silent { @@ -1015,45 +963,23 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. judging = append(judging, node+": not yet reported on its send") continue } - verdict, err := judgeMoves(ctx, open, g, nil, now) - if err != nil { - return changed, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, g, "", nil) { - changed = true - notes = append(notes, p.Note) - } - judging = append(judging, node+": "+gateLine(g)) - case inventory.GatePassed: + // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and + // fails the walk, with nothing put back — apply-failing says it. + switch rep := byNode[node]; rep.Outcome { + case inventory.OutcomeApplied: + decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) passCarried(ctx, open, p, g, "") r.Done = append(r.Done, node) changed = true - case inventory.GateSuperseded: - // Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on - // it; what still waits is released again. - r.Done = append(r.Done, node) - changed = true - notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why)) - default: - // **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other - // machine of the step goes on to its own verdict. - stateWas := p.State - p.Note = "" - failCarried(batched, open, p, g, "") - p.State = stateWas + case inventory.OutcomeFailed, inventory.OutcomeRefused: + decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now) r.Failed = append(r.Failed, node) changed = true - notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note)) - returnedElsewhere(ctx, open, r, node, g) + notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome)) + default: + judging = append(judging, node+": not yet reported on its send") } } - p.Note = "" - sendRollbacks(ctx, open, p, back) - if p.Note != "" { - notes = append(notes, strings.TrimPrefix(p.Note, "; ")) - } if len(judging) > 0 { note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ") p.Note = note @@ -1096,39 +1022,6 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. return true, nil } -// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; -// empty when none would. -func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return "", err - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - for _, node := range pending { - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - continue // sendKept says it waits - } - if err != nil { - return "", err - } - kept := r.Gates[node].Carried - for _, mv := range moves { - // The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9). - if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { - return c.Module == mv.Module && sameCommit(c.To, mv.To) - }) { - continue - } - return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil - } - } - return "", nil -} - // reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the // declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { @@ -1141,29 +1034,6 @@ func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bo return r.Current } -// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's -// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3). -func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) { - for _, c := range g.Carried { - if c.Build == "" { - continue - } - if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked { - continue - } - for node, other := range r.Gates { - if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) { - continue - } - for _, oc := range other.Carried { - if oc.Module == c.Module && sameCommit(oc.To, c.To) { - other.Returned = append(other.Returned, c.Module) - } - } - } - } -} - // openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2): // every machine left that is heard from and has something waiting, except that a build no gate has passed goes // to one machine of the step only. Answers whether a step was kept, or why the walk waits. @@ -1173,10 +1043,6 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s if err != nil { return false, "", err } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return false, "", err - } waiting := map[string][]inventory.CarriedMove{} var nodes, walked []string for _, node := range r.Order[min(r.Next, len(r.Order)):] { @@ -1203,11 +1069,8 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s waiting[node] = moves nodes = append(nodes, node) } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return false, "", err - } - step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) + // **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first. + step := nodes if len(step) == 0 { if len(walked) > 0 { return false, "waiting before " + strings.Join(walked, "; "), nil @@ -1221,50 +1084,9 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true} } p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", ")) - if left := len(nodes) - len(step); left > 0 { - p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left) - } return true, "", nil } -// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the -// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. -func oneAtATime(module string, m catalogue.Manifest) bool { - if module == hostModule { - return true - } - return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) -} - -// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless -// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's -// first machine. Pure. -// -// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a -// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus -// shows only as its silence and no rollback reaches it. -func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, - alone func(module string) bool) []string { - first := map[string]string{} - var out []string - for _, n := range nodes { - var unpassed []string - for _, mv := range waiting[n] { - if !passed(mv.Module, mv.To) || alone(mv.Module) { - unpassed = append(unpassed, mv.Module+"@"+mv.To) - } - } - if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) { - continue - } - for _, k := range unpassed { - first[k] = n - } - out = append(out, n) - } - return out -} - // silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 // §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 8a6d27e1..661cf290 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -43,29 +43,15 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string { for _, m := range set { deps[m] = map[string]bool{} } - // The build seat's holders follow the controller that defines their worker (EdgeWorkerOf, - // novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the - // controller is built by whichever build machine is running, as the runtime image always was. - worker := map[string]map[string]bool{} - for _, e := range edges { - if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] { - if worker[e.To] == nil { - worker[e.To] = map[string]bool{} - } - worker[e.To][e.From] = true - } - } for _, e := range edges { // A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's - // build needs nothing of A's first. The other kinds order: stands-on and declared after - // the base is built, built-by after the build machine is built and running — except for - // what the build machine itself stands on, and for the controller whose worker the build - // machine binds. The runtime image is built by the builder and the builder is built on the - // runtime image; the image comes first, built by the builder that is running. + // build needs nothing of A's first. Stands-on and declared order after the base is built, and the + // build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206). if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages { continue } - if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) { + // **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it. + if e.Kind == inventory.EdgeBuiltBy { continue } deps[e.From][e.To] = true @@ -106,29 +92,6 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string { return tiers } -// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along -// the build edges alone. -func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool { - seen := map[string]bool{} - var walk func(string) bool - walk = func(m string) bool { - if m == base { - return true - } - if seen[m] { - return false - } - seen[m] = true - for _, e := range edges { - if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) { - return true - } - } - return false - } - return walk(to) -} - // reachableFrom is the moved modules plus everything that depends on them, through every layer: // what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine // is not changed by a new build machine, so a built-by edge orders and gates a plan and never @@ -164,7 +127,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string { } // hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages -// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is +// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is // no cycle, and saying one was is a false report in every such plan's log. func hasCycle(tiers [][]string, edges []inventory.Edge) bool { if len(tiers) == 0 { @@ -175,7 +138,7 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool { last[m] = true } for _, e := range edges { - if e.Kind == inventory.EdgePackages { + if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy { continue } if last[e.From] && last[e.To] { @@ -221,12 +184,6 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) return nil } inTier := map[string]bool{} - all := map[string]bool{} - for _, tier := range p.Tiers { - for _, m := range tier { - all[m] = true - } - } for _, m := range p.Tiers[p.Tier] { inTier[m] = true } @@ -241,8 +198,8 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) for _, e := range edges { // A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the // member before is running on its machines before the member after is asked. - ordered := e.Kind == edgeGroupOrder || - e.Kind == inventory.EdgeBuiltBy && !isBaseOf(e.From, e.To, edges, all) + // A built-by edge waits for nothing since novox/hq ADR 0360 §3. + ordered := e.Kind == edgeGroupOrder if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) { seen[e.To] = true out = append(out, e.To) @@ -888,6 +845,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if err != nil { return false, err } + // **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine + // running it in one step, with no first machine and no gate, whatever module it is. + policy.Together = true var reports []inventory.Reported if !policy.Together { // Read for the choice of the first machine as well as for its report. @@ -1117,15 +1077,87 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return changed, nil } } + // **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent + // has reported on that send, or is silent past ApplySilentAfter; never a health reading. + end := "" + if p.Tier == len(p.Tiers)-1 { + reports, err := inv.LastReports(ctx) + if err != nil { + return false, err + } + waiting, failed, silent := walkReports(*p, reports, time.Now().UTC()) + if len(waiting) > 0 { + note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send" + changed := p.State != inventory.PlanRolling || p.Note != note + p.State = inventory.PlanRolling + p.Note = note + return changed, nil + } + var said []string + if len(failed) > 0 { + said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", ")) + } + if len(silent) > 0 { + said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter, + strings.Join(silent, ", "))) + } + end = strings.Join(said, "; ") + } p.Tier++ p.State = inventory.PlanBuilding - p.Note = "" + p.Note = end if p.Tier < len(p.Tiers) { fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", ")) } return true, nil } +// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4): +// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter, +// each sorted. Pure. +func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time) (waiting, failed, silent []string) { + byNode := map[string]inventory.Reported{} + for _, r := range reports { + byNode[r.Node] = r + } + seen := map[string]bool{} + for _, s := range p.Modules { + if s == nil { + continue + } + for node, sent := range s.Rest { + if seen[node] { + continue + } + r := byNode[node] + at := sent.At + if at.IsZero() && s.SentAt != nil { + at = *s.SentAt + } + switch { + case r.At != nil && sent.ReportsOn(r): + seen[node] = true + if r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused { + failed = append(failed, node) + } + case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter: + seen[node] = true + silent = append(silent, node) + default: + if !slices.Contains(waiting, node) { + waiting = append(waiting, node) + } + } + } + } + // A machine one module's send was reported on and another's not yet is the same send: reported. + waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] }) + sort.Strings(waiting) + sort.Strings(failed) + sort.Strings(silent) + return waiting, failed, silent +} + // restOf is, for one module, every machine of its rest that the send reached and what it carried there. func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration { out := map[string]inventory.SentDeclaration{} diff --git a/cmd/mesh-controller/release_plan_test.go b/cmd/mesh-controller/release_plan_test.go index d26bcf57..5425de3c 100644 --- a/cmd/mesh-controller/release_plan_test.go +++ b/cmd/mesh-controller/release_plan_test.go @@ -1,6 +1,7 @@ package main import ( + "slices" "testing" "time" @@ -49,8 +50,10 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { if pos["mesh-tools"] != 0 || pos["builder"] != 1 { t.Fatalf("the runtime then the builder: %v", tiers) } - if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) { - t.Fatalf("what the builder builds comes after the builder: %v", tiers) + // Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it + // stands on. + if pos["shop"] != 1 || pos["postgres"] != 1 { + t.Fatalf("what the builder builds waits for the builder: %v", tiers) } if pos["shop-plugin"] <= pos["shop"] { t.Fatalf("a plugin after what it is declared on: %v", tiers) @@ -67,24 +70,23 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { // A change to a package all three build from moves all three. The builder holds // the build seat, whose worker the controller defines, so it follows the controller (worker-of, // novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the - // build machine that is running. The proxy is built by the new builder: the controller, the builder, - // the proxy — the live plan of every controller merge. (This read "the builder, then the controller - // and the proxy together" before issue 206, and the fixture had no worker-of edge.) + // build machine that is running. The proxy is built by whichever builder runs, so it goes beside the + // controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before). small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges) smallTiers := tiersOf(small, edges) - if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" { - t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers) + if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" { + t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers) } // The builder alone moved: the builder, and nothing it builds. if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 { t.Fatalf("a build machine change rebuilds the build machine alone: %v", only) } - // Only a runtime dependency gates on deployment, and only when the module rolls out. + // Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it. p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges) p.Tier = pos["builder"] rollsOut := func(m string) bool { return m == "builder" } - if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" { + if g := gates(p, edges, rollsOut); len(g) != 0 { t.Fatalf("the builder gates the tier after it: %v", g) } p.Tier = 0 @@ -253,10 +255,10 @@ func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) { t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers) } } - // With what it builds moved beside it, the agent comes first and they follow: an order, not a widening. + // With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing. p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"}, []string{"build-agent", "redis"}, edges) - if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 { - t.Fatalf("the agent, then what moved beside it: %v", p.Tiers) + if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 { + t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers) } } diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 8b6f7155..223d5e76 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -228,53 +228,6 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { } } -// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits -// for the next step, and one whose every build passed goes now (ADR 0353 §2). -func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, - } - passed := func(module, commit string) bool { return module == "app" && commit == "c2" } - never := func(string) bool { return false } - step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) - } - if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { - t.Fatalf("a machine whose every build passed waits: %v", step) - } -} - -// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step -// carries either to one machine only, while what else passed goes everywhere at once. -func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: hostModule, To: "h2"}}, - "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "dns", To: "d2"}}, - "ace": {{Module: "dns", To: "d2"}}, - } - resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} - alone := func(m string) bool { - if m == "dns" { - return oneAtATime(m, resolver) - } - return oneAtATime(m, catalogue.Manifest{Module: m}) - } - step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, - func(string, string) bool { return true }, alone) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) - } - if oneAtATime("app", catalogue.Manifest{Module: "app"}) { - t.Fatal("a leaf module is held to one machine at a time") - } -} - // A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { b := aBacklogOnThree(t) diff --git a/cmd/mesh-controller/worker_order_test.go b/cmd/mesh-controller/worker_order_test.go index c633ca55..86fb9073 100644 --- a/cmd/mesh-controller/worker_order_test.go +++ b/cmd/mesh-controller/worker_order_test.go @@ -38,8 +38,9 @@ func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing. if pos["build-agent"] <= pos["mesh-controller"] { t.Fatalf("the build machine after the controller that defines its worker: %v", tiers) } - if pos["route-proxy"] <= pos["build-agent"] { - t.Fatalf("what the build machine builds comes after it: %v", tiers) + // Built-by orders nothing (novox/hq ADR 0360 §3): whichever builder runs builds the proxy, beside the controller. + if pos["route-proxy"] != 0 { + t.Fatalf("what the builder builds waits for a new one: %v", tiers) } if hasCycle(tiers, edges) { t.Fatalf("no cycle here: %v", tiers) -- 2.54.0 From 0e974138751ef287d096dc73667e41864727f4f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 19:21:42 +0200 Subject: [PATCH 2/4] Assert the backlog walk sends every machine at once and passes on the report (hq ADR 0360) --- cmd/mesh-controller/release_test.go | 7 +- cmd/mesh-controller/rest_at_once_test.go | 149 ++++------------------- 2 files changed, 28 insertions(+), 128 deletions(-) diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 35b7d35c..b7a6c6c0 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -111,8 +111,13 @@ func aBacklog(t *testing.T) *backlogMesh { if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { return nil, err } + // A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict). + outcome := inventory.OutcomeApplied + if b.broken[node] { + outcome = inventory.OutcomeFailed + } if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil { return nil, err } } diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 223d5e76..39cebc05 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -59,14 +59,13 @@ func events(t *testing.T) *[]string { return &said } -// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second -// was judged to its verdict before the third was sent. -func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { +// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); +// before, the first machine was judged before the rest were sent (ADR 0353 §1). +func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() said := events(t) - gateEvery = 0 for i := 0; i < 12; i++ { advancePlans(ctx, b.open) } @@ -74,27 +73,27 @@ func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } - desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") - if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { - t.Fatalf("sends and judgings: %v", *said) + for _, n := range []string{"desk", "laptop", "anchor"} { + if !slices.Contains(*said, "send "+n) { + t.Fatalf("%s was not sent: %v", n, *said) + } } - if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { - t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) + if slices.Contains(*said, "judge") { + t.Fatalf("a machine was judged by its health: %v", *said) } if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } } -// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own -// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. -func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { +// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other +// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). +func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() inv := b.open.inventory - gateEvery = 0 - b.broken["desk"] = true // desk fails what it is sent in the rest's step + b.broken["desk"] = true // desk fails what it is sent advancePlans(ctx, b.open) for i := 0; i < 12 && b.release(t).Open(); i++ { advancePlans(ctx, b.open) @@ -103,97 +102,14 @@ func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } - if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { - t.Fatalf("laptop was never sent: %v", b.sent) + if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the others have no verdict of their own: %+v", p.Release) } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop has no verdict of its own: %+v", p.Release) - } - // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } - // app failed on desk: marked, and put back wherever it was sent (ADR 0236). - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { - t.Fatalf("desk was not put back: %v", sent) - } -} - -// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that -// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. -func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once - release := b.release(t) - judging := release.Release.Gate - if judging == nil || judging.Pending || judging.Passes == 0 { - t.Fatalf("the release is not judging anchor: %+v", judging) - } - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-app-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again what it runs: %v", b.sent) - } - p, err := inv.PlanByID(ctx, "plan-app") - if err != nil { - t.Fatal(err) - } - g := p.Modules["app"].Gate - if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { - t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) - } - if p.Modules["app"].Previous != "c1" { - t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) - } -} - -// A walk's first send that was made and not kept — the controller replaced between the two — is not made again -// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). -func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - // The old controller sends anchor app c2 and is replaced before it keeps the send. - made := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if _, err := made(ctx, open, names); err != nil { - return nil, err - } - return nil, context.Canceled - } - advancePlans(ctx, g.open) - sendRollout = made - if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { - t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) - } - last, _, err := g.open.inventory.SentTo(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if len(g.sent) != 1 { - t.Fatalf("the send was made again: %v", g.sent) - } - s := g.plan(t).Modules["app"] - if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { - t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) - } - // What anchor ran before was kept before the send, so a failure still puts it back (§5). - if s.Previous != "c1" || s.Gate.From != "c1" { - t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { + t.Fatalf("desk was put back: %v", sent) } } @@ -208,7 +124,7 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { if err != nil || p == nil { t.Fatalf("no release opened: %v", err) } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) } if err := inv.SavePlan(ctx, p); err != nil { @@ -219,35 +135,15 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { t.Fatal(err) } advancePlans(ctx, b.open) - if len(b.sent) != 1 { + if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { t.Fatalf("anchor was sent again: %v", b.sent) } - g := b.release(t).Release.Gate + g := b.release(t).Release.Gates["anchor"] if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { t.Fatalf("anchor is not judged on the send made: %+v", g) } } -// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. -func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { - b := aBacklogOnThree(t) - ctx := t.Context() - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - for i := 0; i < 6 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, s := range b.sent { - if !slices.Equal(s, []string{"anchor"}) { - t.Fatalf("sent %v: the rest after a failed first machine", b.sent) - } - } -} - // The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is // ordered after every other. func TestTheControlNodeIsSentLastInAStep(t *testing.T) { @@ -259,7 +155,6 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) { t.Fatal(err) } said := events(t) - gateEvery = 0 for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { advancePlans(ctx, b.open) } @@ -290,7 +185,6 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { return reporting(ctx, open, names) } t.Cleanup(func() { sendRollout = reporting }) - gateEvery = 0 for i := 0; i < 8; i++ { advancePlans(ctx, b.open) } @@ -306,7 +200,8 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { } advancePlans(ctx, b.open) p = b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). + if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { -- 2.54.0 From 5b455b8a243837371a8e7ac937f3b95f2a3680c0 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 19:35:07 +0200 Subject: [PATCH 3/4] Keep the first-machine gate behind a switch for walks begun before hq ADR 0360, and test both paths --- cmd/mesh-controller/check_builds_test.go | 1 + cmd/mesh-controller/found_wait_test.go | 3 + cmd/mesh-controller/gate_followup_test.go | 7 + cmd/mesh-controller/gate_test.go | 32 +++ cmd/mesh-controller/issue352_test.go | 8 + cmd/mesh-controller/issue648_test.go | 11 +- cmd/mesh-controller/lean_walk_test.go | 147 +++++++++++++ cmd/mesh-controller/person_wait_test.go | 7 + cmd/mesh-controller/planner_records_test.go | 2 +- cmd/mesh-controller/queue_test.go | 1 + cmd/mesh-controller/release.go | 231 +++++++++++++++++++- cmd/mesh-controller/release_plan.go | 11 +- cmd/mesh-controller/release_test.go | 2 + cmd/mesh-controller/replay318_test.go | 1 + cmd/mesh-controller/replay335_test.go | 2 + cmd/mesh-controller/replay336_test.go | 1 + cmd/mesh-controller/replays_test.go | 1 + cmd/mesh-controller/rest_at_once_test.go | 208 ++++++++++++++++-- cmd/mesh-controller/tier_send_test.go | 7 + 19 files changed, 641 insertions(+), 42 deletions(-) diff --git a/cmd/mesh-controller/check_builds_test.go b/cmd/mesh-controller/check_builds_test.go index 1a093834..0327b85a 100644 --- a/cmd/mesh-controller/check_builds_test.go +++ b/cmd/mesh-controller/check_builds_test.go @@ -125,6 +125,7 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) { // build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and // is named in testdata/store-tests. func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index cf8eba68..caa75ac7 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -22,6 +22,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth { } func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) now := time.Now() sent := now.Add(-time.Minute) f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, @@ -48,6 +49,7 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { // The whole walk: a module whose directory was used as found long before still gets its fix to every machine, // its pass kept and the wait said; a directory this very send found holds it and puts it back. func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) for _, c := range []struct { name string found time.Duration // when the directory was found, against now @@ -116,6 +118,7 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { // The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over. func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) k, _ := withConditionsInMemory(t) ctx := t.Context() rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index b8377d77..a78963dd 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -57,6 +57,7 @@ func healthyContainer(module string) inventory.ResourceHealth { // tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, // counted over the gate's spacing — before it says its own. The broken one alone is put back. func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() @@ -99,6 +100,7 @@ const broker_runtime = "node-tools" // **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick // while a module beside it is not yet healthy counts the healthy one once. func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() withGateBounds(t, time.Hour, time.Hour, time.Hour) @@ -123,6 +125,7 @@ func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { // **A carried build's pass carries its own wait, never another's** (review (e)). func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory @@ -147,6 +150,7 @@ func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { // carried its own module, healthy, and a build waiting on that machine that never became healthy. At the // bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it. func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory @@ -227,6 +231,7 @@ func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { // **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that // needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent. func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory @@ -289,6 +294,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { // tools are marked and put back in the very judging that found them broken — their registered build is the // one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() @@ -431,6 +437,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi // **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous // build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group. func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index d8062644..651c90ec 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -36,6 +36,10 @@ func aGateMesh(t *testing.T) *gateMesh { ctx := t.Context() inv := open.inventory g := &gateMesh{open: open, health: map[string]health{}} + // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. + wasGated := gateFirstMachine + gateFirstMachine = true + t.Cleanup(func() { gateFirstMachine = wasGated }) g.keeper, _ = withConditionsInMemory(t) g.told = &conditions.Told{} was := doctorFrom @@ -271,6 +275,34 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { } } +// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health +// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent. +func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { + g := aGateMesh(t) + gateFirstMachine = false + ctx := t.Context() + judged := false + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + judged = true + return wasGather(ctx, open, component) + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p := g.plan(t) + if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { + t.Fatalf("sent %v, not both machines in one send", g.sent) + } + if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { + t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) + } + if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { + t.Fatalf("a rolled build left a condition: %+v %v", obs, err) + } +} + // A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the // core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go index 0e48b13b..e3a6f7e1 100644 --- a/cmd/mesh-controller/issue352_test.go +++ b/cmd/mesh-controller/issue352_test.go @@ -25,6 +25,7 @@ import ( // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module // to another build supersedes the judging: no verdict, nothing put back. func TestReplay352(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) @@ -67,6 +68,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { // A plan's own first send waits while a release judges the same module on that machine with another build. func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -82,6 +84,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { // A merge plan's judging is superseded the same way: another send moved its module on the first machine. func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() @@ -140,6 +143,7 @@ func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { // A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept // without its send reads the report against the send made last, as before. Pure. func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490} for _, c := range []struct { r inventory.Reported @@ -185,6 +189,7 @@ func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) { // A gate judges only the build the machine was last sent: last sent another build of the module, the // judging is superseded, whatever the machine reports. Pure. func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) at := time.Now() f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, @@ -212,6 +217,7 @@ func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) { // A move of another build of a module to a machine where a release or a plan is judging that module // waits for that judging; the same build to that machine is already there. Pure. func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) at := time.Now() release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{ Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{ @@ -249,6 +255,7 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { // The controller is never put back to a build that reaches less of the store's schema than the store // has, or to one that never said what it reaches: the current build is kept, and the condition says so. func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory @@ -345,6 +352,7 @@ func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) // The lease's holder names the build the declaration told it it is, and the version stamp only without one. func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) t.Setenv(RunningBuildVar, " ad62528c47c7 ") if h := holderOf("x"); h.Build != "ad62528c47c7" { t.Fatalf("the holder's build is %q", h.Build) diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go index 45261cc0..0557e774 100644 --- a/cmd/mesh-controller/issue648_test.go +++ b/cmd/mesh-controller/issue648_test.go @@ -20,8 +20,8 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { ctx := t.Context() inv := g.open.inventory - // app is the build machine, so a module of the next tier is built by it: the tier gates the next on app - // running on its machines. + // app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates + // nothing. for _, m := range []catalogue.Manifest{ {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, {Module: "web", Version: "w1"}, @@ -71,7 +71,8 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) } - // A machine that has not answered its last send is still waited on: no move is not "never sent". + // A machine that has not answered its last send is not waited on either: a built-by edge gates no tier + // (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop. p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { t.Fatal(err) @@ -81,7 +82,7 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { } advancePlans(ctx, g.open) p = g.plan(t) - if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") { - t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note) + if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { + t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note) } } diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go index 560ba49d..48ed5077 100644 --- a/cmd/mesh-controller/lean_walk_test.go +++ b/cmd/mesh-controller/lean_walk_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "slices" "testing" "time" @@ -39,3 +40,149 @@ func TestAWalkEndsOnItsMachinesReports(t *testing.T) { t.Fatalf("the walk still waits once every machine reported: %v", waiting) } } + +// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before +// novox/hq ADR 0360 started. +func withTheGate(t *testing.T) { + t.Helper() + was := gateFirstMachine + gateFirstMachine = true + t.Cleanup(func() { gateFirstMachine = was }) +} + +// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); +// before, the first machine was judged before the rest were sent (ADR 0353 §1). +func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, n := range []string{"desk", "laptop", "anchor"} { + if !slices.Contains(*said, "send "+n) { + t.Fatalf("%s was not sent: %v", n, *said) + } + } + if slices.Contains(*said, "judge") { + t.Fatalf("a machine was judged by its health: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other +// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). +func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + b.broken["desk"] = true // desk fails what it is sent + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the others have no verdict of their own: %+v", p.Release) + } + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { + t.Fatalf("desk was put back: %v", sent) + } +} + +// A release's send kept before it was made, and made, is judged by the controller that takes over without being +// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7). +func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gates["anchor"] + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put +// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7). +func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). + if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index e52812b4..d5337899 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -30,6 +30,7 @@ func userUnit(module, account string) inventory.ResourceHealth { // Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as // before, so that a fault is never excused. func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) notInGroup := relogin("lights", "operator") notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why" systemUnit := userUnit("lights", "operator") @@ -74,6 +75,7 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { // The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module // with a container down beside it is not yet healthy, as before. func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) now := time.Now() since := now.Add(-time.Minute) account := relogin("lights", "operator") @@ -102,6 +104,7 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { // The module's condition says the wait in one sentence for a person, and clears on the first statement that // no longer says it — said as the module's own fault when its unit still fails after the new login. func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() k, _ := withConditionsInMemory(t) @@ -158,6 +161,7 @@ func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { // A module that is healthy on its own keeps its pass when another module of the same send fails at the // bound: it is neither put back nor left without a verdict for every other walk to wait on. func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory @@ -207,6 +211,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers // no button, since nothing but their own new login can do it. func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ relogin("openrazer", "operator"), userUnit("openrazer", "operator")}) plainExample(t, o, "openrazer waits for a new login on g14", @@ -238,6 +243,7 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { // When a module not working turns into one waiting for a new login, and back, the clearing line says what it // became, never that it works again (issue 318 review). func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() store := conditions.NewInMemory() @@ -286,6 +292,7 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { // Which moves add an account group, read from the builds' manifests (issue 318 review). func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) user := func(groups ...any) catalogue.Manifest { return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", "name": "operator", "groups": groups}}} diff --git a/cmd/mesh-controller/planner_records_test.go b/cmd/mesh-controller/planner_records_test.go index 6d3e261c..9f56ef4c 100644 --- a/cmd/mesh-controller/planner_records_test.go +++ b/cmd/mesh-controller/planner_records_test.go @@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) { {"the route proxy's program: the route proxy alone", "mesh-controller", []string{"examples/route-proxy/main.go"}, "route-proxy", true}, {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false}, + []string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false}, {"the route proxy's directory in the catalogue: it alone", "mesh-catalog", []string{"modules/route-proxy/module.json"}, "route-proxy", false}, {"the build agent's directory: it alone, nothing it builds", "mesh-catalog", diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index 16096151..dfd656d3 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -616,6 +616,7 @@ func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) { // A plan that stopped at its first machine is retried: the module sent to that machine again, the // send recorded as the first anew, and the plan goes on — unless a newer plan holds the module. func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() asked := asksRecorded(t) diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 453a4493..4937b5fe 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -750,11 +750,49 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, if r.Rest { return advanceRest(ctx, open, p, now) } - if r.Gate == nil { + if r.Gate == nil && !gateFirstMachine { // **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step. r.Rest = true return advanceRest(ctx, open, p, now) } + if r.Gate == nil { + heard, err := releaseHeard(ctx, open) + if err != nil { + return false, err + } + for r.Next < len(r.Order) { + node := r.Order[r.Next] + if !heard[node] { + r.Skipped = append(r.Skipped, node) + r.Next++ + continue + } + moves, err := gatedMoves(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + note := fmt.Sprintf("waiting before %s: %v", node, err) + changed := p.Note != note + p.Note = note + return changed, nil + } + if err != nil { + return false, fmt.Errorf("reading what waits on %s: %w", node, err) + } + if len(moves) == 0 { + r.Done = append(r.Done, node) + r.Next++ + continue + } + r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true} + p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves)) + return true, nil + } + p.State, p.Tier = inventory.PlanDone, len(p.Tiers) + p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) + if len(r.Skipped) > 0 { + p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") + } + return true, nil + } g := r.Gate if g.Pending { node := firstOf(g.Machines) @@ -894,6 +932,24 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } } if len(pending) > 0 { + // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on + // a machine since its step was chosen could reach a second machine in it; the step is chosen again. + if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { + return false, err + } else if grew != "" { + for _, n := range pending { + delete(r.Gates, n) + } + opened, waiting, err := openRestStep(ctx, open, p) + if err != nil { + return true, err + } + p.Note = "chosen again: " + grew + if !opened && waiting != "" { + p.Note += "; " + waiting + } + return true, nil + } var sent, already, waiting []string for _, node := range pending { out, err := sendKept(ctx, open, p, node, r.Gates[node], now) @@ -937,6 +993,7 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } changed, noteWas := false, p.Note var notes, judging []string + batched, back := batchingRollbacks(ctx) for _, node := range r.Order { g := r.Gates[node] if g == nil || g.Verdict != "" || g.Silent { @@ -963,23 +1020,64 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. judging = append(judging, node+": not yet reported on its send") continue } - // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and - // fails the walk, with nothing put back — apply-failing says it. - switch rep := byNode[node]; rep.Outcome { - case inventory.OutcomeApplied: - decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) + if !gateFirstMachine { + // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and + // fails the walk, with nothing put back — apply-failing says it. + switch rep := byNode[node]; rep.Outcome { + case inventory.OutcomeApplied: + decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) + passCarried(ctx, open, p, g, "") + r.Done = append(r.Done, node) + changed = true + case inventory.OutcomeFailed, inventory.OutcomeRefused: + decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now) + r.Failed = append(r.Failed, node) + changed = true + notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome)) + default: + judging = append(judging, node+": not yet reported on its send") + } + continue + } + verdict, err := judgeMoves(ctx, open, g, nil, now) + if err != nil { + return changed, err + } + switch verdict { + case "": + if putBackBroken(ctx, open, p, g, "", nil) { + changed = true + notes = append(notes, p.Note) + } + judging = append(judging, node+": "+gateLine(g)) + case inventory.GatePassed: passCarried(ctx, open, p, g, "") r.Done = append(r.Done, node) changed = true - case inventory.OutcomeFailed, inventory.OutcomeRefused: - decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now) + case inventory.GateSuperseded: + // Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on + // it; what still waits is released again. + r.Done = append(r.Done, node) + changed = true + notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why)) + default: + // **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other + // machine of the step goes on to its own verdict. + stateWas := p.State + p.Note = "" + failCarried(batched, open, p, g, "") + p.State = stateWas r.Failed = append(r.Failed, node) changed = true - notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome)) - default: - judging = append(judging, node+": not yet reported on its send") + notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note)) + returnedElsewhere(ctx, open, r, node, g) } } + p.Note = "" + sendRollbacks(ctx, open, p, back) + if p.Note != "" { + notes = append(notes, strings.TrimPrefix(p.Note, "; ")) + } if len(judging) > 0 { note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ") p.Note = note @@ -1022,6 +1120,42 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. return true, nil } +// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; +// empty when none would. +func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { + if !gateFirstMachine { + return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made + } + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return "", err + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return "", err + } + for _, node := range pending { + moves, err := gatedMoves(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + continue // sendKept says it waits + } + if err != nil { + return "", err + } + kept := r.Gates[node].Carried + for _, mv := range moves { + // The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9). + if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { + return c.Module == mv.Module && sameCommit(c.To, mv.To) + }) { + continue + } + return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil + } + } + return "", nil +} + // reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the // declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { @@ -1034,6 +1168,29 @@ func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bo return r.Current } +// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's +// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3). +func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) { + for _, c := range g.Carried { + if c.Build == "" { + continue + } + if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked { + continue + } + for node, other := range r.Gates { + if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) { + continue + } + for _, oc := range other.Carried { + if oc.Module == c.Module && sameCommit(oc.To, c.To) { + other.Returned = append(other.Returned, c.Module) + } + } + } + } +} + // openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2): // every machine left that is heard from and has something waiting, except that a build no gate has passed goes // to one machine of the step only. Answers whether a step was kept, or why the walk waits. @@ -1043,6 +1200,10 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s if err != nil { return false, "", err } + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return false, "", err + } waiting := map[string][]inventory.CarriedMove{} var nodes, walked []string for _, node := range r.Order[min(r.Next, len(r.Order)):] { @@ -1069,8 +1230,15 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s waiting[node] = moves nodes = append(nodes, node) } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return false, "", err + } // **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first. step := nodes + if gateFirstMachine { + step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) + } if len(step) == 0 { if len(walked) > 0 { return false, "waiting before " + strings.Join(walked, "; "), nil @@ -1084,9 +1252,50 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true} } p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", ")) + if left := len(nodes) - len(step); left > 0 { + p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left) + } return true, "", nil } +// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the +// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. +func oneAtATime(module string, m catalogue.Manifest) bool { + if module == hostModule { + return true + } + return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) +} + +// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless +// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's +// first machine. Pure. +// +// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a +// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus +// shows only as its silence and no rollback reaches it. +func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, + alone func(module string) bool) []string { + first := map[string]string{} + var out []string + for _, n := range nodes { + var unpassed []string + for _, mv := range waiting[n] { + if !passed(mv.Module, mv.To) || alone(mv.Module) { + unpassed = append(unpassed, mv.Module+"@"+mv.To) + } + } + if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) { + continue + } + for _, k := range unpassed { + first[k] = n + } + out = append(out, n) + } + return out +} + // silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 // §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 661cf290..8378c1f7 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -27,6 +27,10 @@ import ( // alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the // plan where it left it. +// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since +// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on. +var gateFirstMachine = false + // planWaitBound is how long a plan may wait on one thing before `status` names it red. const planWaitBound = 30 * time.Minute @@ -846,8 +850,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return false, err } // **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine - // running it in one step, with no first machine and no gate, whatever module it is. - policy.Together = true + // running it in one step, with no first machine and no gate, whatever module it is. A module a controller + // before ADR 0360 already sent to its first machine finishes under its gate. + if !gateFirstMachine && state.FirstAt == nil { + policy.Together = true + } var reports []inventory.Reported if !policy.Together { // Read for the choice of the first machine as well as for its report. diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index b7a6c6c0..a2fd2589 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -169,6 +169,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan { // The backlog goes out one machine at a time: the first judged before the second is sent, each build's // pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -224,6 +225,7 @@ func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { // A release that fails on its first machine puts back what it carried there, goes no further, and the // next one waits for a person, said as a condition; a person releases it with why. func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 53fb53ca..df8a33b6 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -31,6 +31,7 @@ import ( // nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence // for the operator, that a new login is needed on the laptop. func TestReplay318(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go index 2ec3e0f3..27031906 100644 --- a/cmd/mesh-controller/replay335_test.go +++ b/cmd/mesh-controller/replay335_test.go @@ -16,6 +16,7 @@ import ( // at its gate with the pass's own words, and put it back. Written with only what the controller had before // its fix, so it is laid over the commit before. func TestReplay335(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports) t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass) } @@ -77,6 +78,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the // carried module's step said the first machine never applied it, and the passed build was put back. func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) tm := aTierMesh(t, "m01", "m02") ctx := t.Context() diff --git a/cmd/mesh-controller/replay336_test.go b/cmd/mesh-controller/replay336_test.go index 1b99cf71..20fa6dec 100644 --- a/cmd/mesh-controller/replay336_test.go +++ b/cmd/mesh-controller/replay336_test.go @@ -19,6 +19,7 @@ import ( // had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and // seeing — so it is laid over the commit before. func TestReplay336(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 2798542d..f8a95346 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -139,6 +139,7 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { // one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is // written only with what the controller had before the judging, for the prover to lay over that commit. func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 39cebc05..39a0cb55 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -11,7 +11,7 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by +// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by // its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them — // a machine already sent what a send is for is not sent again, and its judging goes on from what was read. // @@ -59,13 +59,15 @@ func events(t *testing.T) *[]string { return &said } -// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); -// before, the first machine was judged before the rest were sent (ADR 0353 §1). -func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { +// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second +// was judged to its verdict before the third was sent. +func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() said := events(t) + gateEvery = 0 for i := 0; i < 12; i++ { advancePlans(ctx, b.open) } @@ -73,27 +75,28 @@ func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } - for _, n := range []string{"desk", "laptop", "anchor"} { - if !slices.Contains(*said, "send "+n) { - t.Fatalf("%s was not sent: %v", n, *said) - } + desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") + if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { + t.Fatalf("sends and judgings: %v", *said) } - if slices.Contains(*said, "judge") { - t.Fatalf("a machine was judged by its health: %v", *said) + if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { + t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) } if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } } -// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other -// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). -func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { +// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own +// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. +func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() inv := b.open.inventory - b.broken["desk"] = true // desk fails what it is sent + gateEvery = 0 + b.broken["desk"] = true // desk fails what it is sent in the rest's step advancePlans(ctx, b.open) for i := 0; i < 12 && b.release(t).Open(); i++ { advancePlans(ctx, b.open) @@ -102,20 +105,106 @@ func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } - if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the others have no verdict of their own: %+v", p.Release) + if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { + t.Fatalf("laptop was never sent: %v", b.sent) } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop has no verdict of its own: %+v", p.Release) + } + // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { - t.Fatalf("desk was put back: %v", sent) + // app failed on desk: marked, and put back wherever it was sent (ADR 0236). + if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { + t.Fatal("app's build is not marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { + t.Fatalf("desk was not put back: %v", sent) + } +} + +// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that +// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. +func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once + release := b.release(t) + judging := release.Release.Gate + if judging == nil || judging.Pending || judging.Passes == 0 { + t.Fatalf("the release is not judging anchor: %+v", judging) + } + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-app-c2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again what it runs: %v", b.sent) + } + p, err := inv.PlanByID(ctx, "plan-app") + if err != nil { + t.Fatal(err) + } + g := p.Modules["app"].Gate + if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { + t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) + } + if p.Modules["app"].Previous != "c1" { + t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) + } +} + +// A walk's first send that was made and not kept — the controller replaced between the two — is not made again +// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). +func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + // The old controller sends anchor app c2 and is replaced before it keeps the send. + made := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if _, err := made(ctx, open, names); err != nil { + return nil, err + } + return nil, context.Canceled + } + advancePlans(ctx, g.open) + sendRollout = made + if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { + t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) + } + last, _, err := g.open.inventory.SentTo(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if len(g.sent) != 1 { + t.Fatalf("the send was made again: %v", g.sent) + } + s := g.plan(t).Modules["app"] + if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { + t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) + } + // What anchor ran before was kept before the send, so a failure still puts it back (§5). + if s.Previous != "c1" || s.Gate.From != "c1" { + t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) } } // A release's send kept before it was made, and made, is judged by the controller that takes over without being // made again (ADR 0353 §8). func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -124,7 +213,7 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { if err != nil || p == nil { t.Fatalf("no release opened: %v", err) } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) } if err := inv.SavePlan(ctx, p); err != nil { @@ -135,18 +224,89 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { t.Fatal(err) } advancePlans(ctx, b.open) - if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { + if len(b.sent) != 1 { t.Fatalf("anchor was sent again: %v", b.sent) } - g := b.release(t).Release.Gates["anchor"] + g := b.release(t).Release.Gate if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { t.Fatalf("anchor is not judged on the send made: %+v", g) } } +// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits +// for the next step, and one whose every build passed goes now (ADR 0353 §2). +func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { + withTheGate(t) + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, + } + passed := func(module, commit string) bool { return module == "app" && commit == "c2" } + never := func(string) bool { return false } + step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) + } + if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { + t.Fatalf("a machine whose every build passed waits: %v", step) + } +} + +// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step +// carries either to one machine only, while what else passed goes everywhere at once. +func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { + withTheGate(t) + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: hostModule, To: "h2"}}, + "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "dns", To: "d2"}}, + "ace": {{Module: "dns", To: "d2"}}, + } + resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} + alone := func(m string) bool { + if m == "dns" { + return oneAtATime(m, resolver) + } + return oneAtATime(m, catalogue.Manifest{Module: m}) + } + step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, + func(string, string) bool { return true }, alone) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) + } + if oneAtATime("app", catalogue.Manifest{Module: "app"}) { + t.Fatal("a leaf module is held to one machine at a time") + } +} + +// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. +func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { + withTheGate(t) + b := aBacklogOnThree(t) + ctx := t.Context() + gateEvery = 0 + b.broken["anchor"] = true + advancePlans(ctx, b.open) + for i := 0; i < 6 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, s := range b.sent { + if !slices.Equal(s, []string{"anchor"}) { + t.Fatalf("sent %v: the rest after a failed first machine", b.sent) + } + } +} + // The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is // ordered after every other. func TestTheControlNodeIsSentLastInAStep(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() @@ -155,6 +315,7 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) { t.Fatal(err) } said := events(t) + gateEvery = 0 for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { advancePlans(ctx, b.open) } @@ -170,6 +331,7 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) { // A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put // back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -185,6 +347,7 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { return reporting(ctx, open, names) } t.Cleanup(func() { sendRollout = reporting }) + gateEvery = 0 for i := 0; i < 8; i++ { advancePlans(ctx, b.open) } @@ -200,8 +363,7 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { } advancePlans(ctx, b.open) p = b.release(t) - // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). - if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) } if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index 97241324..590900ad 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -171,6 +171,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) { // The incident's tier, small: a dozen modules whose first machine is the same are sent there in one // send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) var modules []string for i := 1; i <= 12; i++ { @@ -223,6 +224,7 @@ func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { // settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the // modules sent with them: at the bound only the node tools are put back. func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) ctx := t.Context() inv := tm.open.inventory @@ -276,6 +278,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { // A machine-level condition about anything else holds the machine back as a whole: everything the send // moved there fails together, once, with that reason — not one module for it. func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) tm := aTierMesh(t, "app1", "app2", "app3") ctx := t.Context() @@ -307,6 +310,7 @@ func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { // The machine-level conditions, sorted: one naming a module is that module's; the core behind is the // core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole. func TestTheConditionsAboutAMachineItself(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) since := time.Now().Add(-time.Minute) about := func(kind, id string) conditions.Condition { return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(), @@ -339,6 +343,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) { // rollback that failed; and `plans retry` asks it again rather than refusing a build that was never // judged (the incident's "NOT put back: no build kept"). func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() @@ -394,6 +399,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { // them, they are registered back and marked in the judging that finds them broken, the plan goes on judging // the other module to its own pass, and only then fails. func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { t.Run("gate kept on "+order[0], func(t *testing.T) { @@ -454,6 +460,7 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { // decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool // runner's rule: one reading, not before the node-engine's witness window for it has ended. func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) tm := aTierMesh(t, broker.RuntimeModule, "app1") ctx := t.Context() // One reading counted per advance, as every test with a gate left open does: a gate judged with no time -- 2.54.0 From 704b21b3c547b65cf4a1ac3e0909ab90c79d171c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 19:36:28 +0200 Subject: [PATCH 4/4] =?UTF-8?q?Carry=20the=20new=20controller's=20user=20l?= =?UTF-8?q?ist=20to=20the=20bus=20when=20its=20walk=20ends,=20as=20the=20g?= =?UTF-8?q?ate's=20pass=20did=20(hq=20ADR=200236=20=C2=A734,=20ADR=200360)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/release_plan.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 8378c1f7..63430ce0 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -15,6 +15,7 @@ import ( "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/builder" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -1109,6 +1110,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, strings.Join(silent, ", "))) } end = strings.Join(said, "; ") + // **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360): + // once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind. + if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) { + carryUserList(ctx, open, p) + } } p.Tier++ p.State = inventory.PlanBuilding -- 2.54.0