From da8a5e9bc0f769e89f3fcac950a1e3ac757d81b7 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 19:19:49 +0200 Subject: [PATCH] Send a merge's builds to every machine at once, with no first-machine gate (hq ADR 0360) The operator chose the leanest rollout: no gate, no one machine first, failures loud and fixed forward. A walk sends every rolled-out module to every machine running it in one step, built-by orders no tier, and a walk ends on its machines' reports. The backlog walk sends every machine at once and passes each on its applied report. --- cmd/mesh-controller/lean_walk_test.go | 41 +++++ cmd/mesh-controller/planner_rules_test.go | 20 ++- cmd/mesh-controller/release.go | 208 ++-------------------- cmd/mesh-controller/release_plan.go | 136 ++++++++------ cmd/mesh-controller/release_plan_test.go | 26 +-- cmd/mesh-controller/rest_at_once_test.go | 47 ----- cmd/mesh-controller/worker_order_test.go | 5 +- 7 files changed, 168 insertions(+), 315 deletions(-) create mode 100644 cmd/mesh-controller/lean_walk_test.go diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go new file mode 100644 index 00000000..560ba49d --- /dev/null +++ b/cmd/mesh-controller/lean_walk_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "slices" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send, +// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading. +func TestAWalkEndsOnItsMachinesReports(t *testing.T) { + now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + sent := now.Add(-time.Minute) + long := now.Add(-inventory.ApplySilentAfter - time.Minute) + at := now.Add(-10 * time.Second) + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ + "app": {Rest: map[string]inventory.SentDeclaration{ + "ace": {Digest: "d-ace", At: sent}, + "novox": {Digest: "d-novox", At: sent}, + "shanks": {Digest: "d-shanks", At: sent}, + "laptop": {Digest: "d-laptop", At: long}, + }}, + "proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}}, + }} + reports := []inventory.Reported{ + {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, + {Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"}, + {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"}, + } + waiting, failed, silent := walkReports(p, reports, now) + if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) || + !slices.Equal(silent, []string{"laptop"}) { + t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent) + } + reports[2].Declared = "d-shanks" + if waiting, _, _ := walkReports(p, reports, now); len(waiting) != 0 { + t.Fatalf("the walk still waits once every machine reported: %v", waiting) + } +} diff --git a/cmd/mesh-controller/planner_rules_test.go b/cmd/mesh-controller/planner_rules_test.go index 69ed505f..9e422c3a 100644 --- a/cmd/mesh-controller/planner_rules_test.go +++ b/cmd/mesh-controller/planner_rules_test.go @@ -145,8 +145,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) { repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, {what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")}, repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - {what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, + // Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered. + {what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")}, + repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, {what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")}, repo: "one", paths: []string{"modules/a/x"}, want: "a"}, {what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")}, @@ -296,18 +297,18 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { []string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read, "mesh-controller | build-agent", ""}, // A package all three build from: all three; the build seat's holder after the controller whose worker - // it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by). + // it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3). {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"a vendored package all three build from: all three", "mesh-controller", - []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"go.sum: all three", "mesh-controller", - []string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"a file added to the proxy's package: the proxy", "mesh-controller", []string{"examples/route-proxy/new.go"}, read, "route-proxy", ""}, // Before any build source is said: as before. {"no build source said: a README moves all three, as before", "mesh-controller", - []string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""}, + []string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""}, // In the catalogue, where they live, each by its own build source. {"the route proxy's recipe: it alone", "mesh-catalog", []string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""}, @@ -331,7 +332,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { // Files not all said: everything the repository builds, as before. m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head", Paths: []string{"README.md"}, PathsTruncated: true} - if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" { + if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" { t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got) } } @@ -452,8 +453,9 @@ func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *test kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages, inventory.EdgeBuiltBy, inventory.EdgeWorkerOf} widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true} + // Built-by orders nothing since novox/hq ADR 0360 §3. orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, - inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true} + inventory.EdgeWorkerOf: true} // Directory names drawn from one pool, so two repositories hold directories of the same name, and one // is a prefix of another. dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 37d68311..453a4493 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -751,42 +751,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return advanceRest(ctx, open, p, now) } if r.Gate == nil { - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, err - } - for r.Next < len(r.Order) { - node := r.Order[r.Next] - if !heard[node] { - r.Skipped = append(r.Skipped, node) - r.Next++ - continue - } - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - note := fmt.Sprintf("waiting before %s: %v", node, err) - changed := p.Note != note - p.Note = note - return changed, nil - } - if err != nil { - return false, fmt.Errorf("reading what waits on %s: %w", node, err) - } - if len(moves) == 0 { - r.Done = append(r.Done, node) - r.Next++ - continue - } - r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true} - p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves)) - return true, nil - } - p.State, p.Tier = inventory.PlanDone, len(p.Tiers) - p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) - if len(r.Skipped) > 0 { - p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - return true, nil + // **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step. + r.Rest = true + return advanceRest(ctx, open, p, now) } g := r.Gate if g.Pending { @@ -927,24 +894,6 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } } if len(pending) > 0 { - // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on - // a machine since its step was chosen could reach a second machine in it; the step is chosen again. - if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { - return false, err - } else if grew != "" { - for _, n := range pending { - delete(r.Gates, n) - } - opened, waiting, err := openRestStep(ctx, open, p) - if err != nil { - return true, err - } - p.Note = "chosen again: " + grew - if !opened && waiting != "" { - p.Note += "; " + waiting - } - return true, nil - } var sent, already, waiting []string for _, node := range pending { out, err := sendKept(ctx, open, p, node, r.Gates[node], now) @@ -988,7 +937,6 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } changed, noteWas := false, p.Note var notes, judging []string - batched, back := batchingRollbacks(ctx) for _, node := range r.Order { g := r.Gates[node] if g == nil || g.Verdict != "" || g.Silent { @@ -1015,45 +963,23 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. judging = append(judging, node+": not yet reported on its send") continue } - verdict, err := judgeMoves(ctx, open, g, nil, now) - if err != nil { - return changed, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, g, "", nil) { - changed = true - notes = append(notes, p.Note) - } - judging = append(judging, node+": "+gateLine(g)) - case inventory.GatePassed: + // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and + // fails the walk, with nothing put back — apply-failing says it. + switch rep := byNode[node]; rep.Outcome { + case inventory.OutcomeApplied: + decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) passCarried(ctx, open, p, g, "") r.Done = append(r.Done, node) changed = true - case inventory.GateSuperseded: - // Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on - // it; what still waits is released again. - r.Done = append(r.Done, node) - changed = true - notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why)) - default: - // **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other - // machine of the step goes on to its own verdict. - stateWas := p.State - p.Note = "" - failCarried(batched, open, p, g, "") - p.State = stateWas + case inventory.OutcomeFailed, inventory.OutcomeRefused: + decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now) r.Failed = append(r.Failed, node) changed = true - notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note)) - returnedElsewhere(ctx, open, r, node, g) + notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome)) + default: + judging = append(judging, node+": not yet reported on its send") } } - p.Note = "" - sendRollbacks(ctx, open, p, back) - if p.Note != "" { - notes = append(notes, strings.TrimPrefix(p.Note, "; ")) - } if len(judging) > 0 { note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ") p.Note = note @@ -1096,39 +1022,6 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. return true, nil } -// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; -// empty when none would. -func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return "", err - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - for _, node := range pending { - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - continue // sendKept says it waits - } - if err != nil { - return "", err - } - kept := r.Gates[node].Carried - for _, mv := range moves { - // The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9). - if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { - return c.Module == mv.Module && sameCommit(c.To, mv.To) - }) { - continue - } - return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil - } - } - return "", nil -} - // reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the // declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { @@ -1141,29 +1034,6 @@ func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bo return r.Current } -// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's -// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3). -func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) { - for _, c := range g.Carried { - if c.Build == "" { - continue - } - if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked { - continue - } - for node, other := range r.Gates { - if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) { - continue - } - for _, oc := range other.Carried { - if oc.Module == c.Module && sameCommit(oc.To, c.To) { - other.Returned = append(other.Returned, c.Module) - } - } - } - } -} - // openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2): // every machine left that is heard from and has something waiting, except that a build no gate has passed goes // to one machine of the step only. Answers whether a step was kept, or why the walk waits. @@ -1173,10 +1043,6 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s if err != nil { return false, "", err } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return false, "", err - } waiting := map[string][]inventory.CarriedMove{} var nodes, walked []string for _, node := range r.Order[min(r.Next, len(r.Order)):] { @@ -1203,11 +1069,8 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s waiting[node] = moves nodes = append(nodes, node) } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return false, "", err - } - step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) + // **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first. + step := nodes if len(step) == 0 { if len(walked) > 0 { return false, "waiting before " + strings.Join(walked, "; "), nil @@ -1221,50 +1084,9 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true} } p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", ")) - if left := len(nodes) - len(step); left > 0 { - p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left) - } return true, "", nil } -// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the -// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. -func oneAtATime(module string, m catalogue.Manifest) bool { - if module == hostModule { - return true - } - return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) -} - -// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless -// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's -// first machine. Pure. -// -// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a -// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus -// shows only as its silence and no rollback reaches it. -func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, - alone func(module string) bool) []string { - first := map[string]string{} - var out []string - for _, n := range nodes { - var unpassed []string - for _, mv := range waiting[n] { - if !passed(mv.Module, mv.To) || alone(mv.Module) { - unpassed = append(unpassed, mv.Module+"@"+mv.To) - } - } - if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) { - continue - } - for _, k := range unpassed { - first[k] = n - } - out = append(out, n) - } - return out -} - // silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 // §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 8a6d27e1..661cf290 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -43,29 +43,15 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string { for _, m := range set { deps[m] = map[string]bool{} } - // The build seat's holders follow the controller that defines their worker (EdgeWorkerOf, - // novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the - // controller is built by whichever build machine is running, as the runtime image always was. - worker := map[string]map[string]bool{} - for _, e := range edges { - if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] { - if worker[e.To] == nil { - worker[e.To] = map[string]bool{} - } - worker[e.To][e.From] = true - } - } for _, e := range edges { // A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's - // build needs nothing of A's first. The other kinds order: stands-on and declared after - // the base is built, built-by after the build machine is built and running — except for - // what the build machine itself stands on, and for the controller whose worker the build - // machine binds. The runtime image is built by the builder and the builder is built on the - // runtime image; the image comes first, built by the builder that is running. + // build needs nothing of A's first. Stands-on and declared order after the base is built, and the + // build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206). if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages { continue } - if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) { + // **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it. + if e.Kind == inventory.EdgeBuiltBy { continue } deps[e.From][e.To] = true @@ -106,29 +92,6 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string { return tiers } -// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along -// the build edges alone. -func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool { - seen := map[string]bool{} - var walk func(string) bool - walk = func(m string) bool { - if m == base { - return true - } - if seen[m] { - return false - } - seen[m] = true - for _, e := range edges { - if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) { - return true - } - } - return false - } - return walk(to) -} - // reachableFrom is the moved modules plus everything that depends on them, through every layer: // what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine // is not changed by a new build machine, so a built-by edge orders and gates a plan and never @@ -164,7 +127,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string { } // hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages -// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is +// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is // no cycle, and saying one was is a false report in every such plan's log. func hasCycle(tiers [][]string, edges []inventory.Edge) bool { if len(tiers) == 0 { @@ -175,7 +138,7 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool { last[m] = true } for _, e := range edges { - if e.Kind == inventory.EdgePackages { + if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy { continue } if last[e.From] && last[e.To] { @@ -221,12 +184,6 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) return nil } inTier := map[string]bool{} - all := map[string]bool{} - for _, tier := range p.Tiers { - for _, m := range tier { - all[m] = true - } - } for _, m := range p.Tiers[p.Tier] { inTier[m] = true } @@ -241,8 +198,8 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) for _, e := range edges { // A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the // member before is running on its machines before the member after is asked. - ordered := e.Kind == edgeGroupOrder || - e.Kind == inventory.EdgeBuiltBy && !isBaseOf(e.From, e.To, edges, all) + // A built-by edge waits for nothing since novox/hq ADR 0360 §3. + ordered := e.Kind == edgeGroupOrder if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) { seen[e.To] = true out = append(out, e.To) @@ -888,6 +845,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if err != nil { return false, err } + // **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine + // running it in one step, with no first machine and no gate, whatever module it is. + policy.Together = true var reports []inventory.Reported if !policy.Together { // Read for the choice of the first machine as well as for its report. @@ -1117,15 +1077,87 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return changed, nil } } + // **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent + // has reported on that send, or is silent past ApplySilentAfter; never a health reading. + end := "" + if p.Tier == len(p.Tiers)-1 { + reports, err := inv.LastReports(ctx) + if err != nil { + return false, err + } + waiting, failed, silent := walkReports(*p, reports, time.Now().UTC()) + if len(waiting) > 0 { + note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send" + changed := p.State != inventory.PlanRolling || p.Note != note + p.State = inventory.PlanRolling + p.Note = note + return changed, nil + } + var said []string + if len(failed) > 0 { + said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", ")) + } + if len(silent) > 0 { + said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter, + strings.Join(silent, ", "))) + } + end = strings.Join(said, "; ") + } p.Tier++ p.State = inventory.PlanBuilding - p.Note = "" + p.Note = end if p.Tier < len(p.Tiers) { fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", ")) } return true, nil } +// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4): +// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter, +// each sorted. Pure. +func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time) (waiting, failed, silent []string) { + byNode := map[string]inventory.Reported{} + for _, r := range reports { + byNode[r.Node] = r + } + seen := map[string]bool{} + for _, s := range p.Modules { + if s == nil { + continue + } + for node, sent := range s.Rest { + if seen[node] { + continue + } + r := byNode[node] + at := sent.At + if at.IsZero() && s.SentAt != nil { + at = *s.SentAt + } + switch { + case r.At != nil && sent.ReportsOn(r): + seen[node] = true + if r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused { + failed = append(failed, node) + } + case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter: + seen[node] = true + silent = append(silent, node) + default: + if !slices.Contains(waiting, node) { + waiting = append(waiting, node) + } + } + } + } + // A machine one module's send was reported on and another's not yet is the same send: reported. + waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] }) + sort.Strings(waiting) + sort.Strings(failed) + sort.Strings(silent) + return waiting, failed, silent +} + // restOf is, for one module, every machine of its rest that the send reached and what it carried there. func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration { out := map[string]inventory.SentDeclaration{} diff --git a/cmd/mesh-controller/release_plan_test.go b/cmd/mesh-controller/release_plan_test.go index d26bcf57..5425de3c 100644 --- a/cmd/mesh-controller/release_plan_test.go +++ b/cmd/mesh-controller/release_plan_test.go @@ -1,6 +1,7 @@ package main import ( + "slices" "testing" "time" @@ -49,8 +50,10 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { if pos["mesh-tools"] != 0 || pos["builder"] != 1 { t.Fatalf("the runtime then the builder: %v", tiers) } - if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) { - t.Fatalf("what the builder builds comes after the builder: %v", tiers) + // Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it + // stands on. + if pos["shop"] != 1 || pos["postgres"] != 1 { + t.Fatalf("what the builder builds waits for the builder: %v", tiers) } if pos["shop-plugin"] <= pos["shop"] { t.Fatalf("a plugin after what it is declared on: %v", tiers) @@ -67,24 +70,23 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { // A change to a package all three build from moves all three. The builder holds // the build seat, whose worker the controller defines, so it follows the controller (worker-of, // novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the - // build machine that is running. The proxy is built by the new builder: the controller, the builder, - // the proxy — the live plan of every controller merge. (This read "the builder, then the controller - // and the proxy together" before issue 206, and the fixture had no worker-of edge.) + // build machine that is running. The proxy is built by whichever builder runs, so it goes beside the + // controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before). small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges) smallTiers := tiersOf(small, edges) - if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" { - t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers) + if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" { + t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers) } // The builder alone moved: the builder, and nothing it builds. if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 { t.Fatalf("a build machine change rebuilds the build machine alone: %v", only) } - // Only a runtime dependency gates on deployment, and only when the module rolls out. + // Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it. p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges) p.Tier = pos["builder"] rollsOut := func(m string) bool { return m == "builder" } - if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" { + if g := gates(p, edges, rollsOut); len(g) != 0 { t.Fatalf("the builder gates the tier after it: %v", g) } p.Tier = 0 @@ -253,10 +255,10 @@ func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) { t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers) } } - // With what it builds moved beside it, the agent comes first and they follow: an order, not a widening. + // With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing. p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"}, []string{"build-agent", "redis"}, edges) - if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 { - t.Fatalf("the agent, then what moved beside it: %v", p.Tiers) + if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 { + t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers) } } diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 8b6f7155..223d5e76 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -228,53 +228,6 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { } } -// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits -// for the next step, and one whose every build passed goes now (ADR 0353 §2). -func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, - } - passed := func(module, commit string) bool { return module == "app" && commit == "c2" } - never := func(string) bool { return false } - step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) - } - if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { - t.Fatalf("a machine whose every build passed waits: %v", step) - } -} - -// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step -// carries either to one machine only, while what else passed goes everywhere at once. -func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: hostModule, To: "h2"}}, - "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "dns", To: "d2"}}, - "ace": {{Module: "dns", To: "d2"}}, - } - resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} - alone := func(m string) bool { - if m == "dns" { - return oneAtATime(m, resolver) - } - return oneAtATime(m, catalogue.Manifest{Module: m}) - } - step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, - func(string, string) bool { return true }, alone) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) - } - if oneAtATime("app", catalogue.Manifest{Module: "app"}) { - t.Fatal("a leaf module is held to one machine at a time") - } -} - // A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { b := aBacklogOnThree(t) diff --git a/cmd/mesh-controller/worker_order_test.go b/cmd/mesh-controller/worker_order_test.go index c633ca55..86fb9073 100644 --- a/cmd/mesh-controller/worker_order_test.go +++ b/cmd/mesh-controller/worker_order_test.go @@ -38,8 +38,9 @@ func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing. if pos["build-agent"] <= pos["mesh-controller"] { t.Fatalf("the build machine after the controller that defines its worker: %v", tiers) } - if pos["route-proxy"] <= pos["build-agent"] { - t.Fatalf("what the build machine builds comes after it: %v", tiers) + // Built-by orders nothing (novox/hq ADR 0360 §3): whichever builder runs builds the proxy, beside the controller. + if pos["route-proxy"] != 0 { + t.Fatalf("what the builder builds waits for a new one: %v", tiers) } if hasCycle(tiers, edges) { t.Fatalf("no cycle here: %v", tiers)