diff --git a/cmd/mesh-controller/core_downgrade.go b/cmd/mesh-controller/core_downgrade.go new file mode 100644 index 00000000..e66f11c9 --- /dev/null +++ b/cmd/mesh-controller/core_downgrade.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// **A send never moves the mesh's own core to a build older than the one its machine runs** (novox/hq issue +// 201). On 2026-10-02 two merges a minute apart made two plans; the controller's plan rolled the new controller, +// which widened its seat's row, and the node-engine's plan then sent the control node the controller build +// before it. The older binary read the row it could not serve and the mesh lost its voice for ten minutes — the +// thing that would have corrected it was the thing that was down. ADR 0185 took the outage out of it (a +// controller behind its row serves what it can); the race is closed here, without needing its trigger: whatever +// composed the older build, the send that would carry it is refused. +// +// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the +// commits' dates. A gate's own put-back is the one send that may carry it, for the modules it puts back: it says +// so (sendScope.puttingBack), and the controller's put-back is held to the store's schema already (issue 352). +// A person's push carries what it carries, as every other refusal of a send leaves it. +// +// The refusal is said in the plan's words — the send's error, which the plan keeps as its note and tries again +// — and as a warning condition on the machine, cleared by the first send there that carries no such move. + +// errCoreDowngrade is a send refused because it would put a core module on a machine back behind the build that +// machine runs. +var errCoreDowngrade = errors.New("a send would put the mesh's own core back behind the build its machine runs") + +// kindCoreDowngrade is that refusal, as a condition. +const kindCoreDowngrade = "core-downgrade-refused" + +// coreDowngradeSource is the condition's source: the sends, which raise and clear it. +const coreDowngradeSource = "sends" + +// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the +// store and its seat's row, and the node-engine, which reads what the controller declares. +var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine} + +// buildAsked is when the newest build of a module from a commit that worked was asked — or recorded, when the +// request time is not known — and false when the records hold none. A variable, so a test reads its own order. +var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (time.Time, bool, error) { + builds, err := inv.Builds(ctx, module, askedLookBack) + if err != nil { + return time.Time{}, false, err + } + for _, b := range builds { + if b.Worked() && sameCommit(b.Commit, commit) { + return b.AskedOrAt(), true, nil + } + } + return time.Time{}, false, nil +} + +// askedLookBack is how many of a module's newest builds are read for when a build was asked. +const askedLookBack = 50 + +// coreDowngrade is one move of a core module to an older build than its machine runs. +type coreDowngrade struct { + move inventory.CarriedMove + component string + fromAsked, asked time.Time +} + +func (d coreDowngrade) String() string { + return fmt.Sprintf("%s would be sent %s %s, older than the %s it runs (asked %s, before %s) — no send but a "+ + "gate's own put-back moves the %s back (novox/hq issue 201)", d.move.Node, d.move.Module, short(d.move.To), + short(d.move.From), d.asked.UTC().Format(time.RFC3339), d.fromAsked.UTC().Format(time.RFC3339), d.component) +} + +// coreDowngrades is the moves of a machine's send that put a core module back to a build asked before the one it +// runs, leaving out what a put-back puts back. A move whose either build the records do not hold is not proved +// older, and is left to the gate that judges it. Pure but for asked. +func coreDowngrades(moves []inventory.CarriedMove, scope sendScope, + asked func(module, commit string) (time.Time, bool, error)) ([]coreDowngrade, error) { + var out []coreDowngrade + for _, mv := range moves { + component := coreComponent(mv.Module) + if !guarded(component) || mv.From == "" || mv.To == "" || sameCommit(mv.From, mv.To) { + continue + } + if scope.puttingBack && scope.modules[mv.Module] { + continue + } + from, known, err := asked(mv.Module, mv.From) + if err != nil || !known { + if err != nil { + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.From), err) + } + continue + } + to, known, err := asked(mv.Module, mv.To) + if err != nil { + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.To), err) + } + if known && to.Before(from) { + out = append(out, coreDowngrade{move: mv, component: component, fromAsked: from, asked: to}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].move.Module < out[j].move.Module }) + return out, nil +} + +func guarded(component string) bool { + for _, c := range downgradeGuarded { + if c == component { + return true + } + } + return false +} + +// refuseCoreDowngrades says, for one machine's moves, each that would put a core module back, raising its +// condition; and clears the condition of each guarded component that machine's send does not move back. +func refuseCoreDowngrades(ctx context.Context, inv *inventory.Inventory, scope sendScope, node string, + moves []inventory.CarriedMove) ([]string, error) { + found, err := coreDowngrades(moves, scope, func(module, commit string) (time.Time, bool, error) { + return buildAsked(ctx, inv, module, commit) + }) + if err != nil { + return nil, err + } + var said []string + refused := map[string]bool{} + for _, d := range found { + said = append(said, d.String()) + refused[d.component] = true + sayCoreDowngrade(ctx, d) + } + for _, c := range downgradeGuarded { + if !refused[c] { + clearCoreDowngrade(ctx, c, node) + } + } + return said, nil +} + +// coreDowngradeObservation is a refused downgrade as a condition: `core...`. +func coreDowngradeObservation(component, node, summary string) conditions.Observation { + return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade, + Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource, + Summary: summary} +} + +func sayCoreDowngrade(ctx context.Context, d coreDowngrade) { + fmt.Printf("NOT SENT: %s\n", d) + keeper := conditionsFrom + if keeper == nil { + return + } + if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.move.Node, d.String())); err != nil { + fmt.Printf("the condition for %s's refused downgrade on %s could not be raised: %v\n", d.move.Module, + d.move.Node, err) + } +} + +func clearCoreDowngrade(ctx context.Context, component, node string) { + keeper := conditionsFrom + if keeper == nil { + return + } + key := coreDowngradeObservation(component, node, "").Key() + if _, err := keeper.Clear(ctx, key, "a send to "+node+" carries no older "+component); err != nil { + fmt.Printf("the condition %s could not be cleared: %v\n", key, err) + } +} diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index b4dd98e2..f5aa1689 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str } return } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll } inv := open.inventory sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines) var back []string for _, r := range b.pending { if err != nil { diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go new file mode 100644 index 00000000..9ec8f3bd --- /dev/null +++ b/cmd/mesh-controller/issue201_test.go @@ -0,0 +1,169 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there, +// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for +// ten minutes. No send but a gate's own put-back moves the controller or the node-engine back behind the build its +// machine runs, by the builds' registration order. + +// askedIn answers when a build was asked from a fixed table: module@commit → when. +func askedIn(table map[string]time.Time) func(module, commit string) (time.Time, bool, error) { + return func(module, commit string) (time.Time, bool, error) { + at, known := table[module+"@"+commit] + return at, known, nil + } +} + +func TestACoreModuleIsNeverMovedBehindTheBuildItsMachineRuns(t *testing.T) { + old, now := t0, t0.Add(time.Minute) + asked := askedIn(map[string]time.Time{ + "mesh-controller@c1": old, "mesh-controller@c2": now, + "mesh-host@h1": old, "mesh-host@h2": now, + "app@a1": old, "app@a2": now, + }) + back := []inventory.CarriedMove{ + {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}, + {Module: "mesh-host", Node: "anchor", From: "h2", To: "h1"}, + {Module: "app", Node: "anchor", From: "a2", To: "a1"}, + } + found, err := coreDowngrades(back, sendScope{judged: map[string]bool{"anchor": true}}, asked) + if err != nil { + t.Fatal(err) + } + if len(found) != 2 || found[0].move.Module != "mesh-controller" || found[0].component != lease.ComponentController || + found[1].move.Module != "mesh-host" || found[1].component != lease.ComponentEngine { + t.Fatalf("the controller and the node-engine sent back, and nothing else, are refused: %+v", found) + } + if said := found[0].String(); !strings.Contains(said, "anchor would be sent mesh-controller c1") || + !strings.Contains(said, "issue 201") { + t.Fatalf("the refusal says the machine, the module and the build: %s", said) + } + + // Forward, the same build, or a build the records do not hold: not refused. + for _, mv := range []inventory.CarriedMove{ + {Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}, + {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c2"}, + {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c9"}, + {Module: "mesh-controller", Node: "anchor", From: "c9", To: "c1"}, + } { + if found, err := coreDowngrades([]inventory.CarriedMove{mv}, sendScope{}, asked); err != nil || len(found) != 0 { + t.Fatalf("%+v is no downgrade proved: %+v %v", mv, found, err) + } + } + + // A gate's own put-back carries its own modules back, and nothing else. + putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true} + found, err = coreDowngrades(back, putBack, asked) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].move.Module != "mesh-host" { + t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found) + } + // Naming a module is not a put-back. + if found, _ := coreDowngrades(back[:1], sendScope{modules: map[string]bool{"mesh-controller": true}}, asked); len(found) != 1 { + t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found) + } +} + +// The refusal is a warning condition on the machine, which the next send there carrying no such move clears. +func TestARefusedCoreDowngradeIsSaidAndClearedByTheNextSend(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + was := buildAsked + buildAsked = func(_ context.Context, _ *inventory.Inventory, module, commit string) (time.Time, bool, error) { + return askedIn(map[string]time.Time{"mesh-controller@c1": t0, "mesh-controller@c2": t0.Add(time.Minute)})(module, commit) + } + t.Cleanup(func() { buildAsked = was }) + + said, err := refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor", + []inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}}) + if err != nil || len(said) != 1 { + t.Fatalf("the downgrade is refused: %v %v", said, err) + } + key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() + c, open, err := k.Get(ctx, key) + if err != nil || !open || !strings.Contains(c.Summary, "mesh-controller c1") { + t.Fatalf("the condition %s is raised, saying the build: %+v %v %v", key, c, open, err) + } + + said, err = refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor", + []inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}}) + if err != nil || len(said) != 0 { + t.Fatalf("a move forward is sent: %v %v", said, err) + } + if _, open, err := k.Get(ctx, key); err != nil || open { + t.Fatalf("the condition is cleared by a send carrying no older controller: %v %v", open, err) + } +} + +// TestReplay201 replays the race on a store: the control node runs the newer controller, the mesh's record of +// the controller is the build before it, and a plan's gated send there — the node-engine's plan of 2026-10-02 — +// is refused, said as the plan's words; the gate's own put-back of the controller is not. +func TestReplay201(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + record := func(commit string, asked time.Time) inventory.Build { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) + b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, + Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + return b + } + before := record("c1aaaaaaaaaa", old) + record("c2bbbbbbbbbb", now) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { + t.Fatal(err) + } + // The mesh's record goes back to the build before, as the record went on 2026-10-02. + if err := inv.RestoreModule(ctx, before); err != nil { + t.Fatal(err) + } + wasMoves := machineMoves + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + t.Cleanup(func() { machineMoves = wasMoves }) + + judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) + if _, err := ungatedIn(judged, open, []string{"anchor"}, ""); !errors.Is(err, errCoreDowngrade) || + !strings.Contains(err.Error(), "mesh-controller c1aaaaaa") { + t.Fatalf("a plan's send of the older controller to the machine running the newer is refused: %v", err) + } + putBack := withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}) + if kept, err := ungatedIn(putBack, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("the gate's own put-back of the controller is sent: %v %v", kept, err) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 7cd4f3b6..e2caf2d4 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -182,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{ "register it, so it was not made.", module, m), Resolved: "Resolved: " + module + " on " + m + " is settled"} }), + // A send that would have put the controller or the node-engine back behind what a machine runs (novox/hq + // issue 201). + kindCoreDowngrade: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + component := idPart(o, 0) + if o.Scope != conditions.ScopeCore || !guarded(component) { + component = "core component" + } + return words{Headline: "An older " + component + " was not sent to " + m, + Needs: "decide which build of the " + component + " the mesh should hold; the details say which.", + Explanation: fmt.Sprintf("A send would have put the mesh's own %s on %s back to an older build than "+ + "the one it runs, so it was not sent and %s keeps its build.", component, m, m), + Resolved: "Resolved: sends to " + m + " carry no older " + component} + }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index bc16fd84..5f23d856 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -51,6 +51,9 @@ type sendScope struct { modules map[string]bool // person is a person's act: it carries what it carries. person bool + // puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build + // older than the one its machine runs (novox/hq issue 201). + puttingBack bool } type sendScopeKey struct{} @@ -248,9 +251,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st if err != nil { return nil, err } - var kept, refused []string + var kept, refused, older []string for _, n := range names { - moves, err := machineMoves(ctx, open, f, n, false) + // Every move, gated or not: a move to a build that passed a gate elsewhere is no gate's concern here, and + // may still be a core module's downgrade (novox/hq issue 201). + every, err := machineMoves(ctx, open, f, n, true) if err != nil { return nil, err } @@ -262,11 +267,33 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st return nil, err } } - var waiting []string - for _, mv := range moves { + var made []inventory.CarriedMove + for _, mv := range every { if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { continue } + made = append(made, mv) + } + // **No send but a gate's own put-back moves the controller or the node-engine to an older build than + // the one its machine runs** (novox/hq issue 201). + downgrades, err := refuseCoreDowngrades(ctx, open.inventory, scope, n, made) + if err != nil { + return nil, err + } + if len(downgrades) > 0 { + if n == addedHolder { + fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s\n", n, + strings.Join(downgrades, "; ")) + continue + } + older = append(older, downgrades...) + continue + } + var waiting []string + for _, mv := range made { + if f.gated(mv.Module, mv.To) { + continue + } if !scope.judged[n] && !scope.modules[mv.Module] { waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) } @@ -281,6 +308,9 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", "))) } } + if len(older) > 0 { + return nil, fmt.Errorf("%w: %s", errCoreDowngrade, strings.Join(older, "; ")) + } if len(refused) > 0 { return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+ "0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))