novox/mesh-controller
8
0
Fork 0
Code Issues Pull Requests 5 Releases Wiki Activity

Keep the core over an older record instead of refusing the send (issue 201, review 90)
mesh/merge-gate out of date: judged against trunk f8daec01d3f0, the trunk is now ba685b17f5e1; judged again on a recheck or when it is merged
Details
mesh/repo-check out of date: judged against trunk f8daec01d3f0, the trunk is now ba685b17f5e1; judged again on a recheck or when it is merged
Details
mesh/delivery superseded: a newer head of the same pull request
Details

Browse Source 
Refusing the whole send wedged a machine whose gate put back the
controller and whose put-back send was refused: every later automatic
send there was refused, retried each tick, and held up every other walk.

A send now composes the controller and the node-engine at the build the
machine was last sent when the mesh's record is older, and carries the
rest; nothing is refused or retried, and a plan whose own build is the
older one ends with its gate superseded. A sent build that failed its
gate is never kept, so the next send completes a put-back. The condition
names the two ways out and clears on any send that carries the record,
a person's push included.
This commit is contained in:
jochen
2026-10-12 18:51:17 +02:00
parent 6bb1c9b768
commit d170491256
6 changed files with 267 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/issue201_test.go
+109 -79
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
@@ -15,104 +14,76 @@ import (
// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
// ten minutes. No send but a gate's own put-back moves the controller or the node-engine back behind the build its
// machine runs, by the builds' registration order.
// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
// behind the build its machine was last sent: the send keeps it there, and carries the rest.
// askedIn answers when a build was asked from a fixed table: module@commit → when.
func askedIn(table map[string]time.Time) func(module, commit string) (time.Time, bool, error) {
return func(module, commit string) (time.Time, bool, error) {
// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
// their gate.
func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
return func(module, commit string) (askedBuild, bool, error) {
at, known := table[module+"@"+commit]
return at, known, nil
b := askedBuild{asked: at}
for _, f := range failed {
if f == module+"@"+commit {
b.failedGate = true
}
}
return b, known, nil
}
}
func TestACoreModuleIsNeverMovedBehindTheBuildItsMachineRuns(t *testing.T) {
func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
old, now := t0, t0.Add(time.Minute)
asked := askedIn(map[string]time.Time{
table := map[string]time.Time{
"mesh-controller@c1": old, "mesh-controller@c2": now,
"mesh-host@h1": old, "mesh-host@h2": now,
"app@a1": old, "app@a2": now,
})
back := []inventory.CarriedMove{
{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"},
{Module: "mesh-host", Node: "anchor", From: "h2", To: "h1"},
{Module: "app", Node: "anchor", From: "a2", To: "a1"},
}
found, err := coreDowngrades(back, sendScope{judged: map[string]bool{"anchor": true}}, asked)
sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
if err != nil {
t.Fatal(err)
}
if len(found) != 2 || found[0].move.Module != "mesh-controller" || found[0].component != lease.ComponentController ||
found[1].move.Module != "mesh-host" || found[1].component != lease.ComponentEngine {
t.Fatalf("the controller and the node-engine sent back, and nothing else, are refused: %+v", found)
if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
}
if said := found[0].String(); !strings.Contains(said, "anchor would be sent mesh-controller c1") ||
!strings.Contains(said, "issue 201") {
t.Fatalf("the refusal says the machine, the module and the build: %s", said)
if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
}
// Forward, the same build, or a build the records do not hold: not refused.
for _, mv := range []inventory.CarriedMove{
{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"},
{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c2"},
{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c9"},
{Module: "mesh-controller", Node: "anchor", From: "c9", To: "c1"},
} {
if found, err := coreDowngrades([]inventory.CarriedMove{mv}, sendScope{}, asked); err != nil || len(found) != 0 {
t.Fatalf("%+v is no downgrade proved: %+v %v", mv, found, err)
// The record forward, the same build, or a build the records do not hold: nothing kept.
for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
if err != nil || len(found) != 0 {
t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
}
}
// A sent build that failed its gate is never ahead: the record put back before it is sent.
if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
found[0].module != "mesh-host" {
t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
}
// A gate's own put-back carries its own modules back, and nothing else.
putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
found, err = coreDowngrades(back, putBack, asked)
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].move.Module != "mesh-host" {
if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
}
// Naming a module is not a put-back.
if found, _ := coreDowngrades(back[:1], sendScope{modules: map[string]bool{"mesh-controller": true}}, asked); len(found) != 1 {
if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
askedIn(table)); len(found) != 2 {
t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
}
}
// The refusal is a warning condition on the machine, which the next send there carrying no such move clears.
func TestARefusedCoreDowngradeIsSaidAndClearedByTheNextSend(t *testing.T) {
k, _ := withConditionsInMemory(t)
ctx := t.Context()
was := buildAsked
buildAsked = func(_ context.Context, _ *inventory.Inventory, module, commit string) (time.Time, bool, error) {
return askedIn(map[string]time.Time{"mesh-controller@c1": t0, "mesh-controller@c2": t0.Add(time.Minute)})(module, commit)
}
t.Cleanup(func() { buildAsked = was })
said, err := refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}})
if err != nil || len(said) != 1 {
t.Fatalf("the downgrade is refused: %v %v", said, err)
}
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
c, open, err := k.Get(ctx, key)
if err != nil || !open || !strings.Contains(c.Summary, "mesh-controller c1") {
t.Fatalf("the condition %s is raised, saying the build: %+v %v %v", key, c, open, err)
}
said, err = refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}})
if err != nil || len(said) != 0 {
t.Fatalf("a move forward is sent: %v %v", said, err)
}
if _, open, err := k.Get(ctx, key); err != nil || open {
t.Fatalf("the condition is cleared by a send carrying no older controller: %v %v", open, err)
}
}
// TestReplay201 replays the race on a store: the control node runs the newer controller, the mesh's record of
// the controller is the build before it, and a plan's gated send there — the node-engine's plan of 2026-10-02 —
// is refused, said as the plan's words; the gate's own put-back of the controller is not.
func TestReplay201(t *testing.T) {
// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
// controller is the build before it, c1, which passed its gate when it rolled out.
func core201(t *testing.T) *stores {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
@@ -128,6 +99,10 @@ func TestReplay201(t *testing.T) {
return b
}
before := record("c1aaaaaaaaaa", old)
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
t.Fatal(err)
}
record("c2bbbbbbbbbb", now)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
@@ -139,7 +114,6 @@ func TestReplay201(t *testing.T) {
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
t.Fatal(err)
}
// The mesh's record goes back to the build before, as the record went on 2026-10-02.
if err := inv.RestoreModule(ctx, before); err != nil {
t.Fatal(err)
}
@@ -156,14 +130,70 @@ func TestReplay201(t *testing.T) {
return f.moves(node, modules, sent, known, all), nil
}
t.Cleanup(func() { machineMoves = wasMoves })
return open
}
// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
// as a person's push does, clears the condition.
func TestReplay201(t *testing.T) {
open := core201(t)
ctx := t.Context()
k, _ := withConditionsInMemory(t)
judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
if _, err := ungatedIn(judged, open, []string{"anchor"}, ""); !errors.Is(err, errCoreDowngrade) ||
!strings.Contains(err.Error(), "mesh-controller c1aaaaaa") {
t.Fatalf("a plan's send of the older controller to the machine running the newer is refused: %v", err)
if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
}
putBack := withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})
if kept, err := ungatedIn(putBack, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
t.Fatalf("the gate's own put-back of the controller is sent: %v %v", kept, err)
keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
}
shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
}
key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
}
putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
}
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
if _, found, _ := k.Get(ctx, key); !found {
t.Fatal("a send that kept the newer controller cleared the condition")
}
settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
if _, found, _ := k.Get(ctx, key); found {
t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
}
}
// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
// failed its gate is never kept.
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
open := core201(t)
ctx := t.Context()
k, _ := withConditionsInMemory(t)
if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
t.Fatal(err)
}
healer := withScope(ctx, sendScope{})
if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
t.Fatalf("the healer's send is made: %v %v", kept, err)
}
if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
}
if open, _ := k.Open(ctx); len(open) != 0 {
t.Fatalf("a completed put-back raised a condition: %+v", open)
}
}