package main import ( "context" "encoding/json" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) // novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there, // whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for // ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back // behind the build its machine was last sent: the send keeps it there, and carries the rest. // askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed // their gate. func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) { return func(module, commit string) (askedBuild, bool, error) { at, known := table[module+"@"+commit] b := askedBuild{asked: at} for _, f := range failed { if f == module+"@"+commit { b.failedGate = true } } return b, known, nil } } func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) { old, now := t0, t0.Add(time.Minute) table := map[string]time.Time{ "mesh-controller@c1": old, "mesh-controller@c2": now, "mesh-host@h1": old, "mesh-host@h2": now, "app@a1": old, "app@a2": now, } sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"} record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"} found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table)) if err != nil { t.Fatal(err) } if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController || found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine { t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found) } if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") || !strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") { t.Fatalf("it says the machine, the build kept and the two ways out: %s", said) } // The record forward, the same build, or a build the records do not hold: nothing kept. for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} { found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]}, map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table)) if err != nil || len(found) != 0 { t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err) } } // A sent build that failed its gate is never ahead: the record put back before it is sent. if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 || found[0].module != "mesh-host" { t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found) } // A gate's own put-back carries its own modules back, and nothing else. putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true} if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" { t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found) } if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}}, askedIn(table)); len(found) != 2 { t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found) } } // core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the // controller is the build before it, c1, which passed its gate when it rolled out. func core201(t *testing.T) *stores { t.Helper() open := aMesh(t) ctx := t.Context() inv := open.inventory old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) record := func(commit string, asked time.Time) inventory.Build { manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} if err := inv.RecordBuild(ctx, b); err != nil { t.Fatal(err) } return b } before := record("c1aaaaaaaaaa", old) if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { t.Fatal(err) } record("c2bbbbbbbbbb", now) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { t.Fatal(err) } if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { t.Fatal(err) } if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { t.Fatal(err) } if err := inv.RestoreModule(ctx, before); err != nil { t.Fatal(err) } wasMoves := machineMoves machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) if err != nil { return nil, err } sent, known, err := open.inventory.SentBuilds(ctx, node) if err != nil { return nil, err } return f.moves(node, modules, sent, known, all), nil } t.Cleanup(func() { machineMoves = wasMoves }) return open } // TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is // the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer // controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record, // as a person's push does, clears the condition. func TestReplay201(t *testing.T) { open := core201(t) ctx := t.Context() k, _ := withConditionsInMemory(t) judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) } keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) } shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" { t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) } key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) } putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) } settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) if _, found, _ := k.Get(ctx, key); !found { t.Fatal("a send that kept the newer controller cleared the condition") } settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) if _, found, _ := k.Get(ctx, key); found { t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") } } // The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was // refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that // failed its gate is never kept. func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { open := core201(t) ctx := t.Context() k, _ := withConditionsInMemory(t) if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { t.Fatal(err) } healer := withScope(ctx, sendScope{}) if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { t.Fatalf("the healer's send is made: %v %v", kept, err) } if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) } if open, _ := k.Open(ctx); len(open) != 0 { t.Fatalf("a completed put-back raised a condition: %+v", open) } } // Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept // controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own // kept controller as another build sent there after it (issue 352) and be superseded, judging nothing. func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { open := core201(t) ctx := t.Context() withConditionsInMemory(t) inv := open.inventory old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) for _, b := range []struct { commit string at time.Time }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { t.Fatal(err) } if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, Asked: b.at}); err != nil { t.Fatal(err) } } if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { t.Fatal(err) } if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { t.Fatal(err) } // The send as it is made: the controller kept at c2, the app moved to the record. wasSend := sendRollout sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) } t.Cleanup(func() { sendRollout = wasSend }) moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) if err != nil || len(sent) != 1 { t.Fatalf("the gated send is made: %v %v", sent, err) } if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) } builds, _, err := inv.SentBuilds(ctx, "anchor") if err != nil { t.Fatal(err) } g := &inventory.PlanGate{Carried: moves} f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} for _, m := range []string{"app", "mesh-controller"} { if why, superseded := f.supersededOn(m, "anchor"); superseded { t.Fatalf("the gate is superseded by its own send: %s", why) } } }