mesh-controller/cmd/mesh-controller/issue201_test.go

package main

import (
	"context"
	"encoding/json"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/lease"
)

// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
// behind the build its machine was last sent: the send keeps it there, and carries the rest.

// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
// their gate.
func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
	return func(module, commit string) (askedBuild, bool, error) {
		at, known := table[module+"@"+commit]
		b := askedBuild{asked: at}
		for _, f := range failed {
			if f == module+"@"+commit {
				b.failedGate = true
			}
		}
		return b, known, nil
	}
}

func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
	old, now := t0, t0.Add(time.Minute)
	table := map[string]time.Time{
		"mesh-controller@c1": old, "mesh-controller@c2": now,
		"mesh-host@h1": old, "mesh-host@h2": now,
		"app@a1": old, "app@a2": now,
	}
	sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
	record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
	found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
	if err != nil {
		t.Fatal(err)
	}
	if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
		found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
		t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
	}
	if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
		!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
		t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
	}

	// The record forward, the same build, or a build the records do not hold: nothing kept.
	for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
		found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
			map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
		if err != nil || len(found) != 0 {
			t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
		}
	}

	// A sent build that failed its gate is never ahead: the record put back before it is sent.
	if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
		found[0].module != "mesh-host" {
		t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
	}

	// A gate's own put-back carries its own modules back, and nothing else.
	putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
	if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
		t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
	}
	if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
		askedIn(table)); len(found) != 2 {
		t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
	}
}

// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
// controller is the build before it, c1, which passed its gate when it rolled out.
func core201(t *testing.T) *stores {
	t.Helper()
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
	record := func(commit string, asked time.Time) inventory.Build {
		manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
		b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
			Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
		if err := inv.RecordBuild(ctx, b); err != nil {
			t.Fatal(err)
		}
		return b
	}
	before := record("c1aaaaaaaaaa", old)
	if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
		Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
		t.Fatal(err)
	}
	record("c2bbbbbbbbbb", now)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
		Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
		t.Fatal(err)
	}
	if err := inv.RestoreModule(ctx, before); err != nil {
		t.Fatal(err)
	}
	wasMoves := machineMoves
	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
		modules, err := open.inventory.Assigned(ctx, node)
		if err != nil {
			return nil, err
		}
		sent, known, err := open.inventory.SentBuilds(ctx, node)
		if err != nil {
			return nil, err
		}
		return f.moves(node, modules, sent, known, all), nil
	}
	t.Cleanup(func() { machineMoves = wasMoves })
	return open
}

// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
// as a person's push does, clears the condition.
func TestReplay201(t *testing.T) {
	open := core201(t)
	ctx := t.Context()
	k, _ := withConditionsInMemory(t)

	judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
	if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
		t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
	}
	keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
	if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
		t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
	}
	shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
	if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
		t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
	}
	key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
	if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
		t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
	}

	putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
	if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
		t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
	}

	settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
	if _, found, _ := k.Get(ctx, key); !found {
		t.Fatal("a send that kept the newer controller cleared the condition")
	}
	settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
	if _, found, _ := k.Get(ctx, key); found {
		t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
	}
}

// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
// failed its gate is never kept.
func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
	open := core201(t)
	ctx := t.Context()
	k, _ := withConditionsInMemory(t)
	if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
		Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
		Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
		t.Fatal(err)
	}
	healer := withScope(ctx, sendScope{})
	if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
		t.Fatalf("the healer's send is made: %v %v", kept, err)
	}
	if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
		t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
	}
	if open, _ := k.Open(ctx); len(open) != 0 {
		t.Fatalf("a completed put-back raised a condition: %+v", open)
	}
}

// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
	open := core201(t)
	ctx := t.Context()
	withConditionsInMemory(t)
	inv := open.inventory
	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
	for _, b := range []struct {
		commit string
		at     time.Time
	}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
		manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
		if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
			Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
			t.Fatal(err)
		}
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
			Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
			Asked: b.at}); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
		map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
		t.Fatal(err)
	}
	// The send as it is made: the controller kept at c2, the app moved to the record.
	wasSend := sendRollout
	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
		return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
			map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
	}
	t.Cleanup(func() { sendRollout = wasSend })

	moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
	if err != nil || len(sent) != 1 {
		t.Fatalf("the gated send is made: %v %v", sent, err)
	}
	if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
		t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
	}
	builds, _, err := inv.SentBuilds(ctx, "anchor")
	if err != nil {
		t.Fatal(err)
	}
	g := &inventory.PlanGate{Carried: moves}
	f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
	for _, m := range []string{"app", "mesh-controller"} {
		if why, superseded := f.supersededOn(m, "anchor"); superseded {
			t.Fatalf("the gate is superseded by its own send: %s", why)
		}
	}
}