mesh-controller/cmd/mesh-controller/issue352_test.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/lease"
)

// novox/hq issue 352: on 2026-10-09 a release's gate on the control node read the machine's report against a
// newer send another plan had just made there — not against its own send — and failed three builds the
// machine had reported healthy ("has not reported on what it was sent"), put them back on every machine,
// to a controller older than the store's schema, and that controller then judged the newer plan's
// controller passed from the put-back build's health.

// TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports;
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
// to another build supersedes the judging: no verdict, nothing put back.
func TestReplay352(t *testing.T) {
	t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
	t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
}

func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
	b := aBacklog(t)
	ctx := t.Context()
	inv := b.open.inventory
	advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied
	// 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on.
	carried, _, _ := inv.SentBuilds(ctx, "anchor")
	if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil {
		t.Fatal(err)
	}
	reports, _ := inv.LastReports(ctx)
	for _, r := range reports {
		if r.Node == "anchor" && r.Current {
			t.Fatal("the fixture's newer send reads as reported")
		}
	}
	gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here
	for i := 0; i < 4; i++ {
		advancePlans(ctx, b.open)
	}
	p := b.release(t)
	if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") {
		t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note)
	}
	if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") {
		t.Fatalf("the gate does not keep what it sent: %+v", g)
	}
	if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
		t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err)
	}
	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
		t.Fatalf("app was put back to %s", current["app"].Commit)
	}
}

// A plan's own first send waits while a release judges the same module on that machine with another build.
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
	b := aBacklog(t)
	ctx := t.Context()
	advancePlans(ctx, b.open) // the release judges app c2 on anchor
	_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
	if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
		t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
	}
	if len(b.sent) != 1 {
		t.Fatalf("sent %v", b.sent)
	}
}

// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
	g := aGateMesh(t)
	ctx := t.Context()
	inv := g.open.inventory
	advancePlans(ctx, g.open) // anchor is sent app c2 first
	if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil {
		t.Fatal(err)
	}
	gateEvery = 0
	advancePlans(ctx, g.open)
	p := g.plan(t)
	if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
		t.Fatalf("the plan is %s: %s", p.State, p.Note)
	}
	// Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no
	// rollback (a release may walk what the other send left waiting on anchor; that is not a put-back).
	if r := p.Modules["app"].Gate.Rollback; r != "" {
		t.Fatalf("a superseded judging made a rollback: %q", r)
	}
	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
		t.Fatalf("app was put back to %s", current["app"].Commit)
	}
	if failed, _ := inv.GateFailed(ctx, "build-2"); failed {
		t.Fatal("a superseded build was marked failed")
	}
}

func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
	b := aBacklog(t)
	ctx := t.Context()
	inv := b.open.inventory
	advancePlans(ctx, b.open)
	// Another send moves app on anchor to a build this gate does not judge.
	carried, _, _ := inv.SentBuilds(ctx, "anchor")
	carried["app"] = "c3"
	if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil {
		t.Fatal(err)
	}
	gateEvery = 0
	advancePlans(ctx, b.open)
	p := b.release(t)
	if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
		t.Fatalf("the release is %s: %s", p.State, p.Note)
	}
	if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
		t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent)
	}
	if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found {
		t.Fatal("a superseded judging kept a verdict")
	}
	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
		t.Fatalf("app was put back to %s", current["app"].Commit)
	}
}

// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
// without its send reads the report against the send made last, as before. Pure.
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
	sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
	for _, c := range []struct {
		r    inventory.Reported
		want bool
	}{
		{inventory.Reported{Declared: "d-490", Current: false}, true},
		{inventory.Reported{Declared: "d-491", ReportedSequence: 491, Current: true}, true},
		{inventory.Reported{Declared: "d-489", ReportedSequence: 489, Current: false}, false},
		{inventory.Reported{Declared: "other", ReportedSequence: 490}, true},                // the same sequence, said by another digest
		{inventory.Reported{Declared: "d-495", ReportedSequence: 495, Current: true}, true}, // the last send: this one or a later one
		{inventory.Reported{Declared: "", Current: false}, false},
	} {
		if got := sent.ReportsOn(c.r); got != c.want {
			t.Errorf("%+v on %+v: %v", c.r, sent, got)
		}
	}
	byDigest := inventory.SentDeclaration{Digest: "d-1"}
	if !byDigest.ReportsOn(inventory.Reported{Declared: "d-1"}) || byDigest.ReportsOn(inventory.Reported{ReportedSequence: 5}) ||
		!byDigest.ReportsOn(inventory.Reported{Current: true}) {
		t.Fatal("a send kept without a sequence is matched by its digest and by the last send alone")
	}
	f := gateFacts{sent: map[string]inventory.SentDeclaration{"anchor": sent}}
	if !f.reportedOn("anchor", inventory.Reported{Declared: "d-490"}) || f.reportedOn("anchor", inventory.Reported{Declared: "d-1"}) {
		t.Fatal("a gate that kept its send read the report against something other than it")
	}
	if !f.reportedOn("laptop", inventory.Reported{Current: true}) || f.reportedOn("laptop", inventory.Reported{Current: false}) {
		t.Fatal("a gate that did not keep its send does not read the report against the send made last")
	}
	// Through the merge plan's first-machine wait too.
	at := time.Now()
	state := inventory.PlanModule{First: []string{"anchor"}, FirstAt: &at,
		Gate: &inventory.PlanGate{Machines: []string{"anchor"}, Sent: map[string]inventory.SentDeclaration{"anchor": sent}}}
	reports := []inventory.Reported{{Node: "anchor", At: &at, Outcome: inventory.OutcomeApplied, Current: false, Declared: "d-490"}}
	if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting != "" || step.failed != "" {
		t.Fatalf("the first machine's report on the plan's own send read as none: %+v", step)
	}
	reports[0].Declared = "d-480"
	if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting == "" {
		t.Fatalf("a report on an older send read as the plan's: %+v", step)
	}
}

// A gate judges only the build the machine was last sent: last sent another build of the module, the
// judging is superseded, whatever the machine reports. Pure.
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
	at := time.Now()
	f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
		At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
		commits: map[string]string{"mesh-controller": "e6b00e2e"}, sentBuilds: map[string]map[string]string{"anchor": {"mesh-controller": "ef26d4cb"}}}
	taken := at.Add(-30 * time.Second)
	f.holder = &lease.Holder{Taken: taken, Health: &lease.Health{Ready: true}}
	h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f)
	if h != healthSuperseded || !strings.Contains(why, "ef26d4cb") || !strings.Contains(why, "e6b00e2e") {
		t.Fatalf("a controller build the machine no longer runs: %v %q", h, why)
	}
	f.sentBuilds["anchor"]["mesh-controller"] = "e6b00e2e"
	if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
		t.Fatalf("the build sent read as another: %q", why)
	}
	delete(f.sentBuilds, "anchor")
	if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded {
		t.Fatalf("a machine whose send is not known read as superseded: %q", why)
	}
	g := &inventory.PlanGate{To: "c2", Carried: []inventory.CarriedMove{{Module: "late", Node: "anchor", To: "c5"}}}
	if got := judgedCommits(g, []judged{{"app", "anchor"}, {"late", "anchor"}}); got["app"] != "c2" || got["late"] != "c5" {
		t.Fatalf("judged commits %v", got)
	}
}

// A move of another build of a module to a machine where a release or a plan is judging that module
// waits for that judging; the same build to that machine is already there. Pure.
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
	at := time.Now()
	release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
		Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
			{Module: "mesh-controller", Node: "novox", From: "ef26d4cb", To: "2913c54c"}}}}}
	merge := inventory.Plan{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
		"app": {First: []string{"anchor"}, FirstAt: &at, Commit: "c2", Gate: &inventory.PlanGate{Machines: []string{"anchor"}}}}}
	f := moveFacts{plans: []inventory.Plan{release, merge}}
	for _, c := range []struct {
		module, node, to, want string
	}{
		{"mesh-controller", "novox", "e6b00e2e", "release-1"}, // the day's case: a newer controller to the judged machine
		{"mesh-controller", "novox", "2913c54c", ""},          // the same build: already there
		{"mesh-controller", "ace", "2913c54c", "release-1"},   // another machine while the first is judged
		{"mesh-host", "novox", "x", ""},                       // a module the release does not carry
		{"app", "anchor", "c2", ""},
		{"app", "anchor", "c3", "plan-1"},
		{"app", "laptop", "c2", "plan-1"},
	} {
		if got := f.walkedBy(c.module, c.node, c.to); got != c.want {
			t.Errorf("%s %s to %s: walked by %q, want %q", c.module, c.to, c.node, got, c.want)
		}
	}
	release.Release.Gate.Verdict = inventory.GatePassed
	merge.Modules["app"].Gate.Verdict = inventory.GatePassed
	if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" || f.walkedBy("app", "laptop", "c3") != "" {
		t.Fatal("a passed judging still holds a move")
	}
	release.Release.Gate.Verdict = ""
	f.plans[0].State = inventory.PlanSuperseded
	if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" {
		t.Fatal("a closed release still holds a move")
	}
}

// The controller is never put back to a build that reaches less of the store's schema than the store
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	keeper, _ := withConditionsInMemory(t)
	told := &conditions.Told{}
	was := doctorFrom
	doctorFrom = &doctor{open: open, keeper: keeper, teller: told}
	t.Cleanup(func() { doctorFrom = was })
	wasSend := sendRollout
	var sent [][]string
	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
		sent = append(sent, names)
		return names, nil
	}
	t.Cleanup(func() { sendRollout = wasSend })

	build := func(id, commit, digest string, asked time.Time) inventory.Build {
		manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit})
		b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".",
			Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle,
				Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}}
		if err := inv.RecordBuild(ctx, b); err != nil {
			t.Fatal(err)
		}
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit},
			inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil {
			t.Fatal(err)
		}
		return b
	}
	previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour))
	failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute))
	if versionOfBuild(previous) != strings.Repeat("1", 12) {
		t.Fatalf("the build's version is %q", versionOfBuild(previous))
	}
	applied, err := inv.SchemaApplied(ctx)
	if err != nil || applied < 87 {
		t.Fatalf("the store's schema reaches %d (%v)", applied, err)
	}
	// The build before never recorded what it reads: not proved, refused.
	if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") {
		t.Fatalf("an unknown reach: %v %q", ok, why)
	}
	// It reads less than the store has: refused, naming both.
	if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil {
		t.Fatal(err)
	}
	if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") {
		t.Fatalf("a reach behind the store: %v %q", ok, why)
	}
	// Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent.
	at := time.Now().Add(-5 * time.Minute)
	state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at}
	p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at,
		State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}}
	if err := inv.SavePlan(ctx, &p); err != nil {
		t.Fatal(err)
	}
	gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply")
	if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") {
		t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note)
	}
	if len(sent) != 0 {
		t.Fatalf("sent %v: nothing is put back", sent)
	}
	if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit {
		t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit)
	}
	if marked, _ := inv.GateFailed(ctx, failed.ID); !marked {
		t.Fatal("the failed build is not marked failed at its gate")
	}
	open2, _ := keeper.Open(ctx)
	var found bool
	for _, c := range open2 {
		if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") {
			found = true
		}
	}
	if !found {
		t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2)
	}
	// Reaching the store: allowed.
	if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil {
		t.Fatal(err)
	}
	if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
		t.Fatalf("a build that reads the whole schema was refused: %q", why)
	}
	// A build with no bundle to know it by: refused.
	if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") {
		t.Fatalf("a build without a bundle: %v %q", ok, why)
	}
}

// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
	t.Setenv(RunningBuildVar, " ad62528c47c7 ")
	if h := holderOf("x"); h.Build != "ad62528c47c7" {
		t.Fatalf("the holder's build is %q", h.Build)
	}
	t.Setenv(RunningBuildVar, "")
	if h := holderOf("x"); h.Build != version {
		t.Fatalf("without a declared version the holder's build is %q", h.Build)
	}
	if reach, err := schemaReach(); err != nil || reach < 87 {
		t.Fatalf("this build's reach is %d (%v)", reach, err)
	}
}