mesh-controller/cmd/mesh-controller/walk_resumed_silence_1886_test.go

package main

import (
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/inventory"
)

// Issue 1886, the resume of R4e (plan-1792707174627079192): tier 0 was sent at 00:23:28, the walk was stopped at
// 02:01 and resumed at 07:43, and tier 1 was sent at 07:49:54. 35 seconds later the gate named ace silent "within
// 15m0s", and at 07:51:30 shanks, in the same second shanks reported it applied: the plan failed as lost. The walk
// judged each machine on every send it had made it, tier 0's of 00:23 too, narrowed to every module the walk carried
// there — so tier 0's send was held against the templates of tier 1's modules it never carried, and silence was
// measured from 00:23. A machine is judged on the walk's last send to it: that send carries its whole declaration.

func r4eResumed() (inventory.Plan, []inventory.Reported, inventory.ProvLookup, time.Time) {
	tier0 := time.Date(2026, 10, 22, 22, 23, 28, 0, time.UTC)
	tier1 := time.Date(2026, 10, 23, 5, 49, 54, 0, time.UTC)
	now := time.Date(2026, 10, 23, 5, 51, 30, 0, time.UTC)
	t0, t1 := tier0, tier1
	p := inventory.Plan{ID: "plan-1792707174627079192", State: inventory.PlanRolling, Tier: 1,
		Tiers: [][]string{{"pacman"}, {"mesh-build-image"}, {"gitea"}},
		Modules: map[string]*inventory.PlanModule{
			"pacman": {State: "built", SentAt: &t0, Rest: map[string]inventory.SentDeclaration{
				"shanks": {Digest: "d0-shanks", Sequence: 900, At: tier0,
					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}},
				"ace": {Digest: "d0-ace", Sequence: 1100, At: tier0,
					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}}}},
			"mesh-build-image": {State: "built", SentAt: &t1, Rest: map[string]inventory.SentDeclaration{
				"shanks": {Digest: "d1-shanks", Sequence: 971, At: tier1,
					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}},
				"ace": {Digest: "d1-ace", Sequence: 1212, At: tier1,
					Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}}}},
			"gitea": {}}}
	shanksAt := now
	aceAt := time.Date(2026, 10, 23, 0, 57, 50, 0, time.UTC)
	reports := []inventory.Reported{
		// shanks reported on tier 1's send in the second the gate looked.
		{Node: "shanks", Outcome: inventory.OutcomeApplied, At: &shanksAt, Declared: "d1-shanks", Current: true,
			ReportedSequence: 971},
		// ace's last report is from before tier 1's send, on a declaration sequenced after tier 0's.
		{Node: "ace", Outcome: inventory.OutcomeApplied, At: &aceAt, Declared: "d-ace-0257", ReportedSequence: 1150},
	}
	says := map[string]inventory.ProvReport{
		"shanks/pacman":           {Status: inventory.ProvApplied, Templates: "pac-1", At: now},
		"shanks/mesh-build-image": {Status: inventory.ProvApplied, Templates: "img-new", At: now},
		"ace/pacman":              {Status: inventory.ProvApplied, Templates: "pac-1", At: aceAt},
		// ace's client had not placed the old image's files (issue 1884), and has not yet the new one's.
		"ace/mesh-build-image": {Status: inventory.ProvWaiting, Templates: "img-old", At: aceAt},
	}
	prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
	return p, reports, prov, now
}

func TestAResumedWalkJudgesEachMachineOnItsLastSend(t *testing.T) {
	p, reports, prov, now := r4eResumed()
	waiting, failed, silent := walkReports(p, reports, now, prov)
	if len(silent) != 0 {
		t.Fatalf("1886: %v named silent within %s of tier 1's send, from tier 0's send of 00:23", silent, applySilentAfter)
	}
	if len(failed) != 0 || !slices.Equal(waiting, []string{"ace"}) {
		t.Fatalf("1886: waiting %v, failed %v; want ace waited for (sent 96 s ago) and shanks reported", waiting, failed)
	}
	// The gate the plan failed at: mesh-build-image runs on ace and shanks, which tier 2 needs.
	needs := []gateNeed{{module: "mesh-build-image", running: []string{"ace", "shanks"},
		since: func(string) time.Time { return *p.Modules["mesh-build-image"].SentAt }}}
	w, quiet, lost := settleTierGate(needs, reports, silentNodes(p, reports, now, prov))
	if lost || len(quiet) != 0 || !slices.Equal(w, []string{"mesh-build-image on ace"}) {
		t.Fatalf("1886: the gate says waiting %v, silent %v, lost %v; want mesh-build-image on ace waited for", w, quiet, lost)
	}
}

// The machine silent on the walk's last send to it is still silent once the bound has passed since that send.
func TestAMachineSilentOnItsLastSendIsStillSilent(t *testing.T) {
	p, reports, prov, now := r4eResumed()
	later := now.Add(applySilentAfter + time.Minute)
	_, _, silent := walkReports(p, reports, later, prov)
	if !slices.Equal(silent, []string{"ace"}) {
		t.Fatalf("1886: silent %v past the bound from tier 1's send; want ace", silent)
	}
}

// A walk failed only because every machine its gate waited on was silent, whose tiers are all sent, is carried on by
// `plans retry` once those machines reported (issue 1886): it failed on no build and no rollout, so the retry of failed
// builds and of stopped rollouts refused it with "nothing in tier 1 … failed to build or stopped rolling out".
func r4eLost() inventory.Plan {
	p, _, _, _ := r4eResumed()
	p.State = inventory.PlanFailed
	p.Note = "tier 1 built; " + silentGateWords + " 15m0s: shanks"
	// mesh-tools is built and recorded, never sent, as R4e's was.
	p.Tiers[1] = append(p.Tiers[1], "mesh-tools")
	p.Modules["mesh-tools"] = &inventory.PlanModule{State: "built"}
	return p
}

func TestRetryCarriesOnAWalkLostToSilenceOnceItsMachinesReported(t *testing.T) {
	p, reports, prov, now := r4eResumed()
	p = r4eLost()
	if !lostToSilence(p) {
		t.Fatalf("1886: the plan R4e failed as is not read as lost to silence: %q", p.Note)
	}
	if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil {
		t.Fatalf("1886: a walk lost to silence was refused a retry: %v", err)
	}
	reports[1].Current, reports[1].Declared, reports[1].ReportedSequence = true, "d1-ace", 1212
	reports[1].At = &now
	says := map[string]inventory.ProvReport{}
	for _, n := range []string{"ace", "shanks"} {
		says[n+"/pacman"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "pac-1", At: now}
		says[n+"/mesh-build-image"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "img-new", At: now}
	}
	prov = func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
	said, err := resumeLost(&p, silentNodes(p, reports, now.Add(time.Hour), prov))
	if err != nil {
		t.Fatalf("1886: the resume of a walk whose machines reported refused: %v", err)
	}
	if p.State != inventory.PlanBuilding || p.Tier != 1 {
		t.Fatalf("1886: resumed walk is %s at tier %d; want building at tier 1, for the next look to pass its gate", p.State, p.Tier)
	}
	if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(said, "resumed at tier 2 of 3") {
		t.Fatalf("1886: note %q, said %q", p.Note, said)
	}
}

// Every machine its tier was sent to still silent: the gate would fail it again, so the retry says so instead.
func TestRetryRefusesAWalkLostToSilenceWhileEveryMachineIsStillSilent(t *testing.T) {
	p := r4eLost()
	if _, err := resumeLost(&p, map[string]bool{"ace": true, "shanks": true}); err == nil || !strings.Contains(err.Error(), "ace, shanks") {
		t.Fatalf("1886: a walk whose machine is still silent was resumed: %v", err)
	}
	if p.State != inventory.PlanFailed {
		t.Fatalf("1886: a refused resume changed the plan to %s", p.State)
	}
}

// A walk lost to silence with something of its tier never sent is not this: its send is the look's to make.
func TestALostWalkWithATierUnsentIsNotLostToSilence(t *testing.T) {
	p := r4eLost()
	p.Modules["mesh-build-image"].State = "asked"
	if lostToSilence(p) {
		t.Fatal("1886: a tier with a module still asked was read as lost to silence")
	}
	p = r4eLost()
	p.Note = "tier 1 built; mesh-build-image failed to build"
	if lostToSilence(p) {
		t.Fatal("1886: a plan failed by a build was read as lost to silence")
	}
}

// A walk lost to silence is refused its resume beside a newer open plan of its repository and branch, which supersedes
// it, and beside a related open plan: one plan at a time (ADR 0531).
func TestRetryRefusesAWalkLostToSilenceBesideANewerOrARelatedOpenPlan(t *testing.T) {
	p := r4eLost()
	created := time.Date(2026, 10, 22, 22, 15, 9, 0, time.UTC)
	p.Repository, p.Branch, p.Commit, p.Created = "novox/mesh-sdk", "main", "7aacbc16", created
	newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed",
		Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}},
		Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}}
	if err := retryRefusal(p, []inventory.Plan{p, newerOpen}, ownPathOf(nil)); err == nil ||
		!strings.Contains(err.Error(), "plan-newer supersedes it") {
		t.Errorf("1886: a walk lost to silence was resumed under a newer open plan of its branch: %v", err)
	}
	beside := inventory.Plan{ID: "plan-beside", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a",
		Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-build-image"}},
		Modules: map[string]*inventory.PlanModule{"mesh-build-image": {State: "asked"}}}
	if err := retryRefusal(p, []inventory.Plan{p, beside}, ownPathOf(nil)); err == nil ||
		!strings.Contains(err.Error(), "one walk at a time") {
		t.Errorf("1886: a walk lost to silence was resumed beside a related open plan: %v", err)
	}
}