mesh-catalog/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go

package main

import (
	"strings"
	"testing"
	"time"
)

// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336
// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge,
// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still
// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge
// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of
// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is
// still undelivered.
//
// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the
// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge.

const (
	g747              = "fix/a-module-moves-only-when-its-program-changes"
	ctl747, cat747    = "novox/mesh-controller", "novox/mesh-catalog"
	head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452"
	head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6"
	head301, merge301 = "d94764f00168471f", "2edef5a00ae05605"
	merge307          = "0f76b6e2aaaaaaaa"
)

var (
	at747                  = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC)
	id336, id339, id301    = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301)
	unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"}
)

// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df.
func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) }

// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged.
func night747With(t *testing.T, contained bool, built []string) *world {
	t.Helper()
	return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built)
}

// night747Over is night747With with the forge set as given: one that is not told the pair cannot say.
func night747Over(t *testing.T, set func(*fakeForge), built []string) *world {
	t.Helper()
	at := at747
	failedAt := at.Add(11 * time.Minute)
	store := newMemStore()
	moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747
	if built != nil {
		moved339, unchanged339 = built, map[string]string{}
		for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} {
			if !containsString(built, m) {
				unchanged339[m] = "its build source holds none of the files #339 changed"
			}
		}
	}
	m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
		State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
		AnsweredBy: merge336, Check: &Verdict{Gate: "pass"},
		Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"),
		Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true,
			Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2,
			Unchanged: unchangedRouteProxy747},
		Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
			Why: "its walk failed, carrying its own commit"}}}
	m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726",
		State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute),
		AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"),
		// Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so.
		Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true,
			Moved: moved339, Unchanged: unchanged339,
			Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2},
		Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
			Why: "its walk failed, carrying its own commit"}}}
	m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
		State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
		Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"),
		Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner,
			Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
	for _, d := range []*Delivery{m336, m339, m301} {
		if err := store.PutDelivery(d); err != nil {
			t.Fatal(err)
		}
	}
	g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed",
		OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
			Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}}
	if err := store.PutGroup(g); err != nil {
		t.Fatal(err)
	}
	ctlr := newFakeController()
	// The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word.
	ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
		State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}},
		Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
			{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}})
	ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute)
	forge := newFakeForge()
	set(forge)
	w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute))
	if err := w.h.Reconcile(); err != nil {
		t.Fatal(err)
	}
	w.settleAll()
	return w
}

func groupState747(w *world) (string, string) {
	g := w.h.groups[g747]
	return GroupState(g, w.h.membersOf(g))
}

// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the
// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged.
// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose
// own walk was resumed to done, is covered by it too.
func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) {
	w := night747(t, true)
	d := w.delivery(id336)
	if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") ||
		!strings.Contains(desc, "plan-339") {
		t.Fatalf("#336's status reads %s %q", state, desc)
	}
	if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") {
		t.Fatalf("the forge was told %q of #336", s)
	}
	if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") {
		t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc)
	}
	if state, why := groupState747(w); state == "failed" || state == "stopped" {
		t.Fatalf("the group reads %s: %s", state, why)
	}
	// Told once, and kept: a restart tells nothing again.
	notes := len(w.forge.notesOn(merge336))
	again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
	again.settleAll()
	if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" {
		t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes)
	}
}

// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone,
// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the
// group waits for that walk to deliver the catalogue member's change.
func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) {
	w := night747(t, true)
	if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") {
		t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
	}
	if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 {
		t.Fatalf("the catalogue's walk is held back for %v", stopped)
	}
	if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) {
		t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why)
	}
	// Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is
	// delivered.
	w.forge.contains[merge301+" in "+merge307] = true
	r := w.ctl.walk("plan-cat")
	sent := w.now
	r.Modules = map[string]*WalkModule{}
	for _, m := range []string{"build-agent", "route-proxy", "gitea"} {
		r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
	}
	r.State = walkDone
	w.h.WalkMoved(w.ctl.put(r))
	w.later(time.Minute)
	w.settleAll()
	if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") {
		t.Fatalf("the catalogue member reads %s %q", state, desc)
	}
	if state, why := groupState747(w); state != "delivered" {
		t.Fatalf("the group reads %s: %s", state, why)
	}
}

// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change
// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries
// the stopped member's merge is asked to leave it out.
func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) {
	w := night747(t, false)
	d := w.delivery(id336)
	if state, desc := statusOf(d); d.State != Failed || state != "failure" {
		t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc)
	}
	if state, _ := groupState747(w); state != "failed" {
		t.Fatalf("the group reads %s", state)
	}
	if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
		t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
	}
}

// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk
// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only
// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue.
func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) {
	w := night747With(t, true, []string{"mesh-controller"})
	if state, desc := statusOf(w.delivery(id336)); state != "failure" {
		t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc)
	}
	if state, _ := groupState747(w); state != "failed" {
		t.Fatalf("the group reads %s", state)
	}
	if len(w.ctl.withouts) == 0 {
		t.Fatal("the catalogue's walk was not asked to leave the stopped merge out")
	}
}

// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on
// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the
// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk
// is asked to leave the stopped merge out, as with no later walk at all.
func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) {
	for _, c := range []struct {
		name string
		set  func(*fakeForge)
	}{
		{"erroring", func(f *fakeForge) { f.containsDown = true }},
		{"not answering for the pair", func(*fakeForge) {}},
	} {
		t.Run(c.name, func(t *testing.T) {
			w := night747Over(t, c.set, nil)
			if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
				t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
			}
			if state, desc := statusOf(w.delivery(id336)); state != "failure" {
				t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc)
			}
			said := ""
			for _, l := range w.h.WalksWaiting() {
				if l.Walk == "plan-cat" {
					said += l.Says
				}
			}
			w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it
			w.settleAll()
			for _, l := range w.h.Stalled() {
				if l.Walk == "plan-cat" {
					said += " | " + l.Says
				}
			}
			if strings.Count(said, "the forge is asked since") != 2 {
				t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said)
			}
			if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
				t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
			}
			w.later(coverAskBound + time.Minute)
			w.settleAll()
			if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
				t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
			}
		})
	}
}