259 lines
13 KiB
Go
259 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336
|
|
// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge,
|
|
// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still
|
|
// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge
|
|
// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of
|
|
// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is
|
|
// still undelivered.
|
|
//
|
|
// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the
|
|
// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge.
|
|
|
|
const (
|
|
g747 = "fix/a-module-moves-only-when-its-program-changes"
|
|
ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog"
|
|
head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452"
|
|
head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6"
|
|
head301, merge301 = "d94764f00168471f", "2edef5a00ae05605"
|
|
merge307 = "0f76b6e2aaaaaaaa"
|
|
)
|
|
|
|
var (
|
|
at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC)
|
|
id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301)
|
|
unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"}
|
|
)
|
|
|
|
// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df.
|
|
func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) }
|
|
|
|
// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged.
|
|
func night747With(t *testing.T, contained bool, built []string) *world {
|
|
t.Helper()
|
|
return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built)
|
|
}
|
|
|
|
// night747Over is night747With with the forge set as given: one that is not told the pair cannot say.
|
|
func night747Over(t *testing.T, set func(*fakeForge), built []string) *world {
|
|
t.Helper()
|
|
at := at747
|
|
failedAt := at.Add(11 * time.Minute)
|
|
store := newMemStore()
|
|
moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747
|
|
if built != nil {
|
|
moved339, unchanged339 = built, map[string]string{}
|
|
for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} {
|
|
if !containsString(built, m) {
|
|
unchanged339[m] = "its build source holds none of the files #339 changed"
|
|
}
|
|
}
|
|
}
|
|
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
|
|
State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
|
|
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"},
|
|
Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"),
|
|
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true,
|
|
Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2,
|
|
Unchanged: unchangedRouteProxy747},
|
|
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
|
|
Why: "its walk failed, carrying its own commit"}}}
|
|
m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726",
|
|
State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute),
|
|
AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"),
|
|
// Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so.
|
|
Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true,
|
|
Moved: moved339, Unchanged: unchanged339,
|
|
Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2},
|
|
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
|
|
Why: "its walk failed, carrying its own commit"}}}
|
|
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
|
|
State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
|
|
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"),
|
|
Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner,
|
|
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
|
|
for _, d := range []*Delivery{m336, m339, m301} {
|
|
if err := store.PutDelivery(d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed",
|
|
OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
|
|
Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}}
|
|
if err := store.PutGroup(g); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctlr := newFakeController()
|
|
// The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word.
|
|
ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
|
|
State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}},
|
|
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
|
|
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}})
|
|
ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute)
|
|
forge := newFakeForge()
|
|
set(forge)
|
|
w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute))
|
|
if err := w.h.Reconcile(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.settleAll()
|
|
return w
|
|
}
|
|
|
|
func groupState747(w *world) (string, string) {
|
|
g := w.h.groups[g747]
|
|
return GroupState(g, w.h.membersOf(g))
|
|
}
|
|
|
|
// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the
|
|
// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged.
|
|
// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose
|
|
// own walk was resumed to done, is covered by it too.
|
|
func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) {
|
|
w := night747(t, true)
|
|
d := w.delivery(id336)
|
|
if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") ||
|
|
!strings.Contains(desc, "plan-339") {
|
|
t.Fatalf("#336's status reads %s %q", state, desc)
|
|
}
|
|
if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") {
|
|
t.Fatalf("the forge was told %q of #336", s)
|
|
}
|
|
if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") {
|
|
t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc)
|
|
}
|
|
if state, why := groupState747(w); state == "failed" || state == "stopped" {
|
|
t.Fatalf("the group reads %s: %s", state, why)
|
|
}
|
|
// Told once, and kept: a restart tells nothing again.
|
|
notes := len(w.forge.notesOn(merge336))
|
|
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
|
|
again.settleAll()
|
|
if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" {
|
|
t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes)
|
|
}
|
|
}
|
|
|
|
// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone,
|
|
// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the
|
|
// group waits for that walk to deliver the catalogue member's change.
|
|
func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) {
|
|
w := night747(t, true)
|
|
if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") {
|
|
t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
|
}
|
|
if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 {
|
|
t.Fatalf("the catalogue's walk is held back for %v", stopped)
|
|
}
|
|
if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) {
|
|
t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why)
|
|
}
|
|
// Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is
|
|
// delivered.
|
|
w.forge.contains[merge301+" in "+merge307] = true
|
|
r := w.ctl.walk("plan-cat")
|
|
sent := w.now
|
|
r.Modules = map[string]*WalkModule{}
|
|
for _, m := range []string{"build-agent", "route-proxy", "gitea"} {
|
|
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
|
|
}
|
|
r.State = walkDone
|
|
w.h.WalkMoved(w.ctl.put(r))
|
|
w.later(time.Minute)
|
|
w.settleAll()
|
|
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") {
|
|
t.Fatalf("the catalogue member reads %s %q", state, desc)
|
|
}
|
|
if state, why := groupState747(w); state != "delivered" {
|
|
t.Fatalf("the group reads %s: %s", state, why)
|
|
}
|
|
}
|
|
|
|
// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change
|
|
// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries
|
|
// the stopped member's merge is asked to leave it out.
|
|
func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) {
|
|
w := night747(t, false)
|
|
d := w.delivery(id336)
|
|
if state, desc := statusOf(d); d.State != Failed || state != "failure" {
|
|
t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc)
|
|
}
|
|
if state, _ := groupState747(w); state != "failed" {
|
|
t.Fatalf("the group reads %s", state)
|
|
}
|
|
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
|
|
t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
|
|
}
|
|
}
|
|
|
|
// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk
|
|
// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only
|
|
// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue.
|
|
func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) {
|
|
w := night747With(t, true, []string{"mesh-controller"})
|
|
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
|
|
t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc)
|
|
}
|
|
if state, _ := groupState747(w); state != "failed" {
|
|
t.Fatalf("the group reads %s", state)
|
|
}
|
|
if len(w.ctl.withouts) == 0 {
|
|
t.Fatal("the catalogue's walk was not asked to leave the stopped merge out")
|
|
}
|
|
}
|
|
|
|
// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on
|
|
// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the
|
|
// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk
|
|
// is asked to leave the stopped merge out, as with no later walk at all.
|
|
func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) {
|
|
for _, c := range []struct {
|
|
name string
|
|
set func(*fakeForge)
|
|
}{
|
|
{"erroring", func(f *fakeForge) { f.containsDown = true }},
|
|
{"not answering for the pair", func(*fakeForge) {}},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
w := night747Over(t, c.set, nil)
|
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
|
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
|
}
|
|
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
|
|
t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc)
|
|
}
|
|
said := ""
|
|
for _, l := range w.h.WalksWaiting() {
|
|
if l.Walk == "plan-cat" {
|
|
said += l.Says
|
|
}
|
|
}
|
|
w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it
|
|
w.settleAll()
|
|
for _, l := range w.h.Stalled() {
|
|
if l.Walk == "plan-cat" {
|
|
said += " | " + l.Says
|
|
}
|
|
}
|
|
if strings.Count(said, "the forge is asked since") != 2 {
|
|
t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said)
|
|
}
|
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
|
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
|
}
|
|
w.later(coverAskBound + time.Minute)
|
|
w.settleAll()
|
|
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
|
|
t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
|
|
}
|
|
})
|
|
}
|
|
}
|