diff --git a/cmd/mesh-controller/catch_up_integration_test.go b/cmd/mesh-controller/catch_up_integration_test.go new file mode 100644 index 00000000..701dbb53 --- /dev/null +++ b/cmd/mesh-controller/catch_up_integration_test.go @@ -0,0 +1,260 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear. + +// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the +// router holds an operator's Approve for. +type aTrunk struct { + merges []trunkMerge + module map[string]string + approved map[string]bool + read []string // the trunks asked of the forge, as owner/repo + events int // merges read in full from the forge + silent bool // the forge does not answer +} + +func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk { + t.Helper() + tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}} + for i, m := range modules { + c := fmt.Sprintf("%s-c%02d", repo, i+1) + tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1), + Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c), + At: t0.Add(time.Duration(i) * time.Minute)}) + tr.module[c] = m + tr.approved[c] = true + } + was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant + trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) { + tr.read = append(tr.read, owner+"/"+name) + if tr.silent { + return nil, errors.New("the forge did not answer") + } + if page > 1 || owner+"/"+name != "novox/"+repo { + return nil, nil + } + var newest []trunkCommit + for i := len(tr.merges) - 1; i >= 0; i-- { + m := tr.merges[i] + newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At}) + } + return newest, nil + } + readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) { + tr.events++ + m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At) + m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head + _, m.Number, _ = pullOfSubject(tm.Subject) + return m, nil + } + findMergeWarrant = func(_ context.Context, _ catalogueReader, number int, head string) (string, bool, string, error) { + for _, m := range tr.merges { + if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] { + return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil + } + } + return "", false, "the router holds no chosen merge ask for the pull request and head", nil + } + wasBeside := catchUpBeside + catchUpBeside = func(run func()) { run() } + t.Cleanup(func() { + trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside + settledMerges.Clear() + }) + return tr +} + +func mergeKnown(t *testing.T, open *stores, commit string) bool { + t.Helper() + _, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit) + if err != nil { + t.Fatal(err) + } + return known +} + +// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said +// as caught up; a second start plans nothing new. +func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksRecorded(t) + tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app") + hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second)) + hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second)) + if mergeKnown(t, open, "mesh-catalog-c02") { + t.Fatal("the merge to be caught up is known before the catch-up") + } + if (following{open: open}).catchUp(t.Context(), nil) { + t.Fatal("the catch-up says a trunk was left for later") + } + if !mergeKnown(t, open, "mesh-catalog-c02") { + t.Fatal("the merge the controller did not hear was not planned") + } + kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02") + if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") { + t.Fatalf("the merge does not say it was caught up: %q", said) + } + cutAt(t, open, t0.Add(10*time.Minute)) + ws, _ := walks(t, open) + if len(ws) == 0 { + t.Fatal("no walk answers the caught-up merge") + } + if _, in := ws[0].Modules["notes"]; !in { + t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules) + } + before := tr.events + if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before { + t.Fatalf("a second start read %d merge(s) in full again", tr.events-before) + } +} + +// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the +// earlier first, and the forge is read for that repository's trunk only. +func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksRecorded(t) + tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app") + var order []string + inner := readMergeEvent + readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) { + order = append(order, tm.Commit) + return inner(ctx, owner, name, branch, tm) + } + t.Cleanup(func() { readMergeEvent = inner }) + tr.read = nil + heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)) + if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil { + t.Fatal(err) + } + for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} { + if !mergeKnown(t, open, c) { + t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c) + } + } + if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" { + t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order) + } + if len(tr.read) == 0 { + t.Fatal("the forge was not read for the trunk of the merge heard") + } + for _, r := range tr.read { + if r != "novox/mesh-catalog" { + t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read) + } + } +} + +// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window. +func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksRecorded(t) + modules := make([]string, 51) + for i := range modules { + modules[i] = "app" + } + aTrunkOf(t, "mesh-catalog", modules...) + // Fifty are known; the oldest is not. + for i := 2; i <= 51; i++ { + c := fmt.Sprintf("mesh-catalog-c%02d", i) + hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second)) + } + // Paged as the forge pages: fifty a page. + pages := trunkCommitsPage + trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) { + all, err := pages(ctx, owner, name, branch, 1) + if err != nil || page > 2 { + return nil, err + } + if page == 1 { + return all[:50], nil + } + return all[50:], nil + } + t.Cleanup(func() { trunkCommitsPage = pages }) + (following{open: open}).catchUp(t.Context(), nil) + if mergeKnown(t, open, "mesh-catalog-c01") { + t.Fatal("the oldest of fifty-one merges was read and planned, outside the window") + } +} + +// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and +// commit; a second start leaves that one condition open, observed twice; planning the gap clears it. +func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksRecorded(t) + keeper, _ := withConditionsInMemory(t) + tr := aTrunkOf(t, "mesh-catalog", "notes") + tr.approved["mesh-catalog-c01"] = false + catch := func() { (following{open: open}).catchUp(t.Context(), nil) } + catch() + if mergeKnown(t, open, "mesh-catalog-c01") { + t.Fatal("a merge with no Approve on record was planned") + } + raised := func() []conditions.Condition { + t.Helper() + all, err := keeper.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + var out []conditions.Condition + for _, c := range all { + if c.Kind == kindMergeNotHeard { + out = append(out, c) + } + } + return out + } + one := raised() + if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") { + t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one) + } + catch() + if again := raised(); len(again) != 1 || again[0].Observations < 2 { + t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again) + } + tr.approved["mesh-catalog-c01"] = true + catch() + if !mergeKnown(t, open, "mesh-catalog-c01") { + t.Fatal("the gap was not planned once its Approve was on record") + } + if left := raised(); len(left) != 0 { + t.Fatalf("planning the gap left its condition open: %+v", left) + } +} + +// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it. +func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksRecorded(t) + tr := aTrunkOf(t, "mesh-catalog", "notes") + tr.silent = true + if later := (following{open: open}).catchUp(t.Context(), nil); !later { + t.Fatal("a forge that did not answer counted as nothing to catch up") + } + if mergeKnown(t, open, "mesh-catalog-c01") { + t.Fatal("a merge was planned without the forge's word") + } + tr.silent = false + if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") { + t.Fatalf("the next start did not catch up (left for later: %v)", later) + } +} diff --git a/cmd/mesh-controller/catch_up_test.go b/cmd/mesh-controller/catch_up_test.go new file mode 100644 index 00000000..9b5b1c1b --- /dev/null +++ b/cmd/mesh-controller/catch_up_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/asks" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The pieces of the catch-up that need no store (novox/hq ADR 0457, issue 1336). + +func TestAMergeCommitsSubjectNamesItsPullRequest(t *testing.T) { + title, number, ok := pullOfSubject("Merge pull request 'Fix keycloak's sign-in (issue 1332)' (#378) from fix/1332-sign-in into main") + if !ok || number != 378 || title != "Fix keycloak's sign-in (issue 1332)" { + t.Fatalf("read %q %d %v", title, number, ok) + } + if b := headBranchOfSubject("Merge pull request 'x' (#378) from fix/1332-sign-in into main"); b != "fix/1332-sign-in" { + t.Fatalf("the head branch read as %q", b) + } + if _, _, ok := pullOfSubject("Merge branch 'main' of somewhere"); ok { + t.Fatal("a merge that names no pull request read as one") + } +} + +func TestOnlyTheLastFiftyMergesOfATrunkAreRead(t *testing.T) { + pages := map[int][]trunkCommit{} + var all []trunkCommit // newest first: 51 merges, a plain commit between each pair + for i := 51; i >= 1; i-- { + all = append(all, trunkCommit{Commit: fmt.Sprintf("merge-%02d", i), SecondParent: fmt.Sprintf("head-%02d", i)}, + trunkCommit{Commit: fmt.Sprintf("plain-%02d", i)}) + } + for i := 0; i < len(all); i += 50 { + pages[i/50+1] = all[i:min(i+50, len(all))] + } + was := trunkCommitsPage + trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { return pages[page], nil } + t.Cleanup(func() { trunkCommitsPage = was }) + got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main") + if err != nil { + t.Fatal(err) + } + if len(got) != catchUpWindow || got[0].Commit != "merge-02" || got[len(got)-1].Commit != "merge-51" { + t.Fatalf("read %d merges from %s to %s: the oldest of 51 is outside the window", len(got), got[0].Commit, got[len(got)-1].Commit) + } + if got[0].Head != "head-02" { + t.Fatalf("a merge's head is its second parent, not %q", got[0].Head) + } +} + +func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) { + was := trunkCommitsPage + trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { + return nil, errors.New("the forge did not answer") + } + t.Cleanup(func() { trunkCommitsPage = was }) + if got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main"); err == nil { + t.Fatalf("a forge that did not answer read as %d merges: nothing to catch up", len(got)) + } +} + +func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) { + ok := approvedMerge(t) + for _, asker := range []string{sdkdelivery.TellerModule, theDelivery} { + w := ok.warrant + w.Asker = asker + if _, err := caughtUpWarrant(asker, &ok.ask, &w); err != nil { + t.Errorf("the operator's Approve under %s was refused: %v", asker, err) + } + } + if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil { + t.Error("a warrant of an asker the controller takes none from was accepted") + } + declined := ok.warrant + declined.Option, declined.Label = "decline", "Decline" + if _, err := caughtUpWarrant(theDelivery, &ok.ask, &declined); err == nil { + t.Error("a Decline was accepted as a warrant") + } + unproved := ok.warrant + by := *unproved.By + by.Verified = "" + unproved.By = &by + if _, err := caughtUpWarrant(theDelivery, &ok.ask, &unproved); err == nil { + t.Error("an answer whose channel proves nobody was accepted") + } + if _, err := caughtUpWarrant(theDelivery, nil, nil); err == nil { + t.Error("a router that holds nothing was accepted") + } +} + +// Keycloak's merge was asked as mesh-delivery.merge-378-b9104454f9a5-1: the warrant is found under mesh-delivery as under the +// teller, by the pull request's number and the first twelve digits of its head. +func TestAWarrantAskedUnderMeshDeliveryIsFoundByThePullRequestAndItsHead(t *testing.T) { + ok := approvedMerge(t) + head := "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372" + askID := "merge-378-b9104454f9a5-1" + ask, w := ok.ask, ok.warrant + ask.ID, w.Ask, w.Asker = askID, askID, theDelivery + w.AskDigest = ask.Digest() + wasIDs, wasAsk := routerAskIDs, routerAskOf + routerAskIDs = func(_ context.Context, _ catalogueReader, asker string) ([]string, error) { + if asker == theDelivery { + return []string{"merge-377-aaaaaaaaaaaa-1", askID, "merge-378-b9104454f9a5-0"}, nil + } + return nil, nil + } + routerAskOf = func(_ context.Context, _ catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) { + if asker == theDelivery && id == askID { + return &ask, &w, nil + } + return nil, nil, nil // the router holds no closed record of the other tries + } + t.Cleanup(func() { routerAskIDs, routerAskOf = wasIDs, wasAsk }) + why, found, reason, err := findMergeWarrant(t.Context(), nil, 378, head) + if err != nil || !found || !strings.Contains(why, askID) { + t.Fatalf("the warrant was not found: %q %v %q %v", why, found, reason, err) + } + // Another pull request's head, or none, finds none and says why. + if _, found, reason, _ := findMergeWarrant(t.Context(), nil, 378, "ffffffffffff0000"); found || !strings.Contains(reason, "merge-378-ffffffffffff-") { + t.Fatalf("another head found a warrant, or said %q", reason) + } + if _, found, reason, _ := findMergeWarrant(t.Context(), nil, 0, head); found || reason == "" { + t.Fatalf("a merge that names no pull request found a warrant: %v %q", found, reason) + } + // A router that cannot be read is an error, never "no warrant". + routerAskIDs = func(context.Context, catalogueReader, string) ([]string, error) { return nil, errors.New("no answer") } + if _, found, _, err := findMergeWarrant(t.Context(), nil, 378, head); found || err == nil { + t.Fatalf("an unreadable router read as no warrant: %v %v", found, err) + } +}