mesh-controller/cmd/mesh-controller/catch_up_integration_test.go

//go:build integration

package main

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/link"
)

// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear.

// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the
// router holds an operator's Approve for.
type aTrunk struct {
	merges   []trunkMerge
	module   map[string]string
	approved map[string]bool
	read     []string // the trunks asked of the forge, as owner/repo
	events   int      // merges read in full from the forge
	silent   bool     // the forge does not answer
}

func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk {
	t.Helper()
	tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}}
	for i, m := range modules {
		c := fmt.Sprintf("%s-c%02d", repo, i+1)
		tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1),
			Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c),
			At:      t0.Add(time.Duration(i) * time.Minute)})
		tr.module[c] = m
		tr.approved[c] = true
	}
	was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant
	trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) {
		tr.read = append(tr.read, owner+"/"+name)
		if tr.silent {
			return nil, errors.New("the forge did not answer")
		}
		if page > 1 || owner+"/"+name != "novox/"+repo {
			return nil, nil
		}
		var newest []trunkCommit
		for i := len(tr.merges) - 1; i >= 0; i-- {
			m := tr.merges[i]
			newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At})
		}
		return newest, nil
	}
	readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
		tr.events++
		m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At)
		m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head
		_, m.Number, _ = pullOfSubject(tm.Subject)
		return m, nil
	}
	findMergeWarrant = func(_ context.Context, _ catalogueReader, _, _ string, number int, head string) (string, bool, string, error) {
		for _, m := range tr.merges {
			if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] {
				return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil
			}
		}
		return "", false, "the router holds no chosen merge ask for the pull request and head", nil
	}
	wasBeside := catchUpBeside
	catchUpBeside = func(run func()) { run() }
	t.Cleanup(func() {
		trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside
		settledMerges.Clear()
	})
	return tr
}

// aKeptMergeBefore is a merge the controller heard before the merges of a test: where its record begins, so what a trunk holds from
// then on can be a gap (novox/hq ADR 0463 §2).
func aKeptMergeBefore(t *testing.T, open *stores) {
	t.Helper()
	hear(t, open, catalogueMerge("mesh-catalog-seed", "app", t0.Add(-time.Hour)), t0.Add(-time.Hour))
}

func mergeKnown(t *testing.T, open *stores, commit string) bool {
	t.Helper()
	_, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit)
	if err != nil {
		t.Fatal(err)
	}
	return known
}

// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said
// as caught up; a second start plans nothing new.
func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
	hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second))
	hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second))
	if mergeKnown(t, open, "mesh-catalog-c02") {
		t.Fatal("the merge to be caught up is known before the catch-up")
	}
	if (following{open: open}).catchUp(t.Context(), nil) {
		t.Fatal("the catch-up says a trunk was left for later")
	}
	if !mergeKnown(t, open, "mesh-catalog-c02") {
		t.Fatal("the merge the controller did not hear was not planned")
	}
	kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02")
	if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") {
		t.Fatalf("the merge does not say it was caught up: %q", said)
	}
	cutAt(t, open, t0.Add(10*time.Minute))
	ws, _ := walks(t, open)
	if len(ws) == 0 {
		t.Fatal("no walk answers the caught-up merge")
	}
	if _, in := ws[0].Modules["notes"]; !in {
		t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules)
	}
	before := tr.events
	if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before {
		t.Fatalf("a second start read %d merge(s) in full again", tr.events-before)
	}
}

// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the
// earlier first, and the forge is read for that repository's trunk only.
func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
	var order []string
	inner := readMergeEvent
	readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
		order = append(order, tm.Commit)
		return inner(ctx, owner, name, branch, tm)
	}
	t.Cleanup(func() { readMergeEvent = inner })
	tr.read = nil
	heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute))
	if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil {
		t.Fatal(err)
	}
	for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} {
		if !mergeKnown(t, open, c) {
			t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c)
		}
	}
	if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" {
		t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order)
	}
	if len(tr.read) == 0 {
		t.Fatal("the forge was not read for the trunk of the merge heard")
	}
	for _, r := range tr.read {
		if r != "novox/mesh-catalog" {
			t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read)
		}
	}
}

// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window.
func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	modules := make([]string, 51)
	for i := range modules {
		modules[i] = "app"
	}
	aTrunkOf(t, "mesh-catalog", modules...)
	// Fifty are known; the oldest is not.
	for i := 2; i <= 51; i++ {
		c := fmt.Sprintf("mesh-catalog-c%02d", i)
		hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second))
	}
	// Paged as the forge pages: fifty a page, and a trunk of another repository is empty (a short page, or none).
	pages := trunkCommitsPage
	trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) {
		all, err := pages(ctx, owner, name, branch, 1)
		if err != nil || page < 1 || (page-1)*50 >= len(all) {
			return nil, err
		}
		return all[(page-1)*50 : min(page*50, len(all))], nil
	}
	t.Cleanup(func() { trunkCommitsPage = pages })
	(following{open: open}).catchUp(t.Context(), nil)
	if mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatal("the oldest of fifty-one merges was read and planned, outside the window")
	}
}

// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and
// commit; a second start leaves that one condition open, observed twice; planning the gap clears it.
func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	aKeptMergeBefore(t, open)
	keeper, _ := withConditionsInMemory(t)
	tr := aTrunkOf(t, "mesh-catalog", "notes")
	tr.approved["mesh-catalog-c01"] = false
	catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
	catch()
	if mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatal("a merge with no Approve on record was planned")
	}
	raised := func() []conditions.Condition {
		t.Helper()
		all, err := keeper.Open(t.Context())
		if err != nil {
			t.Fatal(err)
		}
		var out []conditions.Condition
		for _, c := range all {
			if c.Kind == kindMergeNotHeard {
				out = append(out, c)
			}
		}
		return out
	}
	one := raised()
	if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") {
		t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one)
	}
	catch()
	if again := raised(); len(again) != 1 || again[0].Observations < 2 {
		t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again)
	}
	tr.approved["mesh-catalog-c01"] = true
	catch()
	if !mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatal("the gap was not planned once its Approve was on record")
	}
	if left := raised(); len(left) != 0 {
		t.Fatalf("planning the gap left its condition open: %+v", left)
	}
}

// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it.
func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	aKeptMergeBefore(t, open)
	tr := aTrunkOf(t, "mesh-catalog", "notes")
	tr.silent = true
	if later := (following{open: open}).catchUp(t.Context(), nil); !later {
		t.Fatal("a forge that did not answer counted as nothing to catch up")
	}
	if mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatal("a merge was planned without the forge's word")
	}
	tr.silent = false
	if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatalf("the next start did not catch up (left for later: %v)", later)
	}
}

// Finding 2 of the review: a gap that adds a module is a gap, judged as hearMerge judges it. With the operator's Approve on record it
// is heard (hearMerge asks the new module in its walk); without, it raises the condition and is not lost in silence.
func TestAGapThatAddsAModuleIsHeardOnItsWarrantAndRaisedWithout(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	aKeptMergeBefore(t, open)
	keeper, _ := withConditionsInMemory(t)
	tr := aTrunkOf(t, "mesh-catalog", "newmodule")
	tr.approved["mesh-catalog-c01"] = false
	catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
	catch()
	if mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatal("a merge adding a module was heard with no Approve on record")
	}
	all, err := keeper.Open(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	var raised int
	for _, c := range all {
		if c.Kind == kindMergeNotHeard {
			raised++
		}
	}
	if raised != 1 {
		t.Fatalf("a merge adding a module with no Approve raised %d condition(s), not one: it was lost in silence", raised)
	}
	tr.approved["mesh-catalog-c01"] = true
	catch()
	if !mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatal("a merge adding a module was not heard once its Approve was on record")
	}
}

// A trunk with merges from before the controller began keeping merges, and merges since: only the later ones are gaps.
func TestAMergeFromBeforeTheRecordBeganIsHistoryAndNoCondition(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	keeper, _ := withConditionsInMemory(t)
	tr := aTrunkOf(t, "mesh-catalog", "notes", "notes", "notes")
	tr.approved["mesh-catalog-c01"], tr.approved["mesh-catalog-c02"], tr.approved["mesh-catalog-c03"] = false, false, false
	// The record begins at the second merge: the first is two days older.
	tr.merges[0].At = t0.Add(-48 * time.Hour)
	hear(t, open, catalogueMerge("mesh-catalog-c02", "notes", t0.Add(time.Minute)), t0.Add(time.Minute))
	delete(tr.approved, "mesh-catalog-c02")
	(following{open: open}).catchUp(t.Context(), nil)
	// The first merge (two days before the record) was not even read from the forge: only the third, which is unknown, was.
	if tr.events != 1 {
		t.Fatalf("%d merge(s) were read from the forge: the one from before the record is history and is not read", tr.events)
	}
	all, err := keeper.Open(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	var raised int
	for _, c := range all {
		if c.Kind == kindMergeNotHeard {
			raised++
		}
	}
	if raised != 1 {
		t.Fatalf("%d condition(s) raised: only the merge since the record began (the third) is a gap", raised)
	}
}

// novox/hq ADR 0463 §2: the record begins at the earliest merge kept on ANY trunk. A trunk the controller kept no merge of has the
// merges made since as gaps (mesh-delivery #1 on novox), and the merges made before as history.
func TestATrunkWithNoKeptMergeHasItsLaterMergesAsGapsAndItsEarlierOnesAsHistory(t *testing.T) {
	open := windowed(t)
	everyHeadWaits(t)
	asksRecorded(t)
	// The record begins on another trunk, at t0 less an hour.
	hear(t, open, repoMerge("one", "c-one", t0.Add(-time.Hour)), t0.Add(-time.Hour))
	tr := aTrunkOf(t, "mesh-catalog", "notes", "notes")
	tr.merges[0].At = t0.Add(-72 * time.Hour) // before the record: history
	(following{open: open}).catchUp(t.Context(), nil)
	if mergeKnown(t, open, "mesh-catalog-c01") {
		t.Fatal("a merge from before the record began was heard")
	}
	if !mergeKnown(t, open, "mesh-catalog-c02") {
		t.Fatal("a merge since the record began, on a trunk with no merge kept, was not caught up")
	}
}