package main import ( "encoding/json" "errors" "fmt" "reflect" "slices" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/inventory" ) // novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one // group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges // to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them // — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it. // endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and // the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS" // after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the // test never ran. func endsItsLine(t *testing.T) { t.Cleanup(func() { fmt.Println() }) } // leaved is a kept merge of a repository, made at a moment. func leaved(repository, commit string, made time.Time) inventory.BatchedMerge { return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made} } // The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it // contains; an earlier merge of that repository and every other repository's merge are walked again. func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) { c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0) c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second)) c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second)) o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second)) again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"}) if err != nil { t.Fatal(err) } var walked []string for _, m := range again { walked = append(walked, m.Commit) } if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) { t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked) } want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit}, {Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}} if !reflect.DeepEqual(out, want) { t.Fatalf("left out: %+v; want %+v", out, want) } // A merge the walk does not answer is refused, naming it: nothing ends. if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil || !strings.Contains(err.Error(), "ffffffffffff") { t.Fatalf("a merge the walk does not answer was left out: %v", err) } if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil { t.Fatal("a merge without its commit was taken") } } // moving is a kept merge that moved modules when it was heard. func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge { m.Event, _ = json.Marshal(keptMerge{Moves: modules}) return m } // A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules // it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until // none more is: a merge sharing a module with that one is left out as well. func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) { stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app") reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one") next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one") free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three") again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"}) if err != nil { t.Fatal(err) } if len(again) != 1 || again[0].Commit != free.Commit { t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again) } want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit}, {Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}}, {Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}} if !reflect.DeepEqual(out, want) { t.Fatalf("left out %+v; want %+v", out, want) } } // A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its // own batch moves is built, and empty tiers go. func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) { plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}}, Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}} got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"}) if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) { t.Fatalf("withheld %v, tiers %v", got, plan.Tiers) } if _, in := plan.Modules["app"]; in { t.Fatal("a withheld module is still in the plan") } } // The seat verb passes the merges to leave out to the command. func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) { got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen", "without": "novox/mesh-lab@b1,novox/mesh-lab@b2"}) want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen", "--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"} if err != nil || !reflect.DeepEqual(got, want) { t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want) } } // A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped, // names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk // that builds the other repository's module alone. func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { open := windowed(t) endsItsLine(t) asksRecorded(t) ctx := t.Context() cat := catalogueMerge("cat1aaaaaaaa", "app", t0) one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) hear(t, open, cat, t0.Add(time.Second)) hear(t, open, one, t0.Add(6*time.Second)) cutAt(t, open, t0.Add(2*time.Minute)) ws, _ := walks(t, open) if len(ws) != 1 || !ws[0].Open() { t.Fatalf("no open walk to stop: %+v", ws) } stopped := ws[0] p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) if err != nil { t.Fatal(err) } if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) } if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { t.Fatalf("the stop answered %+v", said) } again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) if err != nil || len(again) != 1 || again[0].Commit != one.Commit { t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) } if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) } cutAt(t, open, t0.Add(4*time.Minute)) fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) if err != nil || fresh.Batch() || !fresh.Open() { t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) } if _, in := fresh.Modules["one"]; !in { t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) } if _, in := fresh.Modules["app"]; in { t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) } if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) } // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). cutAt(t, open, t0.Add(5*time.Minute)) if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) } } // A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it // contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only // ended. A merge the walk does not answer is refused, and nothing ends. func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { open := windowed(t) endsItsLine(t) asksRecorded(t) ctx := t.Context() first := catalogueMerge("cat1aaaaaaaa", "app", t0) later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) hear(t, open, first, t0.Add(time.Second)) hear(t, open, one, t0.Add(6*time.Second)) hear(t, open, later, t0.Add(11*time.Second)) cutAt(t, open, t0.Add(2*time.Minute)) ws, _ := walks(t, open) walk := ws[0] if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { t.Fatal("a stop leaving out a merge the walk does not answer was taken") } if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { t.Fatalf("a refused stop ended the walk: %s", p.State) } p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) if err != nil { t.Fatal(err) } want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) } if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) } again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) if len(again) != 1 || again[0].Commit != one.Commit { t.Fatalf("walked again: %+v; want the other repository's merge alone", again) } // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. cutAt(t, open, t0.Add(4*time.Minute)) fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) } if _, bs := walks(t, open); len(bs) != 0 { t.Fatalf("a batch of nothing was made: %+v", bs) } } // A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge // walked again joins that batch, never a batch of its own that would walk the older commit after the newer one // (review of #232): its window closes, and its walk is at the newer commit, answering both. func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { open := windowed(t) endsItsLine(t) asksRecorded(t) ctx := t.Context() cat := catalogueMerge("cat1aaaaaaaa", "app", t0) one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) hear(t, open, cat, t0.Add(time.Second)) hear(t, open, one1, t0.Add(6*time.Second)) cutAt(t, open, t0.Add(2*time.Minute)) ws, _ := walks(t, open) walk := ws[0] one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) hear(t, open, one2, t0.Add(3*time.Minute)) _, bs := walks(t, open) if len(bs) != 1 { t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) } waiting := bs[0] p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) if err != nil { t.Fatal(err) } if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) } if _, bs := walks(t, open); len(bs) != 1 { t.Fatalf("%d batches; want the open one alone", len(bs)) } b, _ := open.inventory.PlanByID(ctx, waiting.ID) if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) } // Its window closed by the stop, not by the clock: cut at once. cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) } var answered []string for _, m := range fresh.Delivery.Merges { answered = append(answered, m.Commit+">"+m.Carried) } if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) } } // A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of // the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk // that walked its merges again is refused a retry, saying why. func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { open := windowed(t) endsItsLine(t) asksRecorded(t) ctx := t.Context() cat := catalogueMerge("cat1aaaaaaaa", "app", t0) one1 := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) hear(t, open, cat, t0.Add(time.Second)) hear(t, open, one1, t0.Add(6*time.Second)) cutAt(t, open, t0.Add(2*time.Minute)) ws, _ := walks(t, open) walk := ws[0] sent := t0.Add(150 * time.Second) for name, m := range walk.Modules { if name == "app" || name == "one" { m.First, m.FirstAt = []string{"anchor"}, &sent } } if err := open.inventory.SavePlan(ctx, &walk); err != nil { t.Fatal(err) } cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) hear(t, open, cat2, t0.Add(3*time.Minute)) _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) } p, _ := open.inventory.PlanByID(ctx, walk.ID) if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) } if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) } if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) } } // A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on // with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped. func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { open := windowed(t) endsItsLine(t) asksRecorded(t) ctx := t.Context() cat := catalogueMerge("cat1aaaaaaaa", "app", t0) one := repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)) hear(t, open, cat, t0.Add(time.Second)) hear(t, open, one, t0.Add(6*time.Second)) _, bs := walks(t, open) b := bs[0] r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) if err != nil { t.Fatal(err) } if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) } if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { t.Fatalf("the stopped record holds %+v", left) } if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) } _ = r cutAt(t, open, t0.Add(3*time.Minute)) w, _ := open.inventory.PlanByID(ctx, b.ID) if _, in := w.Modules["app"]; in || w.Batch() { t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) } // A batch whose every merge is left out is stopped itself. hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) _, bs = walks(t, open) last := bs[0] if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { t.Fatalf("a batch left with nothing: %+v %v", said, err) } if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { t.Fatalf("the emptied batch is %s", p.State) } } // The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody // else since it was read refuses all of it, and no merge moves, no plan is written. func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { open := windowed(t) endsItsLine(t) ctx := t.Context() hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) _, bs := walks(t, open) stale := bs[0] moved := stale moved.Note = "written by somebody else" if err := open.inventory.SavePlan(ctx, &moved); err != nil { t.Fatal(err) } fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) if !errors.Is(err, inventory.ErrPlanMoved) { t.Fatalf("a stale plan in the act: %v", err) } if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { t.Fatal("the new plan was written though the act was refused") } if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { t.Fatalf("the merge moved to %q though the act was refused", m.Plan) } } // What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent // with no first machine reached every machine running it; one sent first reached those, and the rest it was sent // to; a module a merge walked again moves is sent again, and is not named. func TestKeptNamesEveryMachineASendReached(t *testing.T) { at := t0 p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ "together": {SentAt: &at}, "first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}}, "again": {SentAt: &at}, "unsent": {}, }} got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}}) want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}}, {Module: "together", Machines: []string{"anchor", "laptop"}}} if !reflect.DeepEqual(got, want) { t.Fatalf("kept %+v; want %+v", got, want) } }