//go:build integration package inventory import ( "context" "errors" "reflect" "slices" "testing" "time" "github.com/novox/mesh-controller/internal/chain" ) // A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for // it stands for what the mesh held before this release. func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) { t.Helper() inv.ExecForTest(t, `insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest, built_against, built_contexts, asked, source_fingerprint, build_sources) values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text, case when $4::text = '' then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb else '[]'::jsonb end, 'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb, '["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb, now() - interval '1 hour', 'fp-' || $1::text, ('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`, id, module, commit, failed) } func rowsInTheTable(t *testing.T, inv *Inventory, table string) string { t.Helper() rows := inv.QueryForTest(t, `select count(*) from `+table) return rows[0][0] } // **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue // 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the // bucket and written to it, the table taking no more. func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) { inv, finish := ForTestBefore(t, 105) finish() ctx := context.Background() aBuildRow(t, inv, "build-1", "notes", "c1", "") aBuildRow(t, inv, "build-2", "notes", "c2", "") aBuildRow(t, inv, "build-3", "wiki", "", "compile error") // Before the move the tables are read, and the bucket is empty. if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 { t.Fatalf("before the move the tables hold %d builds: %v", len(before), err) } if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { t.Fatalf("moved before the move: %v", moved) } reports, err := inv.MoveChain(ctx, nil) if err != nil { t.Fatal(err) } if len(reports) != 1 || reports[0].Kind != KindBuilds || reports[0].Rows != 3 || reports[0].Written != 3 || reports[0].Largest == 0 { t.Fatalf("the move reported %+v", reports) } moved, err := inv.ChainMoved(ctx) if err != nil || moved[KindBuilds] != 3 { t.Fatalf("chain_moved says %v: %v", moved, err) } kept, found, err := inv.chain.Builds.Get(ctx, "build-2") if err != nil || !found { t.Fatalf("build-2 is not in the bucket: %v %v", found, err) } if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" || v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 || v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 { t.Fatalf("build-2 was moved as %+v", v) } // From now on the bucket is what is read and written. if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil { t.Fatal(err) } if n := rowsInTheTable(t, inv, "build"); n != "3" { t.Fatalf("the table took a build after the move: %s rows", n) } after, err := inv.Builds(ctx, "notes", 10) if err != nil || len(after) != 3 { t.Fatalf("notes has %d builds on the bus: %v", len(after), err) } // A second run finds the kind moved and writes nothing: every key still has the one value it was made with. again, err := inv.MoveChain(ctx, nil) if err != nil || len(again) != 1 || !again[0].Already || again[0].Written != 0 { t.Fatalf("the second move %+v: %v", again, err) } for _, id := range []string{"build-1", "build-2", "build-3", "build-4"} { if history, err := inv.chain.Builds.History(ctx, chainBuildKey(id)); err != nil || len(history) != 1 { t.Fatalf("%s has %d values: %v", id, len(history), err) } } } // **A move that stopped half way is finished by the next start, and never lays a row over a newer bucket value** (ADR // 0408 §8): the keys that exist are present, the others are written. func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) { inv, finish := ForTestBefore(t, 105) finish() ctx := context.Background() aBuildRow(t, inv, "build-1", "notes", "c1", "") aBuildRow(t, inv, "build-2", "notes", "c2", "") aBuildRow(t, inv, "build-3", "wiki", "c3", "") // An earlier start wrote build-1, and the bucket has had a newer value since. newer := chain.Build{ID: "build-1", Repository: "novox/notes", Module: "notes", Commit: "newer"} if _, err := inv.chain.Builds.Create(ctx, "build-1", newer); err != nil { t.Fatal(err) } reports, err := inv.MoveChain(ctx, nil) if err != nil { t.Fatal(err) } if reports[0].Written != 2 || reports[0].Present != 1 || reports[0].Rows != 3 { t.Fatalf("the move reported %+v", reports[0]) } got, _, _ := inv.chain.Builds.Get(ctx, "build-1") if !reflect.DeepEqual(got.Value, newer) { t.Fatalf("the newer value was laid over: %+v", got.Value) } if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 { t.Fatalf("chain_moved says %v", moved) } } // **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6): // a Refusal, no key in the bucket, no kind recorded, and the tables still what is read. func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) { inv, finish := ForTestBefore(t, 105) finish() ctx := context.Background() aBuildRow(t, inv, "build-1", "notes", "c1", "") inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest) values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`) _, err := inv.MoveChain(ctx, nil) var refusal *Refusal if !errors.As(err, &refusal) || refusal.Kind != KindBuilds { t.Fatalf("the move answered %v", err) } if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 { t.Fatalf("the bucket holds %v after a refused move", keys) } if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { t.Fatalf("a refused move recorded %v", moved) } if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 { t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err) } } // losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it. type losing struct { chain.Backend lose string } func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) { if key == l.lose { return 1, nil } return l.Backend.Create(ctx, key, value) } // **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it, // nothing is recorded, and the tables are still read. func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) { inv, finish := ForTestBefore(t, 105) finish() ctx := context.Background() aBuildRow(t, inv, "build-1", "notes", "c1", "") aBuildRow(t, inv, "build-2", "notes", "c2", "") inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return nil })) _, err := inv.MoveChain(ctx, nil) var refusal *Refusal if !errors.As(err, &refusal) { t.Fatalf("the move answered %v", err) } if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { t.Fatalf("a refused move recorded %v", moved) } } // **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused: // the tables are still read. func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) { inv, finish := ForTestBefore(t, 105) finish() ctx := context.Background() aBuildRow(t, inv, "build-1", "notes", "c1", "") mem := chain.NewMemory() inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return errors.New("another controller holds the lease") })) _, err := inv.MoveChain(ctx, nil) var refusal *Refusal if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) { t.Fatalf("the move answered %v", err) } if mem.Writes() != 0 { t.Fatalf("the bucket took %d writes without the lease", mem.Writes()) } if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 { t.Fatalf("recorded %v", moved) } } // **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its // id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none. func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { inv := fresh(t) ctx := context.Background() first := aBuild("build-1", "notes", "") first.Commit = "c1" first.Against = []string{"artifact-store://base@sha256:aa"} first.ReuseKey = "rk-1" second := aBuild("build-2", "notes", "") second.Commit = "c2" failed := aBuild("build-3", "notes", "compile error") other := aBuild("build-4", "wiki", "") for _, b := range []Build{first, second, failed, other, first} { if err := inv.RecordBuild(ctx, b); err != nil { t.Fatal(err) } } if n := rowsInTheTable(t, inv, "build"); n != "0" { t.Fatalf("the table took %s builds", n) } kept, found, err := inv.chain.Builds.Get(ctx, "build-1") if err != nil || !found || kept.Value.ReuseKey != "rk-1" { t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err) } if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 { t.Fatalf("a build recorded twice has %d values", len(history)) } list, err := inv.Builds(ctx, "notes", 10) if err != nil || len(list) != 3 { t.Fatalf("notes has %d builds: %v", len(list), err) } one, found, err := inv.BuildByID(ctx, "build-2") if err != nil || !found || one.Commit != "c2" || one.Module != "notes" { t.Fatalf("build-2 reads as %+v %v %v", one, found, err) } if _, found, _ := inv.BuildByID(ctx, "build-nope"); found { t.Fatal("a build nobody recorded was found") } if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" { t.Fatalf("the build of notes at c1 is %q: %v", id, err) } held, err := inv.Held(ctx) if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" { t.Fatalf("held %v: %v", held, err) } against, err := inv.BuiltAgainst(ctx) if err != nil || len(against["notes"]) != 0 { // The newest successful build of notes, build-2, stood on nothing. t.Fatalf("built against %v: %v", against, err) } announced, err := inv.Announceable(ctx) if err != nil || len(announced) != 3 { t.Fatalf("announceable %d: %v", len(announced), err) } prints, err := inv.Fingerprints(ctx) if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { t.Fatalf("fingerprints %v: %v", prints, err) } } // A build request row as the store held it before the move. func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) { t.Helper() inv.ExecForTest(t, `insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at) values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`, id, notAsked) } // **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue // 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none // after the outcome is heard. The table of requests takes none. func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) { inv := fresh(t) ctx := context.Background() ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true} if err := inv.RecordBuildRequest(ctx, ask); err != nil { t.Fatal(err) } if err := inv.RecordBuildRequest(ctx, ask); err != nil { t.Fatalf("a request kept twice: %v", err) } if n := rowsInTheTable(t, inv, "build_request"); n != "0" { t.Fatalf("the table took %s requests", n) } if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 { t.Fatalf("a request nobody answered is a build: %+v %v", builds, err) } if _, found, _ := inv.BuildByID(ctx, "build-1"); found { t.Fatal("a request nobody answered has an outcome") } got, found, err := inv.BuildRequestByID(ctx, "build-1") if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() { t.Fatalf("the request reads back as %+v %v %v", got, found, err) } if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil { t.Fatal(err) } asked, err := inv.RequestsNamed(ctx, "notes") if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" { t.Fatalf("the request is %+v: %v", asked, err) } if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) { t.Fatalf("requested names %v: %v", names, err) } // The outcome comes: the same key, now a build, with the request beside it. built := aBuild("build-1", "notes", "") if err := inv.RecordBuild(ctx, built); err != nil { t.Fatal(err) } if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" { t.Fatalf("the outcome reads as %+v %v %v", one, found, err) } answered, err := inv.RequestsNamed(ctx, "notes") if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" { t.Fatalf("the request answered is %+v: %v", answered, err) } if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" { t.Fatalf("the request was lost with the outcome: %+v", again) } // Heard: the marks of an unanswered request change nothing. if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil { t.Fatal(err) } if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" { t.Fatalf("an answered request was marked not asked: %+v", again) } if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 { // asked, marked unknown, answered. t.Fatalf("the record has %d values", len(history)) } // An outcome heard before its request is kept, and the request lands beside it. if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil { t.Fatal(err) } if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil { t.Fatal(err) } if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" { t.Fatalf("the request kept after its outcome reads %+v", late) } // A request older than the time they are kept is no longer listed. if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old", At: time.Now().Add(-2 * KeptFor)}); err != nil { t.Fatal(err) } if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") { t.Fatalf("a request past its time is listed: %v", names) } } // **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a // request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it // is asked and so asked at most once. func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) { inv, finish := ForTestBefore(t, 105) finish() ctx := context.Background() aBuildRow(t, inv, "build-1", "notes", "c1", "") aRequestRow(t, inv, "build-1", "") aRequestRow(t, inv, "build-2", "") aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder") if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 { t.Fatalf("before the move the table holds %d requests: %v", len(requests), err) } reports, err := inv.MoveChain(ctx, nil) if err != nil { t.Fatal(err) } if reports[0].Rows != 3 || reports[0].TableRows != 4 { t.Fatalf("the move reported %+v", reports[0]) } one, _, _ := inv.chain.Builds.Get(ctx, "build-1") if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" { t.Fatalf("an answered request moved as %+v", one.Value) } two, _, _ := inv.chain.Builds.Get(ctx, "build-2") if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" { t.Fatalf("an unanswered request moved as %+v", two.Value) } never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes") if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending { t.Fatalf("a request never handed over moved as %+v", never.Value) } requests, err := inv.RequestsNamed(ctx, "notes") if err != nil || len(requests) != 3 { t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err) } toAsk, err := inv.RequestsToAskAgain(ctx) if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" { t.Fatalf("to ask again: %+v %v", toAsk, err) } first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID) second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID) if err != nil || err2 != nil || !first || second { t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2) } if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 { t.Fatalf("a claimed request is still to be asked: %+v", left) } if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil { t.Fatal(err) } if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" { t.Fatalf("the request lost its words: %+v", done) } } // **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the // table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting // for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last. func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) { inv, finish := ForTestBefore(t, 105) finish() ctx := context.Background() batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC()) walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC()) walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}} walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}} walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}} for _, p := range []*Plan{batch, walk} { if err := inv.SavePlan(ctx, p); err != nil { t.Fatal(err) } } walk.Note = "second save" if err := inv.SavePlan(ctx, walk); err != nil { t.Fatal(err) } for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} { if added, err := inv.AddMerge(ctx, m); err != nil || !added { t.Fatalf("added %v: %v", added, err) } } waiting := aMerge("m-waiting", "plan-2") if _, err := inv.AddMerge(ctx, waiting); err != nil { t.Fatal(err) } if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil { t.Fatal(err) } wasPlans, err := inv.RecentPlans(ctx, 10) if err != nil || len(wasPlans) != 2 { t.Fatalf("the table holds %d plans: %v", len(wasPlans), err) } wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") reports, err := inv.MoveChain(ctx, nil) if err != nil { t.Fatal(err) } var moved MoveReport for _, r := range reports { if r.Kind == KindRollouts { moved = r } } // Two plans, and the record of the merge whose plan the store never kept. if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 { t.Fatalf("the move of the rollouts reported %+v", moved) } nowPlans, err := inv.RecentPlans(ctx, 10) if err != nil || len(nowPlans) != 2 { t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err) } for k := range wasPlans { was, now := wasPlans[k], nowPlans[k] if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note || !was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) { t.Errorf("plan %s was %+v and is %+v", was.ID, was, now) } } if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" { t.Fatalf("open plans %+v", open) } if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" { t.Fatalf("batches %+v", batches) } if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" { t.Fatalf("the batch answers %+v", got) } nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk") if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) || !nowMerge.Heard.Equal(wasMerge.Heard) || string(nowMerge.Event) != string(wasMerge.Event) { t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge) } alone, _ := inv.AloneMerges(ctx) if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" { t.Fatalf("waiting: %+v", alone) } if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone { t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known) } if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { t.Fatalf("the table is %s plans", n) } // From now on a plan is saved to the bucket and the table takes none. later := aBatch("plan-3", time.Now().UTC()) if err := inv.SavePlan(ctx, later); err != nil { t.Fatal(err) } if n := rowsInTheTable(t, inv, "release_plan"); n != "2" { t.Fatalf("the table took a plan after the move: %s", n) } if again, err := inv.MoveChain(ctx, nil); err != nil || len(again) != 2 || !again[0].Already || !again[1].Already { t.Fatalf("the second move %+v: %v", again, err) } }