mesh-controller/internal/inventory/chain_move_test.go

//go:build integration

package inventory

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"reflect"
	"slices"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/chain"
)

// A build row as the store held it before the move: written by SQL, the only hand-written state these tests have, for
// it stands for what the mesh held before this release.
func aBuildRow(t *testing.T, inv *Inventory, id, module, commit, failed string) {
	t.Helper()
	inv.ExecForTest(t,
		`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made, source_path, manifest,
		                    built_against, built_contexts, asked, source_fingerprint, build_sources)
		 values ($1::text, 'novox/' || $2::text, 'main', $2::text, $3::text, 'ace', $4::text,
		         case when $4::text = ''
		              then ('[{"name":"app","kind":"image","reference":"artifact-store://' || $2::text || '@sha256:' || $1::text || '"}]')::jsonb
		              else '[]'::jsonb end,
		         'modules/' || $2::text, ('{"module":"' || $2::text || '"}')::jsonb,
		         '["artifact-store://base@sha256:aa"]'::jsonb, '[{"repository":"novox/other","ref":"main"}]'::jsonb,
		         now() - interval '1 hour', 'fp-' || $1::text,
		         ('[{"paths":["modules/' || $2::text || '/module.json"]}]')::jsonb)`,
		id, module, commit, failed)
}

// inTheTables is an inventory before the move: its builds and assignments are rows of the store, which a test of a data
// migration of the store (that reads those rows) needs.
func inTheTables(t *testing.T) *Inventory {
	t.Helper()
	inv, finish := ForTestBefore(t, 105)
	finish()
	return inv
}

// reportOf is the report of one kind among those a move made, which are one for each kind in the order they are moved.
func reportOf(t *testing.T, reports []MoveReport, kind string) MoveReport {
	t.Helper()
	for _, r := range reports {
		if r.Kind == kind {
			return r
		}
	}
	t.Fatalf("no report of %s among %+v", kind, reports)
	return MoveReport{}
}

// allAlready says every kind was moved by an earlier start: the second run writes nothing, and reports one for each kind.
func allAlready(t *testing.T, reports []MoveReport) {
	t.Helper()
	if len(reports) != 3 {
		t.Fatalf("the move reported %d kinds, want 3: %+v", len(reports), reports)
	}
	for _, r := range reports {
		if !r.Already || r.Written != 0 {
			t.Fatalf("the second move of %s %+v", r.Kind, r)
		}
	}
}

func rowsInTheTable(t *testing.T, inv *Inventory, table string) string {
	t.Helper()
	rows := inv.QueryForTest(t, `select count(*) from `+table)
	return rows[0][0]
}

// **The move copies every build row into the builds bucket, and a second run writes nothing** (ADR 0408 §8, issue
// 1090): each row under its id, whole; the kind recorded in chain_moved; and from then on a build is read from the
// bucket and written to it, the table taking no more.
func TestTheMoveCopiesTheBuildRowsAndAgainWritesNothing(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	aBuildRow(t, inv, "build-1", "notes", "c1", "")
	aBuildRow(t, inv, "build-2", "notes", "c2", "")
	aBuildRow(t, inv, "build-3", "wiki", "", "compile error")

	// Before the move the tables are read, and the bucket is empty.
	if before, err := inv.Builds(ctx, "", 10); err != nil || len(before) != 3 {
		t.Fatalf("before the move the tables hold %d builds: %v", len(before), err)
	}
	if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
		t.Fatalf("moved before the move: %v", moved)
	}

	reports, err := inv.MoveChain(ctx, nil)
	if err != nil {
		t.Fatal(err)
	}
	if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.Written != 3 || b.Largest == 0 {
		t.Fatalf("the move of the builds reported %+v", reports)
	}
	moved, err := inv.ChainMoved(ctx)
	if err != nil || moved[KindBuilds] != 3 {
		t.Fatalf("chain_moved says %v: %v", moved, err)
	}
	kept, found, err := inv.chain.Builds.Get(ctx, "build-2")
	if err != nil || !found {
		t.Fatalf("build-2 is not in the bucket: %v %v", found, err)
	}
	if v := kept.Value; v.Module != "notes" || v.Commit != "c2" || v.Repository != "novox/notes" || v.Path != "modules/notes" ||
		v.SourceFingerprint != "fp-build-2" || len(v.Made) != 1 || len(v.Against) != 1 || len(v.Read) != 1 || len(v.Sources) != 1 ||
		v.Published == nil || v.Published.Artifact != "sha256:build-2" || v.Asked.IsZero() || v.At.IsZero() || len(v.Manifest) == 0 {
		t.Fatalf("build-2 was moved as %+v", v)
	}

	// From now on the bucket is what is read and written.
	if err := inv.RecordBuild(ctx, aBuild("build-4", "notes", "")); err != nil {
		t.Fatal(err)
	}
	if n := rowsInTheTable(t, inv, "build"); n != "3" {
		t.Fatalf("the table took a build after the move: %s rows", n)
	}
	after, err := inv.Builds(ctx, "notes", 10)
	if err != nil || len(after) != 3 {
		t.Fatalf("notes has %d builds on the bus: %v", len(after), err)
	}

	// A second run finds the kind moved and writes nothing: every key still has the one value it was made with.
	again, err := inv.MoveChain(ctx, nil)
	if err != nil {
		t.Fatal(err)
	}
	allAlready(t, again)
	for _, id := range []string{"build-1", "build-2", "build-3", "build-4"} {
		if history, err := inv.chain.Builds.History(ctx, chainBuildKey(id)); err != nil || len(history) != 1 {
			t.Fatalf("%s has %d values: %v", id, len(history), err)
		}
	}
}

// failingToCreate is a bucket that fails to make one key while told to: the bus going away half way through a move.
type failingToCreate struct {
	chain.Backend
	key  string
	fail *atomic.Bool
}

func (f failingToCreate) Create(ctx context.Context, key string, value []byte) (uint64, error) {
	if key == f.key && f.fail.Load() {
		return 0, errors.New("the bus went away")
	}
	return f.Backend.Create(ctx, key, value)
}

// **A controller restarted half way through the move finishes it: the bucket equals the table, with no key made twice and
// no refusal** (ADR 0408 §8). Before a kind is recorded as moved nothing but the move writes its keys, and every write of
// the kind goes to its table (each method of a kind asks ChainReads first and writes the store until it says moved), so the
// table is the only truth: a key the first start made is the row it was, and the second start settles it to the row it is.
func TestAControllerRestartedMidMoveFinishesItAndOverwritesNothing(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	aBuildRow(t, inv, "build-1", "notes", "c1", "")
	aBuildRow(t, inv, "build-2", "notes", "c2", "")
	aBuildRow(t, inv, "build-3", "wiki", "c3", "")
	var away atomic.Bool
	away.Store(true)
	builds := chain.NewMemory()
	inv.UseChain(chain.New(failingToCreate{builds, "build-3", &away}, chain.NewMemory(), chain.NewMemory(),
		func(context.Context) error { return nil }))
	// The first start made two of the three keys and the bus went away.
	_, err := inv.MoveChain(ctx, nil)
	var refusal *Refusal
	if err == nil || errors.As(err, &refusal) {
		t.Fatalf("the first start answered %v: it should fail to be tried again, not be refused", err)
	}
	if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
		t.Fatalf("a move that did not finish recorded %v", moved)
	}
	if keys, _ := builds.Keys(ctx); len(keys) != 2 {
		t.Fatalf("the first start made %v", keys)
	}
	// The controller is started again, the bus is back, and the move finishes.
	away.Store(false)
	reports, err := inv.MoveChain(ctx, nil)
	if err != nil {
		t.Fatalf("the second start was refused or failed: %v", err)
	}
	if b := reportOf(t, reports, KindBuilds); b.Written != 1 || b.Present != 2 || b.Rows != 3 {
		t.Fatalf("the move reported %+v", b)
	}
	if moved, _ := inv.ChainMoved(ctx); moved[KindBuilds] != 3 {
		t.Fatalf("chain_moved says %v", moved)
	}
	// The bucket is the table: three keys, each made once and not again, each the row's own.
	for _, id := range []string{"build-1", "build-2", "build-3"} {
		history, err := builds.History(ctx, id)
		if err != nil || len(history) != 1 {
			t.Fatalf("%s has %d values: %v", id, len(history), err)
		}
	}
	rows, err := inv.allBuildRows(ctx)
	if err != nil || len(rows) != 3 {
		t.Fatalf("the table holds %d builds: %v", len(rows), err)
	}
	for _, row := range rows {
		got, found, err := inv.chain.Builds.Get(ctx, row.ID)
		want, _ := json.Marshal(toChainBuild(row))
		have, _ := json.Marshal(got.Value)
		if err != nil || !found || string(want) != string(have) {
			t.Fatalf("%s in the bucket is %s, the row says %s (%v)", row.ID, have, want, err)
		}
	}
}

// **A value over the bus's largest message stops the move with nothing written and nothing switched** (ADR 0408 §6):
// a Refusal, no key in the bucket, no kind recorded, and the tables still what is read.
func TestTheMoveRefusesAValueOverTheLargestMessage(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	aBuildRow(t, inv, "build-1", "notes", "c1", "")
	inv.ExecForTest(t, `insert into build (id, repository, module, commit_hash, manifest)
	                    values ('build-big', 'novox/wiki', 'wiki', 'c9', ('{"x":"' || repeat('a', 1100000) || '"}')::jsonb)`)
	_, err := inv.MoveChain(ctx, nil)
	var refusal *Refusal
	if !errors.As(err, &refusal) || refusal.Kind != KindBuilds {
		t.Fatalf("the move answered %v", err)
	}
	if keys, _ := inv.chain.Builds.Keys(ctx); len(keys) != 0 {
		t.Fatalf("the bucket holds %v after a refused move", keys)
	}
	if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
		t.Fatalf("a refused move recorded %v", moved)
	}
	if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 2 {
		t.Fatalf("the tables are no longer read: %d builds, %v", len(builds), err)
	}
}

// losing is a bucket that says it wrote a key and kept nothing: the move's count of keys against rows must see it.
type losing struct {
	chain.Backend
	lose string
}

func (l losing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
	if key == l.lose {
		return 1, nil
	}
	return l.Backend.Create(ctx, key, value)
}

// **The move refuses on a count that differs** (ADR 0408 §8): a row whose key is not in the bucket afterwards stops it,
// nothing is recorded, and the tables are still read.
func TestTheMoveRefusesOnACountThatDiffers(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	aBuildRow(t, inv, "build-1", "notes", "c1", "")
	aBuildRow(t, inv, "build-2", "notes", "c2", "")
	inv.UseChain(chain.New(losing{chain.NewMemory(), "build-2"}, chain.NewMemory(), chain.NewMemory(),
		func(context.Context) error { return nil }))
	_, err := inv.MoveChain(ctx, nil)
	var refusal *Refusal
	if !errors.As(err, &refusal) {
		t.Fatalf("the move answered %v", err)
	}
	if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
		t.Fatalf("a refused move recorded %v", moved)
	}
}

// **Without the lease the move writes nothing**, and fails the way the bus being away does — tried again, not refused:
// the tables are still read.
func TestTheMoveWithoutTheLeaseWritesNothing(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	aBuildRow(t, inv, "build-1", "notes", "c1", "")
	mem := chain.NewMemory()
	inv.UseChain(chain.New(mem, chain.NewMemory(), chain.NewMemory(),
		func(context.Context) error { return errors.New("another controller holds the lease") }))
	_, err := inv.MoveChain(ctx, nil)
	var refusal *Refusal
	if err == nil || errors.As(err, &refusal) || !errors.Is(err, chain.ErrNotHeld) {
		t.Fatalf("the move answered %v", err)
	}
	if mem.Writes() != 0 {
		t.Fatalf("the bucket took %d writes without the lease", mem.Writes())
	}
	if moved, _ := inv.ChainMoved(ctx); len(moved) != 0 {
		t.Fatalf("recorded %v", moved)
	}
}

// **Once builds are on the bus every reader of them reads the bucket** (issue 1090): a build recorded is the key of its
// id, recorded again it stays one, and the lists the planner reads are made from the bucket — the table takes none.
func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
	inv := fresh(t)
	ctx := context.Background()
	first := aBuild("build-1", "notes", "")
	first.Commit = "c1"
	first.Against = []string{"artifact-store://base@sha256:aa"}
	first.ReuseKey = "rk-1"
	second := aBuild("build-2", "notes", "")
	second.Commit = "c2"
	failed := aBuild("build-3", "notes", "compile error")
	other := aBuild("build-4", "wiki", "")
	for _, b := range []Build{first, second, failed, other, first} {
		if err := inv.RecordBuild(ctx, b); err != nil {
			t.Fatal(err)
		}
	}
	if n := rowsInTheTable(t, inv, "build"); n != "0" {
		t.Fatalf("the table took %s builds", n)
	}
	kept, found, err := inv.chain.Builds.Get(ctx, "build-1")
	if err != nil || !found || kept.Value.ReuseKey != "rk-1" {
		t.Fatalf("build-1 on the bus: %+v %v %v", kept.Value, found, err)
	}
	if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 1 {
		t.Fatalf("a build recorded twice has %d values", len(history))
	}
	list, err := inv.Builds(ctx, "notes", 10)
	if err != nil || len(list) != 3 {
		t.Fatalf("notes has %d builds: %v", len(list), err)
	}
	one, found, err := inv.BuildByID(ctx, "build-2")
	if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
		t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
	}
	if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
		t.Fatal("a build nobody recorded was found")
	}
	if id, err := inv.BuildOf(ctx, "notes", "c1"); err != nil || id != "build-1" {
		t.Fatalf("the build of notes at c1 is %q: %v", id, err)
	}
	held, err := inv.Held(ctx)
	if err != nil || held["notes/config"] == "" || held["wiki/config"] == "" {
		t.Fatalf("held %v: %v", held, err)
	}
	against, err := inv.BuiltAgainst(ctx)
	if err != nil || len(against["notes"]) != 0 {
		// The newest successful build of notes, build-2, stood on nothing.
		t.Fatalf("built against %v: %v", against, err)
	}
	announced, err := inv.Announceable(ctx)
	if err != nil || len(announced) != 3 {
		t.Fatalf("announceable %d: %v", len(announced), err)
	}
	prints, err := inv.Fingerprints(ctx)
	if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
		t.Fatalf("fingerprints %v: %v", prints, err)
	}
}

// A build request row as the store held it before the move.
func aRequestRow(t *testing.T, inv *Inventory, id, notAsked string) {
	t.Helper()
	inv.ExecForTest(t,
		`insert into build_request (id, repository, seat, source_path, ref, commit_hash, asked_for, not_asked, asked_at)
		 values ($1::text, 'novox/notes', '', 'modules/notes', 'main', 'c1', 'merge', nullif($2::text, ''), now() - interval '1 hour')`,
		id, notAsked)
}

// **A request is the build record in state asked, and its outcome takes its place and keeps it** (ADR 0408 §1, issue
// 1091): no outcome to read until it comes, the marks of a request that nobody answered kept in the record, and none
// after the outcome is heard. The table of requests takes none.
func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) {
	inv := fresh(t)
	ctx := context.Background()
	ask := BuildRequest{ID: "build-1", Repository: "novox/notes", Path: "modules/notes", Ref: "main", For: "build", AtTerminal: true}
	if err := inv.RecordBuildRequest(ctx, ask); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordBuildRequest(ctx, ask); err != nil {
		t.Fatalf("a request kept twice: %v", err)
	}
	if n := rowsInTheTable(t, inv, "build_request"); n != "0" {
		t.Fatalf("the table took %s requests", n)
	}
	if builds, err := inv.Builds(ctx, "", 10); err != nil || len(builds) != 0 {
		t.Fatalf("a request nobody answered is a build: %+v %v", builds, err)
	}
	if _, found, _ := inv.BuildByID(ctx, "build-1"); found {
		t.Fatal("a request nobody answered has an outcome")
	}
	got, found, err := inv.BuildRequestByID(ctx, "build-1")
	if err != nil || !found || got.Repository != "novox/notes" || got.For != "build" || !got.AtTerminal || got.At.IsZero() {
		t.Fatalf("the request reads back as %+v %v %v", got, found, err)
	}
	if err := inv.MarkOutcomeUnknown(ctx, "build-1", "stopped waiting"); err != nil {
		t.Fatal(err)
	}
	asked, err := inv.RequestsNamed(ctx, "notes")
	if err != nil || len(asked) != 1 || asked[0].Heard || asked[0].OutcomeUnknown != "stopped waiting" {
		t.Fatalf("the request is %+v: %v", asked, err)
	}
	if names, err := inv.RequestedNames(ctx); err != nil || !reflect.DeepEqual(names, []string{"notes"}) {
		t.Fatalf("requested names %v: %v", names, err)
	}

	// The outcome comes: the same key, now a build, with the request beside it.
	built := aBuild("build-1", "notes", "")
	if err := inv.RecordBuild(ctx, built); err != nil {
		t.Fatal(err)
	}
	if one, found, err := inv.BuildByID(ctx, "build-1"); err != nil || !found || one.Module != "notes" {
		t.Fatalf("the outcome reads as %+v %v %v", one, found, err)
	}
	answered, err := inv.RequestsNamed(ctx, "notes")
	if err != nil || len(answered) != 1 || !answered[0].Heard || answered[0].Module != "notes" || answered[0].Failed != "" {
		t.Fatalf("the request answered is %+v: %v", answered, err)
	}
	if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.For != "build" || again.OutcomeUnknown != "stopped waiting" {
		t.Fatalf("the request was lost with the outcome: %+v", again)
	}
	// Heard: the marks of an unanswered request change nothing.
	if err := inv.MarkNotAsked(ctx, "build-1", "late"); err != nil {
		t.Fatal(err)
	}
	if again, _, _ := inv.BuildRequestByID(ctx, "build-1"); again.NotAsked != "" {
		t.Fatalf("an answered request was marked not asked: %+v", again)
	}
	if history, _ := inv.chain.Builds.History(ctx, "build-1"); len(history) != 3 {
		// asked, marked unknown, answered.
		t.Fatalf("the record has %d values", len(history))
	}
	// An outcome heard before its request is kept, and the request lands beside it.
	if err := inv.RecordBuild(ctx, aBuild("build-2", "wiki", "")); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-2", Repository: "novox/wiki", Path: "modules/wiki", For: "plan"}); err != nil {
		t.Fatal(err)
	}
	if late, found, _ := inv.BuildRequestByID(ctx, "build-2"); !found || late.For != "plan" {
		t.Fatalf("the request kept after its outcome reads %+v", late)
	}
	// A request older than the time they are kept is no longer listed.
	if err := inv.RecordBuildRequest(ctx, BuildRequest{ID: "build-old", Repository: "novox/old", Path: "modules/old",
		At: time.Now().Add(-2 * KeptFor)}); err != nil {
		t.Fatal(err)
	}
	if names, _ := inv.RequestedNames(ctx); slices.Contains(names, "old") {
		t.Fatalf("a request past its time is listed: %v", names)
	}
}

// **The move carries the build requests**: an answered one beside its outcome, an unanswered one as state asked, and a
// request never handed over as state asked marked to be asked again once (issue 981 note 7 Q5), which is claimed before it
// is asked and so asked at most once.
func TestTheMoveCarriesTheBuildRequestsAndMarksTheNeverAskedToAskAgain(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	aBuildRow(t, inv, "build-1", "notes", "c1", "")
	aRequestRow(t, inv, "build-1", "")
	aRequestRow(t, inv, "build-2", "")
	aRequestRow(t, inv, "not-asked-c1-modules-notes", "no seat holder")

	if requests, err := inv.RequestsNamed(ctx, "notes"); err != nil || len(requests) != 3 {
		t.Fatalf("before the move the table holds %d requests: %v", len(requests), err)
	}
	reports, err := inv.MoveChain(ctx, nil)
	if err != nil {
		t.Fatal(err)
	}
	if b := reportOf(t, reports, KindBuilds); b.Rows != 3 || b.TableRows != 4 {
		t.Fatalf("the move reported %+v", b)
	}
	one, _, _ := inv.chain.Builds.Get(ctx, "build-1")
	if one.Value.State != "" || one.Value.Request == nil || one.Value.Request.For != "merge" || one.Value.Module != "notes" {
		t.Fatalf("an answered request moved as %+v", one.Value)
	}
	two, _, _ := inv.chain.Builds.Get(ctx, "build-2")
	if two.Value.State != chain.BuildAsked || two.Value.Request.Reask != "" {
		t.Fatalf("an unanswered request moved as %+v", two.Value)
	}
	never, _, _ := inv.chain.Builds.Get(ctx, "not-asked-c1-modules-notes")
	if never.Value.State != chain.BuildAsked || never.Value.Request.NotAsked != "no seat holder" || never.Value.Request.Reask != chain.ReaskPending {
		t.Fatalf("a request never handed over moved as %+v", never.Value)
	}
	requests, err := inv.RequestsNamed(ctx, "notes")
	if err != nil || len(requests) != 3 {
		t.Fatalf("after the move the bucket holds %d requests: %v", len(requests), err)
	}
	toAsk, err := inv.RequestsToAskAgain(ctx)
	if err != nil || len(toAsk) != 1 || toAsk[0].ID != "not-asked-c1-modules-notes" {
		t.Fatalf("to ask again: %+v %v", toAsk, err)
	}
	first, err := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
	second, err2 := inv.ClaimToAskAgain(ctx, toAsk[0].ID)
	if err != nil || err2 != nil || !first || second {
		t.Fatalf("claimed %v then %v: %v %v", first, second, err, err2)
	}
	if left, _ := inv.RequestsToAskAgain(ctx); len(left) != 0 {
		t.Fatalf("a claimed request is still to be asked: %+v", left)
	}
	if err := inv.AskedAgainAs(ctx, toAsk[0].ID, "build-9"); err != nil {
		t.Fatal(err)
	}
	if done, _, _ := inv.BuildRequestByID(ctx, toAsk[0].ID); done.NotAsked != "no seat holder" {
		t.Fatalf("the request lost its words: %+v", done)
	}
}

// **The move carries every plan with the merges it answers, and the plans read the same from the bucket as from the
// table** (ADR 0408 §8, ADR 0276, issue 1092): a batch with a merge, a walk that was built and rolled, a merge waiting
// for a walk of its own, and a merge whose plan the store never kept — each in the record of the plan that held it last.
func TestTheMoveCarriesThePlansAndTheirMergesAndTheyReadTheSame(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	batch := aBatch("plan-1", time.Now().Add(-3*time.Hour).UTC())
	walk := aBatch("plan-2", time.Now().Add(-2*time.Hour).UTC())
	walk.State, walk.Tier, walk.Tiers = PlanRolling, 1, [][]string{{"a"}, {"b"}}
	walk.Modules = map[string]*PlanModule{"a": {State: "built"}, "b": {}}
	walk.Delivery = &PlanDelivery{Awaits: "mesh-delivery", Merges: []PlanMerge{{Repository: "novox/mesh-catalog", Commit: "m-waiting"}}}
	for _, p := range []*Plan{batch, walk} {
		if err := inv.SavePlan(ctx, p); err != nil {
			t.Fatal(err)
		}
	}
	walk.Note = "second save"
	if err := inv.SavePlan(ctx, walk); err != nil {
		t.Fatal(err)
	}
	for _, m := range []BatchedMerge{aMerge("m-batch", "plan-1"), aMerge("m-walk", "plan-2"), aMerge("m-dangling", "plan-never-kept")} {
		if added, err := inv.AddMerge(ctx, m); err != nil || !added {
			t.Fatalf("added %v: %v", added, err)
		}
	}
	waiting := aMerge("m-waiting", "plan-2")
	if _, err := inv.AddMerge(ctx, waiting); err != nil {
		t.Fatal(err)
	}
	if err := inv.AnswerMerge(ctx, waiting.Repository, waiting.Commit, "", true); err != nil {
		t.Fatal(err)
	}
	wasPlans, err := inv.RecentPlans(ctx, 10)
	if err != nil || len(wasPlans) != 2 {
		t.Fatalf("the table holds %d plans: %v", len(wasPlans), err)
	}
	wasMerge, _, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")

	reports, err := inv.MoveChain(ctx, nil)
	if err != nil {
		t.Fatal(err)
	}
	var moved MoveReport
	for _, r := range reports {
		if r.Kind == KindRollouts {
			moved = r
		}
	}
	// Two plans, and the record of the merge whose plan the store never kept.
	if moved.Rows != 3 || moved.TableRows != 6 || moved.Written != 3 {
		t.Fatalf("the move of the rollouts reported %+v", moved)
	}
	nowPlans, err := inv.RecentPlans(ctx, 10)
	if err != nil || len(nowPlans) != 2 {
		t.Fatalf("the bucket holds %d plans: %v", len(nowPlans), err)
	}
	for k := range wasPlans {
		was, now := wasPlans[k], nowPlans[k]
		if was.ID != now.ID || was.State != now.State || was.Tier != now.Tier || was.Revision != now.Revision || was.Note != now.Note ||
			!was.Created.Equal(now.Created) || !reflect.DeepEqual(was.Tiers, now.Tiers) || len(was.Modules) != len(now.Modules) {
			t.Errorf("plan %s was %+v and is %+v", was.ID, was, now)
		}
	}
	if open, _ := inv.OpenPlans(ctx); len(open) != 1 || open[0].ID != "plan-2" {
		t.Fatalf("open plans %+v", open)
	}
	if batches, _ := inv.Batches(ctx); len(batches) != 1 || batches[0].ID != "plan-1" {
		t.Fatalf("batches %+v", batches)
	}
	if got, _ := inv.MergesOf(ctx, "plan-1"); len(got) != 1 || got[0].Commit != "m-batch" {
		t.Fatalf("the batch answers %+v", got)
	}
	nowMerge, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-walk")
	if !known || nowMerge.Plan != "plan-2" || nowMerge.Branch != wasMerge.Branch || !nowMerge.Merged.Equal(wasMerge.Merged) ||
		!nowMerge.Heard.Equal(wasMerge.Heard) || !sameEvent(nowMerge.Event, wasMerge.Event) {
		t.Fatalf("a merge was %+v and is %+v", wasMerge, nowMerge)
	}
	alone, _ := inv.AloneMerges(ctx)
	if len(alone) != 1 || alone[0].Commit != "m-waiting" || alone[0].Plan != "" {
		t.Fatalf("waiting: %+v", alone)
	}
	if dangling, known, _ := inv.MergeOf(ctx, "novox/mesh-catalog", "m-dangling"); !known || dangling.Plan != "" || dangling.Alone {
		t.Fatalf("a merge whose plan was never kept reads as %+v %v", dangling, known)
	}
	if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
		t.Fatalf("the table is %s plans", n)
	}
	// From now on a plan is saved to the bucket and the table takes none.
	later := aBatch("plan-3", time.Now().UTC())
	if err := inv.SavePlan(ctx, later); err != nil {
		t.Fatal(err)
	}
	if n := rowsInTheTable(t, inv, "release_plan"); n != "2" {
		t.Fatalf("the table took a plan after the move: %s", n)
	}
	again, err := inv.MoveChain(ctx, nil)
	if err != nil {
		t.Fatal(err)
	}
	allAlready(t, again)
}

// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
// assignment goes with it.
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
	inv, node := aNodeWithModules(t, "postgres", "web")
	ctx := t.Context()
	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
		t.Fatal(err)
	}
	start := generationNow(t, inv)
	if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
		t.Fatalf("assigned %v: %v", added, err)
	}
	if added, err := inv.Assign(ctx, node, "postgres"); err != nil || added {
		t.Fatalf("assigned again %v: %v", added, err)
	}
	if n := rowsInTheTable(t, inv, "assignment"); n != "0" {
		t.Fatalf("the table took %s assignments", n)
	}
	kept, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
	if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
		t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
	}
	if generationNow(t, inv) <= start {
		t.Fatal("an assignment left the generation where it was")
	}
	if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
		t.Fatalf("a module the mesh does not know: %v", err)
	}
	if on, err := inv.Assigned(ctx, node); err != nil || !reflect.DeepEqual(on, []string{"postgres"}) {
		t.Fatalf("assigned on %s: %v %v", node, on, err)
	}
	if running, err := inv.Running(ctx, "postgres"); err != nil || !reflect.DeepEqual(running, []string{node}) {
		t.Fatalf("running: %v %v", running, err)
	}
	entries, err := inv.Catalogued(ctx)
	if err != nil {
		t.Fatal(err)
	}
	for _, e := range entries {
		switch e.Manifest.Module {
		case "postgres":
			if !reflect.DeepEqual(e.On, []string{node}) {
				t.Errorf("postgres is on %v", e.On)
			}
		case "web":
			if len(e.On) != 0 {
				t.Errorf("web is on %v", e.On)
			}
		}
	}
	if err := inv.ForgetModule(ctx, "postgres"); !errors.Is(err, ErrStillAssigned) {
		t.Fatalf("a module still assigned was forgotten: %v", err)
	}
	// A seat is held by an assignment, or by nothing.
	if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "web"); err == nil {
		t.Fatal("a seat was handed to a module that is not assigned")
	}
	if err := inv.HoldSeat(ctx, "mesh-broker", catalogue.ScopeMesh, node, "postgres"); err != nil {
		t.Fatalf("a seat handed to an assignment: %v", err)
	}

	before := generationNow(t, inv)
	if err := inv.Unassign(ctx, node, "postgres"); err != nil {
		t.Fatal(err)
	}
	if generationNow(t, inv) <= before {
		t.Fatal("an unassignment left the generation where it was")
	}
	// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
	gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
	if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
		t.Fatalf("the unassigned key: %+v %v %v", gone.Value, found, err)
	}
	if history, _ := inv.chain.Assignments.History(ctx, "anchor.postgres"); len(history) != 2 || !history[0].Value.Stands() ||
		history[1].Value.Stands() {
		t.Fatalf("the earlier values of the key: %+v", history)
	}
	if on, _ := inv.Assigned(ctx, node); len(on) != 0 {
		t.Fatalf("an ended assignment is still assigned: %v", on)
	}
	if running, _ := inv.Running(ctx, "postgres"); len(running) != 0 {
		t.Fatalf("an ended assignment is running: %v", running)
	}
	if held, err := inv.Holdings(ctx); err != nil || len(held) != 0 {
		t.Fatalf("the seat the assignment held is held by %+v: %v", held, err)
	}
	if err := inv.Unassign(ctx, node, "postgres"); err == nil || err.Error() != "postgres is not assigned to anchor" {
		t.Fatalf("an unassignment of nothing: %v", err)
	}
	if err := inv.ForgetModule(ctx, "postgres"); err != nil {
		t.Fatalf("a module nothing is assigned could not be forgotten: %v", err)
	}
	// Made again after it ended: a fresh value over the ended one, which stays in history.
	if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
		t.Fatal(added, err)
	}
	if added, err := inv.Assign(ctx, node, "web"); err != nil || added {
		t.Fatal(added, err)
	}
	if err := inv.Unassign(ctx, node, "web"); err != nil {
		t.Fatal(err)
	}
	if added, err := inv.Assign(ctx, node, "web"); err != nil || !added {
		t.Fatalf("assigned again after it ended: %v %v", added, err)
	}
	again, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web")
	if !again.Value.Stands() || !again.Value.Ended.IsZero() {
		t.Fatalf("the assignment made again is %+v", again.Value)
	}
	if history, _ := inv.chain.Assignments.History(ctx, "anchor.web"); len(history) != 3 || history[1].Value.Stands() {
		t.Fatalf("the values of the key: %+v", history)
	}
}

// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
// ends an assignment.
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
	inv, node := aNodeWithModules(t, "dunst", "web")
	ctx := t.Context()
	for _, m := range []string{"dunst", "web"} {
		if _, err := inv.Assign(ctx, node, m); err != nil {
			t.Fatal(err)
		}
	}
	// A build and a commit put in the value by another write.
	cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
	cur.Value.Build, cur.Value.Commit = "build-1", "c1"
	if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
		t.Fatal(err)
	}
	record, _ := inv.NodeByName(ctx, node)
	digest := "sha256:" + strings.Repeat("d", 64)
	plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
		Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
		Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
	raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
	if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
		t.Fatal(err)
	}
	inv.SetProvisioningPlans(inv.SentProvisioningPlans)
	written, removed, err := inv.KeepAssignmentPlans(ctx)
	if err != nil || written != 2 || removed != 0 {
		t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
	}
	dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
	if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
		t.Fatalf("dunst's value is %+v", dunst.Value)
	}
	if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
		t.Fatalf("web has a plan: %+v", web.Value)
	}
	notify, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify")
	if !found || notify.Value.RequiredBy != "desk" || !notify.Value.Assigned.IsZero() || notify.Value.Provisioning == nil {
		t.Fatalf("the pulled-in module's value is %+v %v", notify.Value, found)
	}
	if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
		t.Fatalf("a pulled-in module is assigned: %v", on)
	}
	if written, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || written != 0 || removed != 0 {
		t.Fatalf("a second pass: %d written, %d ended: %v", written, removed, err)
	}
	// Sent again without the pulled-in module: its value ends, the assignments stay.
	only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
	if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
		t.Fatal(err)
	}
	if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
		t.Fatalf("ended %d: %v", removed, err)
	}
	if gone, found, _ := inv.chain.Assignments.Get(ctx, "anchor.notify"); !found || gone.Value.Stands() || gone.Value.Ended.IsZero() {
		t.Fatalf("a module no longer pulled in has the value %+v, which should say it ended", gone.Value)
	}
	if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"dunst", "web"}) {
		t.Fatalf("assigned: %v", on)
	}
}

// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
// already is left as it is.
func TestTheMoveCarriesTheAssignmentsAndWhatEachMachineWasSent(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	for _, name := range []string{"anchor", "laptop"} {
		if _, err := inv.AddNode(ctx, name); err != nil {
			t.Fatal(err)
		}
	}
	for _, m := range []string{"notes", "wiki"} {
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
			t.Fatal(err)
		}
	}
	for _, a := range [][2]string{{"anchor", "notes"}, {"anchor", "wiki"}, {"laptop", "notes"}} {
		if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
			t.Fatal(err)
		}
	}
	aBuildRow(t, inv, "build-notes", "notes", "c-notes", "")
	anchor, _ := inv.NodeByName(ctx, "anchor")
	if err := inv.RecordSent(ctx, anchor.ID, "sent-1", map[string]string{"notes": "c-notes"}); err != nil {
		t.Fatal(err)
	}
	// The mirror of the store put a value under one key, with a plan.
	if _, err := inv.chain.Assignments.Create(ctx, "anchor.wiki", chain.Assignment{Node: "anchor", Module: "wiki",
		Assigned: time.Now().UTC(), Provisioning: json.RawMessage(`{"plan":1}`)}); err != nil {
		t.Fatal(err)
	}
	reports, err := inv.MoveChain(ctx, nil)
	if err != nil {
		t.Fatal(err)
	}
	var moved MoveReport
	for _, r := range reports {
		if r.Kind == KindAssignments {
			moved = r
		}
	}
	if moved.Rows != 3 || moved.Written+moved.Present != 3 {
		t.Fatalf("the move of the assignments reported %+v", moved)
	}
	notes, _, _ := inv.chain.Assignments.Get(ctx, "anchor.notes")
	if notes.Value.Commit != "c-notes" || notes.Value.Build != "build-notes" || notes.Value.Since.IsZero() || notes.Value.Assigned.IsZero() {
		t.Fatalf("anchor.notes moved as %+v", notes.Value)
	}
	if notes.Value.Artifact != "sha256:build-notes" {
		t.Fatalf("what the build published: %+v", notes.Value)
	}
	laptop, _, _ := inv.chain.Assignments.Get(ctx, "laptop.notes")
	if laptop.Value.Commit != "" || laptop.Value.Build != "" {
		t.Fatalf("a machine never sent has a build: %+v", laptop.Value)
	}
	wiki, _, _ := inv.chain.Assignments.Get(ctx, "anchor.wiki")
	if string(wiki.Value.Provisioning) != `{"plan":1}` {
		t.Fatalf("the plan the mirror put in the value was lost: %+v", wiki.Value)
	}
	// From now on the bucket is what is written.
	if _, err := inv.Assign(ctx, "laptop", "wiki"); err != nil {
		t.Fatal(err)
	}
	if n := rowsInTheTable(t, inv, "assignment"); n != "3" {
		t.Fatalf("the table took an assignment after the move: %s", n)
	}
	if on, _ := inv.Assigned(ctx, "laptop"); !reflect.DeepEqual(on, []string{"notes", "wiki"}) {
		t.Fatalf("assigned on laptop: %v", on)
	}
}

// hooked runs a function the first time a key is made in a bucket: the old controller writing the store between the move's
// first pass and its switch.
type hooked struct {
	chain.Backend
	once *sync.Once
	then func()
}

func (h hooked) Create(ctx context.Context, key string, value []byte) (uint64, error) {
	rev, err := h.Backend.Create(ctx, key, value)
	h.once.Do(h.then)
	return rev, err
}

// **A row the old controller changes after the move's first pass is in the bucket when the kind is recorded as moved**
// (ADR 0408 §8, issue 981 note 7 Q2): before the switch the store is the only truth, so under the lock the move reads the
// rows again and writes over every key that differs — a plan's state and a build's failure — and ends an assignment the
// store dropped. The count alone would have kept the older value and refused the next save of the plan.
func TestARowChangedAfterTheFirstPassIsInTheBucketWhenTheKindIsRecordedMoved(t *testing.T) {
	inv, finish := ForTestBefore(t, 105)
	finish()
	ctx := context.Background()
	aBuildRow(t, inv, "build-1", "notes", "c1", "")
	for _, n := range []string{"anchor"} {
		if _, err := inv.AddNode(ctx, n); err != nil {
			t.Fatal(err)
		}
	}
	for _, m := range []string{"notes", "wiki"} {
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
			t.Fatal(err)
		}
	}
	for _, m := range []string{"notes", "wiki"} {
		if _, err := inv.Assign(ctx, "anchor", m); err != nil {
			t.Fatal(err)
		}
	}
	p := aBatch("plan-1", time.Now().Add(-time.Hour).UTC())
	p.State, p.Tiers = PlanBuilding, [][]string{{"notes"}}
	if err := inv.SavePlan(ctx, p); err != nil {
		t.Fatal(err)
	}
	// Between a kind's first pass and its switch the old controller saves the plan again, fails the build and unassigns wiki:
	// each is done when the first key of its kind is made, so the first pass has read the rows as they were.
	after := func(sql string) hooked {
		return hooked{chain.NewMemory(), &sync.Once{}, func() { inv.ExecForTest(t, sql) }}
	}
	inv.UseChain(chain.New(
		after(`update build set failed = 'it broke later' where id = 'build-1'`),
		after(`update release_plan set state = 'rolling', revision = revision + 1, note = 'the old controller was here' where id = 'plan-1'`),
		after(`delete from assignment where module = 'wiki'`),
		func(context.Context) error { return nil }))
	if _, err := inv.MoveChain(ctx, nil); err != nil {
		t.Fatal(err)
	}
	now, err := inv.PlanByID(ctx, "plan-1")
	if err != nil {
		t.Fatal(err)
	}
	if now.State != PlanRolling || now.Note != "the old controller was here" || now.Revision != 2 {
		t.Fatalf("the bucket holds a plan from before the change: %s %q revision %d", now.State, now.Note, now.Revision)
	}
	if b, found, _ := inv.BuildByID(ctx, "build-1"); !found || b.Failed != "it broke later" {
		t.Fatalf("the bucket holds the build as it was: %+v", b)
	}
	if on, _ := inv.Assigned(ctx, "anchor"); !slices.Equal(on, []string{"notes"}) {
		t.Fatalf("anchor is assigned %v, though the store dropped wiki", on)
	}
	// The plan can be saved at the revision the store had.
	now.Note = "saved after the move"
	if err := inv.SavePlan(ctx, &now); err != nil {
		t.Fatalf("the next save of the plan is refused: %v", err)
	}
}

// sameEvent says two encodings of the forge's announcement are one: the store keeps it as jsonb, which spells it with its own
// white space, and whoever reads it (the cut, mesh-delivery) decodes it, so the white space is no part of what it says.
func sameEvent(a, b json.RawMessage) bool {
	var x, y bytes.Buffer
	return json.Compact(&x, a) == nil && json.Compact(&y, b) == nil && x.String() == y.String()
}