mesh-controller/internal/inventory/chain_rollouts_test.go

package inventory

import (
	"context"
	"errors"
	"sync/atomic"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/chain"
)

// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the
// store but for the durations a save measures, taken here by nothing.
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
	t.Helper()
	c := chain.InMemory(nil)
	inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
	inv.UseChain(c)
	t.Cleanup(inv.stopViews)
	return inv, c
}

var plansAt = time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)

func aBatch(id string, at time.Time) *Plan {
	return &Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c-" + id, Created: at, State: PlanAssembling,
		Tiers: [][]string{}, Modules: map[string]*PlanModule{}}
}

func aMerge(commit, plan string) BatchedMerge {
	return BatchedMerge{Repository: "Novox/Mesh-Catalog", Branch: "main", Commit: commit, Merged: plansAt, Heard: plansAt.Add(time.Second),
		Event: []byte(`{"commit":"` + commit + `"}`), Plan: plan}
}

// **A plan is kept under its id; a save of a copy another save has moved since is refused and writes nothing** (ADR 0408
// §2, issue 1092): the plan's own Revision counts its saves, 1 for the first.
func TestAPlanIsKeptUnderItsIdAndASaveOfAStaleCopyIsRefused(t *testing.T) {
	inv, c := anInventoryOnTheChain(t)
	ctx := context.Background()
	p := aBatch("plan-1", plansAt)
	if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
		t.Fatal(err)
	}
	if p.Revision != 1 {
		t.Fatalf("a new plan is at revision %d", p.Revision)
	}
	stale := *p
	p.Note = "moved on"
	if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil || p.Revision != 2 {
		t.Fatalf("the second save: revision %d, %v", p.Revision, err)
	}
	stale.Note = "laid over"
	if err := inv.chainSavePlans(ctx, c, []*Plan{&stale}, nil); !errors.Is(err, ErrPlanMoved) {
		t.Fatalf("a stale copy was saved: %v", err)
	}
	got, err := inv.chainPlanByID(ctx, c, "plan-1")
	if err != nil || got.Note != "moved on" || got.Revision != 2 {
		t.Fatalf("the plan is %+v: %v", got, err)
	}
	// A plan new to the bucket that is already there is refused too.
	again := aBatch("plan-1", plansAt)
	if err := inv.chainSavePlans(ctx, c, []*Plan{again}, nil); !errors.Is(err, ErrPlanMoved) {
		t.Fatalf("a new plan over a kept one: %v", err)
	}
	if _, err := inv.chainPlanByID(ctx, c, "plan-nobody"); err == nil || err.Error() != "no plan plan-nobody" {
		t.Fatalf("a plan nobody kept: %v", err)
	}
}

// **A save keeps the merges the record holds, not the ones its copy read** (issue 1092): a merge that moved into the record
// after the copy was read is not laid back by the copy's save.
func TestASaveKeepsTheMergesTheRecordHoldsAndNotTheCopys(t *testing.T) {
	inv, c := anInventoryOnTheChain(t)
	ctx := context.Background()
	batch := aBatch("plan-1", plansAt)
	if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
		t.Fatal(err)
	}
	if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || !added {
		t.Fatalf("added %v: %v", added, err)
	}
	// The copy was read before the merge came in; saving it neither fails nor loses the merge.
	batch.Note = "window closed"
	if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
		t.Fatalf("a copy read before a merge came in: %v", err)
	}
	merges, err := inv.chainMergesOf(ctx, c, "plan-1")
	if err != nil || len(merges) != 1 || merges[0].Commit != "m1" || merges[0].Repository != "novox/mesh-catalog" {
		t.Fatalf("the merges of the plan: %+v %v", merges, err)
	}
	if string(merges[0].Event) != `{"commit":"m1"}` || merges[0].Heard.IsZero() || merges[0].Plan != "plan-1" {
		t.Fatalf("the merge came back as %+v", merges[0])
	}
	// A merge heard is not heard twice, whatever the spelling of its repository.
	if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || added {
		t.Fatalf("added again %v: %v", added, err)
	}
}

// **A merge added to a plan that is not saved is written by that plan's save, and by nothing else** (issue 1092): until
// then it is in no record — so the event that brought it is handed over again if the save fails.
func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) {
	inv, c := anInventoryOnTheChain(t)
	ctx := context.Background()
	if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-new")); err != nil || !added {
		t.Fatalf("added %v: %v", added, err)
	}
	if _, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); known {
		t.Fatal("a merge for a plan not saved is in a record")
	}
	// The save is refused: nothing is written, the merge is still carried.
	held := false
	away := chain.New(chain.NewMemory(), chain.NewMemory(), chain.NewMemory(), func(context.Context) error {
		if held {
			return nil
		}
		return errors.New("the lease is not held")
	})
	other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
	other.UseChain(away)
	t.Cleanup(other.stopViews)
	other.moved.pending = inv.moved.pending
	p := aBatch("plan-new", plansAt)
	if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) {
		t.Fatalf("the save answered %v", err)
	}
	if _, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); known {
		t.Fatal("a refused save left the merge in a record")
	}
	held = true
	if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); err != nil {
		t.Fatal(err)
	}
	if m, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); !known || m.Plan != "plan-new" {
		t.Fatalf("the save did not write the merge: %+v %v", m, known)
	}
}

// failingOn is a bucket that refuses to write one key: the second half of a move, interrupted.
type failingOn struct {
	chain.Backend
	key  string
	fail *atomic.Bool
}

func (f failingOn) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
	if key == f.key && f.fail.Load() {
		return 0, errors.New("the bus went away")
	}
	return f.Backend.Update(ctx, key, value, revision)
}

// **A merge moved to a plan is written to the receiving record first, then taken from the giving one; one found in both is
// read from the one it entered last and repaired** (issue 1092, ADR 0299 §7).
func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *testing.T) {
	c := chain.InMemory(nil)
	var fail atomic.Bool
	rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail}
	c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })
	inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
	inv.UseChain(c)
	t.Cleanup(inv.stopViews)
	ctx := context.Background()
	a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
	for _, p := range []*Plan{a, b} {
		if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
		t.Fatal(err)
	}

	// The move stops before the giving record is written: the merge is in both.
	fail.Store(true)
	err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b"}})
	if err == nil {
		t.Fatal("the move was said to be made with the giving record unwritten")
	}
	fail.Store(false)
	if m, _, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); m.Plan != "plan-b" {
		t.Fatalf("a merge in two records is read from %q, not the one it moved to", m.Plan)
	}
	if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
		t.Fatalf("the giving plan still answers %+v", got)
	}
	// The next cut repairs it.
	v, _ := inv.rolloutView(ctx, c)
	if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 {
		t.Fatalf("the merge is held by %d records, want the two of a stopped move", n)
	}
	inv.moved.mu.Lock()
	inv.moved.kind = map[string]bool{KindRollouts: true}
	inv.moved.mu.Unlock()
	repaired := 0
	for k, copies := range mergeHolders(v.list()) {
		if len(copies) < 2 {
			continue
		}
		w := winner(copies)
		for _, h := range copies {
			if h.plan != w.plan {
				key := k
				if _, err := inv.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
					return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
				}); err != nil {
					t.Fatal(err)
				}
				repaired++
			}
		}
	}
	if repaired != 1 {
		t.Fatalf("repaired %d", repaired)
	}
	waitFor(t, func() bool {
		v, _ := inv.rolloutView(ctx, c)
		return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
	})
}

// waitFor polls a condition that a watch makes true, for a moment.
func waitFor(t *testing.T, ok func() bool) {
	t.Helper()
	for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
		if ok() {
			return
		}
	}
	t.Fatal("the condition did not come true")
}

// **A merge no plan answers stays, flagged, in the record that held it last, and is walked alone** (issue 1092): not in
// the merges of the plan, listed among the merges waiting, and answered again by the plan it is moved to.
func TestAWaitingMergeStaysInTheRecordThatHeldItLast(t *testing.T) {
	inv, c := anInventoryOnTheChain(t)
	ctx := context.Background()
	a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
	for _, p := range []*Plan{a, b} {
		if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
		t.Fatal(err)
	}
	if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "", Alone: true}}); err != nil {
		t.Fatal(err)
	}
	if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
		t.Fatalf("a waiting merge is answered by %+v", got)
	}
	alone, err := inv.chainAloneMerges(ctx, c)
	if err != nil || len(alone) != 1 || alone[0].Commit != "m1" || alone[0].Plan != "" || !alone[0].Alone {
		t.Fatalf("waiting: %+v %v", alone, err)
	}
	if m, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); !known || m.Plan != "" {
		t.Fatalf("a waiting merge reads as %+v %v", m, known)
	}
	if later, _ := inv.chainLaterMergesOf(ctx, c, "novox/mesh-catalog", "main", plansAt.Add(-time.Hour)); len(later) != 0 {
		t.Fatalf("a waiting merge is among those a plan answers: %+v", later)
	}
	if kept, first, _ := inv.chainKeptOnBranch(ctx, c, "novox/mesh-catalog", "main"); !kept || first.IsZero() {
		t.Fatalf("a waiting merge is not kept on its branch: %v %v", kept, first)
	}
	// Walked alone: the plan it goes to answers it, and the record that held it lets it go.
	if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b", Alone: true}}); err != nil {
		t.Fatal(err)
	}
	if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 || !got[0].Alone {
		t.Fatalf("plan-b answers %+v", got)
	}
	if left, _ := inv.chainAloneMerges(ctx, c); len(left) != 0 {
		t.Fatalf("still waiting: %+v", left)
	}
	waitFor(t, func() bool {
		v, _ := inv.rolloutView(ctx, c)
		return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
	})
	// A move to a plan with no record is refused and takes nothing from the one that holds the merge.
	err = inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-nowhere"}})
	if !errors.Is(err, ErrMergeUnsaved) {
		t.Fatalf("a move to a plan with no record: %v", err)
	}
	if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 {
		t.Fatalf("the merge was taken from plan-b: %+v", got)
	}
}

// countingBackend counts the reads a bucket answers.
type countingBackend struct {
	chain.Backend
	gets, keys atomic.Int32
}

func (c *countingBackend) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
	c.gets.Add(1)
	return c.Backend.Get(ctx, key)
}

func (c *countingBackend) Keys(ctx context.Context) ([]string, error) {
	c.keys.Add(1)
	return c.Backend.Keys(ctx)
}

// **The surveys are read from a view kept by one watch, never from the bucket key by key** (issue 1092): the plans and the
// merges are asked for many times and the bucket is read for none of them; a write by another process reaches the view
// through its watch.
func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
	counting := &countingBackend{Backend: chain.NewMemory()}
	c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil })
	inv := &Inventory{moved: &movedKinds{}, recorded: func(context.Context, Duration) error { return nil }}
	inv.UseChain(c)
	t.Cleanup(inv.stopViews)
	ctx := context.Background()
	for k, id := range []string{"plan-1", "plan-2", "plan-3"} {
		p := aBatch(id, plansAt.Add(time.Duration(k)*time.Minute))
		if k > 0 {
			p.State = PlanBuilding
		}
		if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
			t.Fatal(err)
		}
	}
	before := counting.gets.Load()
	for n := 0; n < 20; n++ {
		open, err := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
		if err != nil || len(open) != 2 || open[0].ID != "plan-2" {
			t.Fatalf("open plans %+v: %v", open, err)
		}
		if _, err := inv.chainAloneMerges(ctx, c); err != nil {
			t.Fatal(err)
		}
	}
	if counting.gets.Load() != before || counting.keys.Load() != 0 {
		t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load())
	}
	// Another process writes; the view follows.
	other := &Inventory{moved: &movedKinds{}, recorded: inv.recorded}
	other.UseChain(c)
	t.Cleanup(other.stopViews)
	late := aBatch("plan-4", plansAt.Add(time.Hour))
	late.State = PlanRolling
	if err := other.chainSavePlans(ctx, c, []*Plan{late}, nil); err != nil {
		t.Fatal(err)
	}
	waitFor(t, func() bool {
		open, _ := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
		return len(open) == 3
	})
}