Keep rollouts and the merges they answer in the rollouts bucket (issue 1092)

Once chain_moved records the kind, a plan is a key of the rollouts bucket under its id and the open batch is the same key in state assembling. A save refuses a copy another save moved since (the plan's own Revision, kept in its value, counts saves) and keeps the merges the record holds, so a copy read before a merge moved cannot lay it back. The merges a batch or walk answers are the record's held field, with the columns the table had; a merge no plan answers stays, flagged, in the record that held it last, or in the record kept for those. A merge moved to a plan is written to the receiving record first and then taken from the giving one; one found in both is read from the one it entered last and repaired at the next cut. Plans and merges are read from a view kept by one watch of the bucket. A bus that does not answer, a lease lost and a key moved are now not-now like the store being away, so a merge whose plan could not be written is held and handed over again, never acknowledged. The start moves release_plan and batched_merge into the bucket, one kind, counted against both tables.
2026-10-18 03:06:50 +02:00
parent e0fb241263
commit 275ddd76e8
21 changed files with 1939 additions and 40 deletions
@@ -219,10 +219,10 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
 		}
 		if p.Batch() {
 			if err := keepBatch(ctx, inv, &p, now, ""); err != nil {
-				return notNow(err)
+				return unsaved(err)
 			}
 		} else if err := nameMerges(ctx, inv, &p); err != nil {
-			return notNow(err)
+			return unsaved(err)
 		}
 		fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+
 			"answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p))
@@ -239,7 +239,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
 	}
 	holdFor(&batch, m.Head, waits, holding, own, now)
 	if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
-		return notNow(err)
+		return unsaved(err)
 	}
 	fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID,
 		batchWords(batch, now))
@@ -820,6 +820,12 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan
 // walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits.
 func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
 	inv := open.inventory
+	// A merge found in two records is the half-done move of a crash: put right before anything is decided from it.
+	if repaired, err := inv.RepairMerges(ctx); err != nil {
+		return err
+	} else if repaired > 0 {
+		fmt.Printf("%d merge(s) found in two plans, the half of a move a restart interrupted, are kept by the plan they were moved to\n", repaired)
+	}
 	if err := searchAfterFailures(ctx, inv); err != nil {
 		return err
 	}
@@ -0,0 +1,71 @@
+//go:build integration
+
+package main
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/chain"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// refusing is a bucket that refuses to write while it is told to.
+type refusing struct {
+	chain.Backend
+	refuse atomic.Bool
+}
+
+func (r *refusing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
+	if r.refuse.Load() {
+		return 0, nats.ErrTimeout
+	}
+	return r.Backend.Create(ctx, key, value)
+}
+
+func (r *refusing) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
+	if r.refuse.Load() {
+		return 0, nats.ErrTimeout
+	}
+	return r.Backend.Update(ctx, key, value, revision)
+}
+
+// **A merge whose plan cannot be saved is held, recorded nowhere, and recorded when it is handed over again** (ADR 0407
+// decision 3, issue 1092): hearMerge answers ErrTryAgain — the bus leaves the announcement unacknowledged and hands it
+// over again — and the merge is in no record until the plan holding it is written.
+func TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) {
+	open := windowed(t)
+	everyHeadWaits(t)
+	asksWithPaths(t)
+	ctx := t.Context()
+	rollouts := &refusing{Backend: chain.NewMemory()}
+	open.inventory.UseChain(chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }))
+	merge := catalogueMerge("cat1aaaaaaaa", "app", t0)
+
+	rollouts.refuse.Store(true)
+	err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second))
+	if !errors.Is(err, link.ErrTryAgain) {
+		t.Fatalf("a merge whose plan could not be saved answered %v, which the bus takes as done", err)
+	}
+	if _, known, err := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); err != nil || known {
+		t.Fatalf("the merge is kept though its plan was not: %v %v", known, err)
+	}
+	if _, bs := walks(t, open); len(bs) != 0 {
+		t.Fatalf("a batch was kept though its write failed: %+v", bs)
+	}
+
+	rollouts.refuse.Store(false)
+	hear(t, open, merge, t0.Add(10*time.Second))
+	_, bs := walks(t, open)
+	if len(bs) != 1 {
+		t.Fatalf("the merge handed over again made %d batch(es)", len(bs))
+	}
+	if m, known, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); !known || m.Plan != bs[0].ID {
+		t.Fatalf("the merge is %+v %v, and the batch %s", m, known, bs[0].ID)
+	}
+}
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **A merge whose plan could not be saved is held, never taken** (novox/hq ADR 0407 decision 3, issue 1092): the merge is
+// carried until its plan is saved, and the bus acknowledges the announcement only when the controller answers without
+// ErrTryAgain (link.Server.sourceMoved), so a failed save of any kind must answer it.
+func TestAFailureToSaveAMergesPlanIsHeldAndNotTaken(t *testing.T) {
+	cause := errors.New("the plan for novox/mesh-catalog is not written: the plan was written by somebody else since it was read")
+	held := unsaved(cause)
+	if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) {
+		t.Fatalf("a failed save answers %v", held)
+	}
+	if unsaved(nil) != nil {
+		t.Fatal("a save that worked answers an error")
+	}
+	// Not the same as notNow, which holds a merge only for a store that is away.
+	if errors.Is(notNow(cause), link.ErrTryAgain) {
+		t.Fatal("notNow holds every failure; the premise of this test is gone")
+	}
+}
@@ -303,6 +303,16 @@ func notNow(err error) error {
 	return err
 }

+// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save
+// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407
+// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded.
+func unsaved(err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err)
+}
+
 // SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
 // becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
 func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {