Keep a no-holder machine as ADR 0404 says; prove the cut takes the backups before its first tier (issue 1086, review round 1)

cmd/mesh-controller/backups_before_send_integration_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"encoding/json"
+	"os"
 	"strings"
 	"sync"
 	"testing"
@@ -217,3 +218,91 @@ func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItA
 		t.Fatalf("the condition stays: %+v", said)
 	}
 }
+
+// Issue 1086: a plan cut from a merge asked its first tier at the cut, so a module keeping data was sent without the
+// backup the plan promised. The cut now takes the backups first (askFirstTierAtCut, batches.go), and the walk's own
+// step asks the tier once they are done.
+func TestAPlanCutFromAMergeOfABackedUpModuleAsksItsBackupAndNotItsTierUntilItIsDone(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	settings := t.TempDir() + "/merge-window.json"
+	if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv(mergeWindowFileVar, settings)
+	everyHeadWaits(t)
+	asked := asksRecorded(t)
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	before := handActConn
+	handActConn = js.Conn()
+	t.Cleanup(func() { handActConn = before })
+
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
+		Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
+		inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
+		t.Fatal(err)
+	}
+	notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
+		`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
+		`"resources":[{"id":"files","type":"directory"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/notes", Seat: "git", Ref: "main",
+		BuiltFrom: "c0", Head: "c0"}); err != nil {
+		t.Fatal(err)
+	}
+	for _, m := range []string{"keeper", "notes"} {
+		if _, err := assign(ctx, open, "anchor", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+	seat := &aBackupSeatOnTheBus{}
+	seat.serve(t, js.Conn(), "anchor")
+	wasWithin, wasEvery := backupsWithin, backupsPollEvery
+	backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
+	t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
+
+	// 1. A merge of notes is heard and its window closes: the batch is cut through cutBatch.
+	hear(t, open, repoMerge("notes", "c-notes", t0), t0.Add(time.Second))
+	cutAt(t, open, t0.Add(time.Second+90*time.Second))
+	ws, _ := walks(t, open)
+	if len(ws) != 1 {
+		t.Fatalf("%d walk(s)", len(ws))
+	}
+	// 2. Right after the cut no tier is asked, the record has anchor/notes pending, the seat heard one `now`.
+	cut, err := inv.PlanByID(ctx, ws[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(*asked) != 0 || askedInTier(cut) {
+		t.Fatalf("the first tier was asked at the cut: %v %+v", *asked, cut.Modules)
+	}
+	if cut.Backups == nil || cut.Backups.State != inventory.BackupPending || len(cut.Backups.Asked) != 1 ||
+		cut.Backups.Asked[0].Machine != "anchor" || cut.Backups.Asked[0].Module != "notes" {
+		t.Fatalf("the backups of the cut: %+v", cut.Backups)
+	}
+	if seat.asks() != 1 {
+		t.Fatalf("the seat heard %d `now`", seat.asks())
+	}
+	// 3. The backup answers done: the walk's own step asks the first tier, once, and asks no second backup.
+	seat.answers("notes", true, "")
+	advancePlans(ctx, open)
+	advancePlans(ctx, open)
+	after, err := inv.PlanByID(ctx, ws[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(*asked) != 1 || after.Backups.State != inventory.BackupDone || seat.asks() != 1 {
+		t.Fatalf("builds=%v backups=%+v asks=%d", *asked, after.Backups, seat.asks())
+	}
+}