Keep a plan's backups in the store: a migration and the plan's read and write (issues 1065, 1066)

cmd/mesh-controller/backups_before_send_integration_test.go

@@ -0,0 +1,219 @@
+//go:build integration
+
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and
+// answers `backed-up` with the last night it is told (novox/hq ADR 0404).
+type aBackupSeatOnTheBus struct {
+	mu    sync.Mutex
+	nows  []string
+	night map[string]any // module → lastNight
+}
+
+func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) {
+	t.Helper()
+	reply := func(m *nats.Msg, result any) {
+		body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node})
+		_ = m.Respond(body)
+	}
+	now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) {
+		var a struct {
+			Module string `json:"module"`
+		}
+		_ = json.Unmarshal(m.Data, &a)
+		f.mu.Lock()
+		f.nows = append(f.nows, a.Module)
+		f.mu.Unlock()
+		reply(m, map[string]any{"started": a.Module})
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		var out []map[string]any
+		for module, n := range f.night {
+			out = append(out, map[string]any{"module": module, "lastNight": n})
+		}
+		reply(m, out)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() })
+	if err := conn.Flush(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}}
+}
+
+func (f *aBackupSeatOnTheBus) asks() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.nows)
+}
+
+// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits
+// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the
+// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the
+// backups again, and the plan goes on. One bus and one store for the scenarios.
+func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	asked := asksRecorded(t)
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	before := handActConn
+	handActConn = js.Conn()
+	t.Cleanup(func() { handActConn = before })
+
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
+		Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
+	notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
+		`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
+		`"resources":[{"id":"files","type":"directory"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
+		Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
+		t.Fatal(err)
+	}
+	for _, node := range []string{"anchor", "laptop"} {
+		if _, err := assign(ctx, open, node, "keeper"); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := assign(ctx, open, node, "notes"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}}
+	for node, s := range seats {
+		s.serve(t, js.Conn(), node)
+	}
+	// Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup.
+	wasWithin, wasEvery := backupsWithin, backupsPollEvery
+	backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
+	t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
+
+	newPlan := func(id string) {
+		t.Helper()
+		p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
+			State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}}
+		if err := inv.SavePlan(ctx, &p); err != nil {
+			t.Fatal(err)
+		}
+	}
+	read := func(id string) inventory.Plan {
+		t.Helper()
+		p, err := inv.PlanByID(ctx, id)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return p
+	}
+
+	// 1. Both machines are asked; while one has not answered, no build is asked.
+	newPlan("plan-one")
+	advancePlans(ctx, open)
+	if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 {
+		t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked)
+	}
+	seats["anchor"].answers("notes", true, "")
+	advancePlans(ctx, open)
+	if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending {
+		t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups)
+	}
+	seats["laptop"].answers("notes", true, "")
+	advancePlans(ctx, open)
+	if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone {
+		t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups)
+	}
+	finish := read("plan-one")
+	finish.State = inventory.PlanDone
+	if err := inv.SavePlan(ctx, &finish); err != nil {
+		t.Fatal(err)
+	}
+
+	// 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised.
+	seats["anchor"].night, seats["laptop"].night = nil, nil
+	newPlan("plan-two")
+	advancePlans(ctx, open)
+	seats["anchor"].answers("notes", true, "")
+	seats["laptop"].answers("notes", false, "no space left on device")
+	advancePlans(ctx, open)
+	held := read("plan-two")
+	if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 {
+		t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked)
+	}
+	var failed *inventory.PlanBackup
+	for i := range held.Backups.Asked {
+		if held.Backups.Asked[i].State == inventory.BackupFailed {
+			failed = &held.Backups.Asked[i]
+		}
+	}
+	if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" ||
+		strings.Contains(held.Note, "tried again") {
+		t.Fatalf("the failed backup: %+v note=%q", failed, held.Note)
+	}
+	advancePlans(ctx, open) // not retried on the next look
+	if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 ||
+		seats["laptop"].asks() != 2 {
+		t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks())
+	}
+	said, err := probeBackupHeld(ctx, &doctor{open: open})
+	if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") {
+		t.Fatalf("the condition: %+v %v", said, err)
+	}
+
+	// 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on.
+	seats["laptop"].answers("notes", false, "no space left on device")
+	if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by",
+		"mesh-delivery for jochen"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil {
+		t.Fatal("a plan with no failed backup was asked again")
+	}
+	seats["laptop"].night = nil
+	advancePlans(ctx, open)
+	if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 {
+		t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks())
+	}
+	seats["laptop"].answers("notes", true, "")
+	advancePlans(ctx, open)
+	if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 2 {
+		t.Fatalf("the plan did not go on: %+v builds=%v", got.Backups, *asked)
+	}
+	if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
+		t.Fatalf("the condition stays: %+v", said)
+	}
+}

cmd/mesh-controller/delivery.go

@@ -884,6 +884,19 @@ func deliveryCommand(ctx context.Context, args []string) error {
 		fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+
 			"is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers))
 		return nil
+	case "backups-again":
+		if len(positionals) != 1 || strings.TrimSpace(*why) == "" {
+			return errors.New("delivery backups-again <plan> --why <text> [--by <who>]")
+		}
+		var p inventory.Plan
+		var again []inventory.PlanBackup
+		if err := sayingOnTheBus(ctx, inv, func() (err error) {
+			p, again, err = backupsAgain(ctx, inv, positionals[0], *by, strings.TrimSpace(*why))
+			return err
+		}); err != nil {
+			return err
+		}
+		return answer(map[string]any{"plan": p.ID, "asked_again": again})
 	case "walks":
 		var walks []inventory.Plan
 		if *planID != "" {
@@ -924,7 +937,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
 		return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
 			"own-path": sortedKeysOf(ownPathWords())})
 	}
-	return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
+	return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub)
 }
 
 // appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations

cmd/mesh-controller/doctor.go

@@ -174,6 +174,10 @@ var probeRegistry = []probe{
 	{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
 		"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
 		Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
+	// A delivery plan held for a backup that failed or stayed silent (ADR 0404 decision 4): one condition per plan.
+	{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
+		"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
+		run: probeBackupHeld},
 }
 
 // probeVerdict is one probe's outcome in a run.

cmd/mesh-controller/plain_words.go

@@ -603,6 +603,13 @@ var plainWordings = map[string]func(conditions.Observation) words{
 			Explanation: "The upgrade of the mesh's message system did not end healthy in time.",
 			Resolved:    "Resolved: the bus is healthy after its upgrade"}
 	}),
+	kindBackupHeld: worded(func(o conditions.Observation) words {
+		return words{Headline: "A delivery waits for a backup that failed",
+			Needs: "fix what stopped the backup, then release the delivery.",
+			Explanation: "Before a delivery moves a module that keeps data, that data is backed up. A backup failed or " +
+				"did not answer in ten minutes, so nothing of the delivery is built or sent.",
+			Resolved: "Resolved: the backup is done and the delivery goes on"}
+	}),
 	kindConsumerBehind: worded(func(o conditions.Observation) words {
 		return words{Headline: "Messages pile up for a listener",
 			Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +

cmd/mesh-controller/plan_backups.go

@@ -0,0 +1,453 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **A backup before the send** (novox/hq ADR 0404 decisions 2 and 3, design 47, issue 1065). Once a delivery plan
+// has started and before it asks its first tier, the controller asks the node-backup holder of every machine that
+// runs a moved module with an item protected by a backup to back that module up now, and asks nothing of a builder
+// and sends nothing until every one of them answered done. The plan keeps the record (inventory.PlanBackups).
+
+// backupsWithin is how long a backup is given from its ask to say it is done, and backupsPollEvery how soon the plan
+// is looked at again while it waits: variables so a test waits for neither.
+var (
+	backupsWithin    = 10 * time.Minute
+	backupsPollEvery = 10 * time.Second
+)
+
+// night is one module's last night as a holder's `backed-up` says it (lastNight).
+type night struct {
+	OK    bool      `json:"ok"`
+	At    time.Time `json:"at"`
+	Error string    `json:"error,omitempty"`
+}
+
+// askBackupNow asks a machine's backup holder to back a module up now; the holder answers when it has started. A
+// variable so a test needs no bus. An error is the holder's own words when it refused, or why it could not be asked.
+var askBackupNow = func(ctx context.Context, machine, module string) error {
+	return onTheBus(func(conn *nats.Conn) error {
+		answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", machine,
+			map[string]any{"module": module}, 30*time.Second)
+		if err != nil {
+			return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: %w", machine, err)}
+		}
+		if answer.Error != "" {
+			return fmt.Errorf("%s", answer.Error)
+		}
+		return nil
+	})
+}
+
+// backupNotAsked is an ask that could not be made, as opposed to one the holder refused: the step is tried again.
+type backupNotAsked struct{ error }
+
+func (e *backupNotAsked) Unwrap() error { return e.error }
+
+// readLastNights reads a machine's holder's `backed-up` by module: how each module's last backup went. A variable
+// so a test needs no bus.
+var readLastNights = func(ctx context.Context, machine string) (map[string]*night, error) {
+	var out map[string]*night
+	err := onTheBus(func(conn *nats.Conn) error {
+		answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", machine, map[string]any{}, 10*time.Second)
+		if err != nil {
+			return err
+		}
+		if answer.Error != "" {
+			return fmt.Errorf("%s", answer.Error)
+		}
+		var read []struct {
+			Module    string `json:"module"`
+			LastNight *night `json:"lastNight"`
+		}
+		if err := json.Unmarshal(answer.Result, &read); err != nil {
+			return fmt.Errorf("its answer is not readable: %w", err)
+		}
+		out = map[string]*night{}
+		for _, m := range read {
+			out[m.Module] = m.LastNight
+		}
+		return nil
+	})
+	return out, err
+}
+
+// backupFacts are what the step needs of the mesh: the modules with an item protected by a backup, where each
+// runs, and which machines hold node-backup.
+type backupFacts struct {
+	backedUp map[string][]string // module → machines running it
+	holder   map[string]bool
+}
+
+// backupFactsFrom reads the facts from the catalogue's entries and the seat holdings. A module is backed up when
+// one of its own items is (catalogue.DataItem.BackedUp: never a cache, `backup: none` or redundancy alone). A
+// module whose policy records (ADR 0377) is decided per machine after the build, so every machine running it is
+// asked: a superset costs time and never leaves a send without its restore point.
+func backupFactsFrom(entries []inventory.Entry, held []catalogue.Held) backupFacts {
+	f := backupFacts{backedUp: map[string][]string{}, holder: map[string]bool{}}
+	for _, e := range entries {
+		for _, it := range e.Manifest.DataItems() {
+			if it.BackedUp() {
+				f.backedUp[e.Manifest.Module] = slices.Clone(e.On)
+				break
+			}
+		}
+	}
+	for _, h := range held {
+		if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat {
+			f.holder[h.Node] = true
+		}
+	}
+	return f
+}
+
+// loadBackupFacts reads the facts from the inventory. The holders are read as `seats` reads them, from the world
+// every machine's composition makes (theRestOfTheMesh): the seat_holding table keeps one holder of a node seat, and
+// node-backup is held on every machine.
+func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts, error) {
+	return loadBackupFactsFrom(inv.Catalogued, func() ([]catalogue.Held, error) {
+		shelf, err := inv.Catalogue(ctx)
+		if err != nil {
+			return nil, err
+		}
+		world, err := theRestOfTheMesh(ctx, inv, shelf, "")
+		if err != nil {
+			return nil, err
+		}
+		return world.Held, nil
+	}, ctx)
+}
+
+// loadBackupFactsFrom is loadBackupFacts given where the entries and the holdings come from. A source that fails
+// fails the step, to be tried again: a failure never reads as a machine with no holder, which would send without a
+// backup.
+func loadBackupFactsFrom(entries func(context.Context) ([]inventory.Entry, error),
+	held func() ([]catalogue.Held, error), ctx context.Context) (backupFacts, error) {
+	es, err := entries(ctx)
+	if err != nil {
+		return backupFacts{}, fmt.Errorf("the backups of the plan could not be worked out: %w", err)
+	}
+	h, err := held()
+	if err != nil {
+		return backupFacts{}, fmt.Errorf("the backups of the plan could not be worked out: who holds %s: %w",
+			catalogue.BackupSeat, err)
+	}
+	return backupFactsFrom(es, h), nil
+}
+
+// stepBackups takes one step of a plan's backups and says whether the plan may go on to ask its first tier. It
+// asks what is not asked, reads how the asked ones went, and records it all in p.Backups; it waits (false, nil)
+// while any is pending, and stays stopped (false, nil) once one failed, which part B holds and says. A plan whose
+// moved modules have no backed-up item keeps no record and goes on at once. A holder that cannot be asked is tried again
+// within the bound and then failed; an error is left for what the step itself cannot do.
+func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) {
+	proceed, err := stepBackupsOnce(ctx, p, facts, now)
+	if err == nil {
+		noteBackups(p)
+	}
+	return proceed, err
+}
+
+func stepBackupsOnce(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) {
+	if p.Backups == nil {
+		b := wantedBackups(*p, facts)
+		if b == nil {
+			return true, nil
+		}
+		p.Backups = b
+	}
+	b := p.Backups
+	switch b.State {
+	case inventory.BackupDone:
+		return true, nil
+	case inventory.BackupFailed:
+		return false, nil
+	}
+	nights := map[string]map[string]*night{}
+	for i := range b.Asked {
+		a := &b.Asked[i]
+		if a.State != inventory.BackupPending {
+			continue
+		}
+		if a.AskedAt == nil {
+			asked := now
+			if a.TriedAt == nil {
+				a.TriedAt = &asked
+			}
+			if err := askBackupNow(ctx, a.Machine, a.Module); err != nil {
+				var unasked *backupNotAsked
+				if errors.As(err, &unasked) {
+					// A holder off the bus is tried again within the bound, by the look and never by the
+					// "tried again" error path; past it the backup is failed and the plan held.
+					a.Words = "could not be asked: " + err.Error()
+					if now.Sub(*a.TriedAt) >= backupsWithin {
+						a.State = inventory.BackupFailed
+					}
+					continue
+				}
+				a.State, a.Words = inventory.BackupFailed, err.Error()
+				a.AskedAt, a.TriedAt = &asked, nil
+				continue
+			}
+			a.AskedAt, a.TriedAt, a.Words = &asked, nil, ""
+			continue
+		}
+		if _, read := nights[a.Machine]; !read {
+			n, err := readLastNights(ctx, a.Machine)
+			if err != nil {
+				n = nil // not read now: the bound below still ends the wait
+			}
+			nights[a.Machine] = n
+		}
+		if n := nights[a.Machine][a.Module]; n != nil && n.At.After(*a.AskedAt) {
+			if n.OK {
+				a.State, a.Words = inventory.BackupDone, ""
+			} else {
+				a.State, a.Words = inventory.BackupFailed, n.Error
+				if a.Words == "" {
+					a.Words = "the backup failed and did not say why"
+				}
+			}
+			continue
+		}
+		if now.Sub(*a.AskedAt) >= backupsWithin {
+			a.State = inventory.BackupFailed
+			a.Words = "did not answer in " + wordsOfBound(backupsWithin)
+		}
+	}
+	b.State = backupsStateOf(b.Asked)
+	switch b.State {
+	case inventory.BackupDone:
+		return true, nil
+	case inventory.BackupPending:
+		lookAgainForBackups(p.ID)
+	}
+	return false, nil
+}
+
+// wantedBackups is the record of backups a plan has to take, or nil for none: one entry per machine with a holder
+// running a moved module that has a backed-up item, and the machines with none.
+func wantedBackups(p inventory.Plan, facts backupFacts) *inventory.PlanBackups {
+	var asked []inventory.PlanBackup
+	noHolder := map[string]bool{}
+	for _, tier := range p.Tiers {
+		for _, module := range tier {
+			for _, machine := range facts.backedUp[module] {
+				if !facts.holder[machine] {
+					noHolder[machine] = true
+					continue
+				}
+				asked = append(asked, inventory.PlanBackup{Machine: machine, Module: module, State: inventory.BackupPending})
+			}
+		}
+	}
+	if len(asked) == 0 && len(noHolder) == 0 {
+		return nil
+	}
+	sort.SliceStable(asked, func(i, j int) bool {
+		if asked[i].Machine != asked[j].Machine {
+			return asked[i].Machine < asked[j].Machine
+		}
+		return asked[i].Module < asked[j].Module
+	})
+	b := &inventory.PlanBackups{Asked: asked}
+	for m := range noHolder {
+		b.NoHolder = append(b.NoHolder, m)
+	}
+	sort.Strings(b.NoHolder)
+	b.State = backupsStateOf(asked)
+	return b
+}
+
+// backupsStateOf is the whole of the backups' states: failed if any failed, done if all are (or none are asked).
+func backupsStateOf(asked []inventory.PlanBackup) string {
+	state := inventory.BackupDone
+	for _, a := range asked {
+		switch a.State {
+		case inventory.BackupFailed:
+			return inventory.BackupFailed
+		case inventory.BackupPending:
+			state = inventory.BackupPending
+		}
+	}
+	return state
+}
+
+func wordsOfBound(d time.Duration) string {
+	if d == 10*time.Minute {
+		return "ten minutes"
+	}
+	if d%time.Minute == 0 {
+		return fmt.Sprintf("%d minutes", int(d/time.Minute))
+	}
+	return d.String()
+}
+
+var (
+	backupLookMu sync.Mutex
+	backupLooks  = map[string]bool{}
+)
+
+// lookAgainForBackups has the serving controller look at the plans again soon, once per waiting plan: a backup ending
+// is no event the controller hears, so the wait is looked at by a timer, never by blocking the hold on the plans.
+func lookAgainForBackups(plan string) {
+	backupLookMu.Lock()
+	if backupLooks[plan] {
+		backupLookMu.Unlock()
+		return
+	}
+	backupLooks[plan] = true
+	backupLookMu.Unlock()
+	time.AfterFunc(backupsPollEvery, func() {
+		backupLookMu.Lock()
+		delete(backupLooks, plan)
+		backupLookMu.Unlock()
+		askToLook()
+	})
+}
+
+// backupsFirst is the step advanceOnce takes before it asks a walk's first tier: whether the plan may go on.
+func backupsFirst(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) (bool, error) {
+	facts := backupFacts{}
+	if p.Backups == nil {
+		f, err := loadBackupFacts(ctx, inv)
+		if err != nil {
+			return false, err
+		}
+		facts = f
+	}
+	return stepBackups(ctx, p, facts, time.Now().UTC())
+}
+
+// backupsNote begins the note of a plan that waits on or is held by its backups.
+const backupsNote = "backups: "
+
+// noteBackups says in the plan's note what its backups wait on, or, held, which backup failed and in what words and
+// the verb that asks again; a note of its own that the backups ended is cleared. Not the "tried again" note of a
+// failed step: a held plan is not retried (novox/hq ADR 0404 decision 4).
+func noteBackups(p *inventory.Plan) {
+	if p.Backups == nil {
+		return
+	}
+	var pending, failed []string
+	for _, a := range p.Backups.Asked {
+		switch a.State {
+		case inventory.BackupPending:
+			pending = append(pending, a.Module+" on "+a.Machine)
+		case inventory.BackupFailed:
+			failed = append(failed, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words))
+		}
+	}
+	switch p.Backups.State {
+	case inventory.BackupPending:
+		p.Note = backupsNote + "waiting for the backup of " + strings.Join(pending, ", ")
+	case inventory.BackupFailed:
+		p.Note = backupsNote + "held — the backup of " + strings.Join(failed, "; ") + "; nothing is asked or sent. " +
+			"`delivery-backups-again` asks it again"
+	default:
+		if strings.HasPrefix(p.Note, backupsNote) {
+			p.Note = ""
+		}
+	}
+}
+
+// waitingOnBackups says a plan has not asked its first tier because its backups are pending or failed: no tier
+// late, as S3 and the lines of `plans` read it.
+func waitingOnBackups(p inventory.Plan) bool {
+	return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Backups != nil &&
+		p.Backups.State != inventory.BackupDone
+}
+
+// askBackupsAgain clears the failed backups of a held plan, so the next step asks those again and no other. It
+// says which. Refused for a plan with no failed backup.
+func askBackupsAgain(p *inventory.Plan, by, why string) ([]inventory.PlanBackup, error) {
+	if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed {
+		return nil, fmt.Errorf("%s has no failed backup: there is nothing to ask again", p.ID)
+	}
+	var again []inventory.PlanBackup
+	for i := range p.Backups.Asked {
+		a := &p.Backups.Asked[i]
+		if a.State != inventory.BackupFailed {
+			continue
+		}
+		again = append(again, *a)
+		a.State, a.AskedAt, a.TriedAt, a.Words = inventory.BackupPending, nil, nil, ""
+	}
+	p.Backups.State = backupsStateOf(p.Backups.Asked)
+	p.Note = backupsNote + "asked again by " + by + ": " + why
+	return again, nil
+}
+
+// backupsAgain is askBackupsAgain for the plan with this id, kept: the verb `delivery-backups-again`.
+func backupsAgain(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, []inventory.PlanBackup, error) {
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return inventory.Plan{}, nil, err
+	}
+	defer release()
+	p, err := inv.PlanByID(ctx, id)
+	if err != nil {
+		return inventory.Plan{}, nil, err
+	}
+	again, err := askBackupsAgain(&p, by, why)
+	if err != nil {
+		return p, nil, err
+	}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		return p, nil, err
+	}
+	return p, again, nil
+}
+
+// kindBackupHeld is a delivery plan held for a backup that failed or stayed silent; backupHeldProbe its row.
+const (
+	kindBackupHeld  = "backup-held"
+	backupHeldProbe = "DK"
+)
+
+// observeBackupHolds is one condition per plan held for its backups, with the backups' own words as its evidence.
+func observeBackupHolds(plans []inventory.Plan) []conditions.Observation {
+	var out []conditions.Observation
+	for _, p := range plans {
+		if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed {
+			continue
+		}
+		var said, machines []string
+		for _, a := range p.Backups.Asked {
+			if a.State == inventory.BackupFailed {
+				said = append(said, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words))
+				machines = append(machines, a.Machine)
+			}
+		}
+		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.ID, Token: "backup-held",
+			Kind: kindBackupHeld, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
+			Machine: machines[0], Also: machines[1:],
+			Summary: fmt.Sprintf("the delivery %s is held: %d of its backups failed or did not answer, so nothing is "+
+				"asked or sent", p.ID, len(said)),
+			Said: strings.Join(said, "; ")})
+	}
+	return out
+}
+
+// probeBackupHeld reads the open plans for the ones held for their backups.
+func probeBackupHeld(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
+	plans, err := d.open.inventory.OpenPlans(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return observeBackupHolds(plans), nil
+}

cmd/mesh-controller/plan_backups_hold_test.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A backup that failed or stayed silent holds the plan, is said, and a verb asks it again (novox/hq ADR 0404
+// decision 4, issue 1066).
+
+// aHeldPlan is a plan whose backup on ace failed and whose backup on g14 is done.
+func aHeldPlan(t *testing.T) (*inventory.Plan, backupFacts) {
+	t.Helper()
+	entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))}
+	facts := backupFactsFrom(entries, holders("ace", "g14"))
+	seat := &aFakeBackupSeat{nights: map[string]*night{}}
+	seat.install(t)
+	p := aPlanOf("mail")
+	stepBackups(t.Context(), p, facts, backupT0)
+	seat.nights["ace/mail"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"}
+	seat.nights["g14/mail"] = &night{OK: true, At: backupT0.Add(time.Minute)}
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed || err != nil {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	return p, facts
+}
+
+// The held plan says which machine, which module and the backup's own words in its note, and is not called tried again.
+func TestAHeldPlanSaysTheMachineTheModuleAndTheWordsInItsNote(t *testing.T) {
+	p, _ := aHeldPlan(t)
+	for _, want := range []string{"ace", "mail", "no space left on device", "delivery-backups-again"} {
+		if !strings.Contains(p.Note, want) {
+			t.Fatalf("note %q lacks %q", p.Note, want)
+		}
+	}
+	if strings.Contains(p.Note, "tried again") || p.State != inventory.PlanBuilding {
+		t.Fatalf("a held plan is not failed and not retried: %q %s", p.Note, p.State)
+	}
+	if !waitingOnBackups(*p) {
+		t.Fatal("a plan held by its backups is no tier late (S3 leaves it out)")
+	}
+}
+
+// delivery-backups-again clears the failed records, and the next step asks those backups again and only those.
+func TestBackupsAgainClearsTheFailedRecordsAndTheNextStepAsksThemAgain(t *testing.T) {
+	p, facts := aHeldPlan(t)
+	again, err := askBackupsAgain(p, "mesh-delivery", "a person released it")
+	if err != nil || len(again) != 1 || again[0].Machine != "ace" || again[0].Module != "mail" {
+		t.Fatalf("again=%+v err=%v", again, err)
+	}
+	if p.Backups.State != inventory.BackupPending {
+		t.Fatalf("state %q", p.Backups.State)
+	}
+	seat := &aFakeBackupSeat{nights: map[string]*night{}}
+	seat.install(t)
+	now := backupT0.Add(10 * time.Minute)
+	if proceed, err := stepBackups(t.Context(), p, facts, now); proceed || err != nil {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	if strings.Join(seat.asked, " ") != "ace/mail" {
+		t.Fatalf("asked %v; only the failed backup is asked again", seat.asked)
+	}
+	seat.nights["ace/mail"] = &night{OK: true, At: now.Add(time.Minute)}
+	if proceed, _ := stepBackups(t.Context(), p, facts, now.Add(2*time.Minute)); !proceed {
+		t.Fatal("the plan must go on once the backup it asked again is done")
+	}
+	if strings.Contains(p.Note, "delivery-backups-again") {
+		t.Fatalf("the hold's note stays after it ended: %q", p.Note)
+	}
+}
+
+// Refused for a plan with no failed backup.
+func TestBackupsAgainIsRefusedForAPlanWithNoFailedBackup(t *testing.T) {
+	if _, err := askBackupsAgain(aPlanOf("mail"), "x", "y"); err == nil {
+		t.Fatal("a plan with no backups was asked again")
+	}
+	p := aPlanOf("mail")
+	p.Backups = &inventory.PlanBackups{State: inventory.BackupPending,
+		Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}}
+	if _, err := askBackupsAgain(p, "x", "y"); err == nil {
+		t.Fatal("a plan whose backups are pending was asked again")
+	}
+}
+
+// The verb's argument vector: the plan and a reason are required.
+func TestTheVerbDeliveryBackupsAgainTakesAPlanAndAWhy(t *testing.T) {
+	got, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"})
+	want := []string{"delivery", "backups-again", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}
+	if err != nil || strings.Join(got, "|") != strings.Join(want, "|") {
+		t.Fatalf("got %v %v, want %v", got, err, want)
+	}
+	if _, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1"}); err == nil {
+		t.Fatal("no reason was accepted")
+	}
+}
+
+// One condition per held plan, the backup one, with the words in its evidence; none for a plan only waiting.
+func TestAHeldPlanRaisesOneConditionWithTheBackupsWords(t *testing.T) {
+	held, _ := aHeldPlan(t)
+	waiting := aPlanOf("mail")
+	waiting.ID = "plan-w"
+	waiting.Backups = &inventory.PlanBackups{State: inventory.BackupPending,
+		Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}}
+	got := observeBackupHolds([]inventory.Plan{*held, *waiting})
+	if len(got) != 1 {
+		t.Fatalf("observations %+v", got)
+	}
+	o := got[0]
+	if o.Kind != kindBackupHeld || o.Scope != conditions.ScopePlan || o.ID != held.ID ||
+		!strings.Contains(o.Said, "no space left on device") || !strings.Contains(o.Said, "mail") || !strings.Contains(o.Said, "ace") {
+		t.Fatalf("%+v", o)
+	}
+}
+
+// A holder off the bus cannot be asked: the plan waits within the bound, retried by the look and not by the error
+// path, and at the bound the backup is failed with the reason, so the hold and the condition take over.
+func TestABackupHolderThatCannotBeAskedIsFailedAtTheBoundWithTheReason(t *testing.T) {
+	seat := &aFakeBackupSeat{}
+	seat.install(t)
+	was := askBackupNow
+	t.Cleanup(func() { askBackupNow = was })
+	calls := 0
+	askBackupNow = func(_ context.Context, machine, module string) error {
+		calls++
+		return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: no responders", machine)}
+	}
+	entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace"}, anItem("a", ""))}
+	facts := backupFactsFrom(entries, holders("ace"))
+	p := aPlanOf("mail")
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0); proceed || err != nil || p.Backups.State != inventory.BackupPending {
+		t.Fatalf("proceed=%v err=%v %+v", proceed, err, p.Backups)
+	}
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin/2)); proceed || err != nil ||
+		p.Backups.State != inventory.BackupPending || calls != 2 || p.Backups.Asked[0].AskedAt != nil {
+		t.Fatalf("within the bound: proceed=%v err=%v calls=%d %+v", proceed, err, calls, p.Backups)
+	}
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin)); proceed || err != nil {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	a := p.Backups.Asked[0]
+	if p.Backups.State != inventory.BackupFailed || a.State != inventory.BackupFailed ||
+		!strings.Contains(a.Words, "could not be asked: ") || !strings.Contains(a.Words, "no responders") {
+		t.Fatalf("%+v", p.Backups)
+	}
+	if strings.Contains(p.Note, "tried again") || !strings.Contains(p.Note, "held") {
+		t.Fatalf("note %q", p.Note)
+	}
+}

cmd/mesh-controller/plan_backups_test.go

@@ -0,0 +1,239 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// The backups before a delivery plan asks anything (novox/hq ADR 0404 decisions 2 and 3, issue 1065).
+
+// aModuleWith is a catalogue entry for a module keeping the given data items, on the given machines.
+func aModuleWith(t *testing.T, name string, on []string, items ...string) inventory.Entry {
+	t.Helper()
+	raw := `{"module":"` + name + `","version":"1"`
+	if len(items) > 0 {
+		raw += `,"data":{"own":[` + strings.Join(items, ",") + `]}`
+	}
+	raw += `}`
+	var m catalogue.Manifest
+	if err := json.Unmarshal([]byte(raw), &m); err != nil {
+		t.Fatal(err)
+	}
+	return inventory.Entry{Manifest: m, On: on}
+}
+
+func anItem(id, extra string) string {
+	if extra != "" {
+		extra = "," + extra
+	}
+	return `{"id":"` + id + `","path":"/x/` + id + `","class":"valuable","why":"w"` + extra + `}`
+}
+
+func holders(nodes ...string) []catalogue.Held {
+	var out []catalogue.Held
+	for _, n := range nodes {
+		out = append(out, catalogue.Held{Claim: catalogue.BackupSeat, Node: n, Module: "restic"})
+	}
+	return out
+}
+
+// aFakeBackupSeat stands in for the machines' node-backup holders.
+type aFakeBackupSeat struct {
+	asked  []string // "machine/module", in order
+	nights map[string]*night
+	refuse map[string]string
+}
+
+func (f *aFakeBackupSeat) install(t *testing.T) {
+	t.Helper()
+	was, wasRead := askBackupNow, readLastNights
+	t.Cleanup(func() { askBackupNow, readLastNights = was, wasRead })
+	askBackupNow = func(_ context.Context, machine, module string) error {
+		f.asked = append(f.asked, machine+"/"+module)
+		if why := f.refuse[machine+"/"+module]; why != "" {
+			return fmt.Errorf("%s", why)
+		}
+		return nil
+	}
+	readLastNights = func(_ context.Context, machine string) (map[string]*night, error) {
+		out := map[string]*night{}
+		for k, n := range f.nights {
+			if m, mod, _ := strings.Cut(k, "/"); m == machine {
+				out[mod] = n
+			}
+		}
+		return out, nil
+	}
+}
+
+func aPlanOf(modules ...string) *inventory.Plan {
+	p := &inventory.Plan{ID: "plan-b", State: inventory.PlanBuilding, Tiers: [][]string{modules},
+		Modules: map[string]*inventory.PlanModule{}}
+	for _, m := range modules {
+		p.Modules[m] = &inventory.PlanModule{}
+	}
+	return p
+}
+
+var backupT0 = time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
+
+// Only a module with an item protected by a backup is asked, and on the machines that run it.
+func TestOnlyModulesWithAnItemProtectedByABackupAreAsked(t *testing.T) {
+	seat := &aFakeBackupSeat{}
+	seat.install(t)
+	entries := []inventory.Entry{
+		aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", "")),
+		aModuleWith(t, "dumped", []string{"ace"}, anItem("b", `"backup":{"dump":"pg_dump","into":"d"}`), anItem("d", "")),
+		aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)),
+		aModuleWith(t, "unbacked", []string{"ace"}, anItem("e", `"backup":"none"`)),
+		aModuleWith(t, "cached", []string{"ace"}, `{"id":"f","path":"/x/f","class":"cache"}`),
+		aModuleWith(t, "dataless", []string{"ace"}),
+	}
+	p := aPlanOf("copied", "dumped", "mirrored", "unbacked", "cached", "dataless")
+	facts := backupFactsFrom(entries, holders("ace", "g14"))
+	proceed, err := stepBackups(t.Context(), p, facts, backupT0)
+	if err != nil || proceed {
+		t.Fatalf("proceed=%v err=%v; the plan must wait for the backups it asked", proceed, err)
+	}
+	want := "ace/copied ace/dumped g14/copied"
+	if got := strings.Join(seat.asked, " "); got != want {
+		t.Fatalf("asked %q, want %q", got, want)
+	}
+	if p.Backups == nil || p.Backups.State != inventory.BackupPending || len(p.Backups.Asked) != 3 {
+		t.Fatalf("the plan's record: %+v", p.Backups)
+	}
+}
+
+// A machine with no backup holder is asked nothing, the plan says so, and it does not hold the plan.
+func TestANodeWithNoBackupHolderAsksNothingAndThePlanSaysSo(t *testing.T) {
+	seat := &aFakeBackupSeat{}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "bare"}, anItem("a", ""))}
+	p := aPlanOf("copied")
+	proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0)
+	if err != nil || proceed {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	if strings.Join(seat.asked, " ") != "ace/copied" {
+		t.Fatalf("asked %v", seat.asked)
+	}
+	if p.Backups == nil || len(p.Backups.NoHolder) != 1 || p.Backups.NoHolder[0] != "bare" {
+		t.Fatalf("the plan must say bare has no holder: %+v", p.Backups)
+	}
+	// Only the no-holder machine: nothing to wait for, the plan goes on, and still says it.
+	only := []inventory.Entry{aModuleWith(t, "copied", []string{"bare"}, anItem("a", ""))}
+	q := aPlanOf("copied")
+	proceed, err = stepBackups(t.Context(), q, backupFactsFrom(only, holders("ace")), backupT0)
+	if err != nil || !proceed || q.Backups == nil || q.Backups.NoHolder[0] != "bare" || q.Backups.State != inventory.BackupDone {
+		t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, q.Backups)
+	}
+}
+
+// Nothing of the plan goes on until every backup answered done; a backup is not asked twice.
+func TestNothingIsAskedOfTheBuildOrSentUntilEveryBackupAnsweredDone(t *testing.T) {
+	seat := &aFakeBackupSeat{nights: map[string]*night{}}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))}
+	facts := backupFactsFrom(entries, holders("ace", "g14"))
+	p := aPlanOf("copied")
+	if proceed, _ := stepBackups(t.Context(), p, facts, backupT0); proceed {
+		t.Fatal("went on before any backup answered")
+	}
+	// ace is done, g14 still running (its last night is from before the ask).
+	seat.nights["ace/copied"] = &night{OK: true, At: backupT0.Add(time.Minute)}
+	seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(-time.Hour)}
+	if proceed, _ := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed {
+		t.Fatal("went on while g14's backup had not answered")
+	}
+	seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(3 * time.Minute)}
+	proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(4*time.Minute))
+	if err != nil || !proceed || p.Backups.State != inventory.BackupDone {
+		t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, p.Backups)
+	}
+	if len(seat.asked) != 2 {
+		t.Fatalf("asked %v; each backup is asked once", seat.asked)
+	}
+}
+
+// A failed backup, and a silent one past the bound, are recorded with the backup's own words and the plan stops.
+func TestAFailedOrSilentBackupIsRecordedAndThePlanStopsAsking(t *testing.T) {
+	seat := &aFakeBackupSeat{nights: map[string]*night{}}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))}
+	facts := backupFactsFrom(entries, holders("ace", "g14"))
+	p := aPlanOf("copied")
+	stepBackups(t.Context(), p, facts, backupT0)
+	seat.nights["ace/copied"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"}
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin+time.Minute)); proceed || err != nil {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	got := map[string]inventory.PlanBackup{}
+	for _, b := range p.Backups.Asked {
+		got[b.Machine] = b
+	}
+	if got["ace"].State != inventory.BackupFailed || got["ace"].Words != "no space left on device" {
+		t.Fatalf("ace: %+v", got["ace"])
+	}
+	if got["g14"].State != inventory.BackupFailed || !strings.Contains(got["g14"].Words, "did not answer in ten minutes") {
+		t.Fatalf("g14: %+v", got["g14"])
+	}
+	if p.Backups.State != inventory.BackupFailed {
+		t.Fatalf("state %q", p.Backups.State)
+	}
+	// It stays stopped, asks nothing more, and says no retry in the plan's note.
+	n := len(seat.asked)
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(time.Hour)); proceed || err != nil || len(seat.asked) != n {
+		t.Fatalf("a failed backup must stay stopped: proceed=%v err=%v asked=%v", proceed, err, seat.asked)
+	}
+}
+
+// A holder that refuses the ask says why, in the record.
+func TestAHolderThatRefusesTheAskIsRecordedFailedWithItsWords(t *testing.T) {
+	seat := &aFakeBackupSeat{refuse: map[string]string{"ace/copied": "copied declares nothing to back up"}}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace"}, anItem("a", ""))}
+	p := aPlanOf("copied")
+	if proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0); proceed || err != nil {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	if b := p.Backups.Asked[0]; b.State != inventory.BackupFailed || b.Words != "copied declares nothing to back up" {
+		t.Fatalf("%+v", b)
+	}
+}
+
+// A plan moving no backed-up item asks nothing and keeps no record (design 47: costs nothing).
+func TestAPlanThatMovesNoBackedUpItemAsksNothing(t *testing.T) {
+	seat := &aFakeBackupSeat{}
+	seat.install(t)
+	entries := []inventory.Entry{
+		aModuleWith(t, "dataless", []string{"ace"}),
+		aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)),
+	}
+	p := aPlanOf("dataless", "mirrored")
+	proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0)
+	if err != nil || !proceed || p.Backups != nil || len(seat.asked) != 0 {
+		t.Fatalf("proceed=%v err=%v record=%+v asked=%v", proceed, err, p.Backups, seat.asked)
+	}
+}
+
+// The holders of a node seat are every machine's, and a source that fails is no machine without a holder.
+func TestTheBackupHoldersAreEveryMachinesAndAFailedReadIsNotNoHolder(t *testing.T) {
+	entries := func(context.Context) ([]inventory.Entry, error) {
+		return []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))}, nil
+	}
+	f, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return holders("ace", "g14"), nil }, t.Context())
+	if err != nil || !f.holder["ace"] || !f.holder["g14"] {
+		t.Fatalf("%+v %v", f, err)
+	}
+	if _, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return nil, fmt.Errorf("store down") },
+		t.Context()); err == nil || !strings.Contains(err.Error(), "store down") {
+		t.Fatalf("a failed read of the holders must fail the step: %v", err)
+	}
+}

cmd/mesh-controller/release_plan.go

@@ -807,6 +807,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 				return changed, nil
 			}
 		}
+		// **A backup before the send** (novox/hq ADR 0404, issue 1065): a walk that starts asks the backup holders of
+		// the machines running a moved module that keeps data, and asks its first tier only when every one is done.
+		if p.Tier == 0 && !askedInTier(*p) {
+			if proceed, err := backupsFirst(ctx, inv, p); err != nil {
+				return false, err
+			} else if !proceed {
+				return false, nil
+			}
+		}
 		if len(unasked) < len(tier) {
 			fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", "))
 		}
@@ -1721,6 +1730,9 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
 		// Nor is waiting for the walk before it to end (a related walk, novox/hq ADR 0276, ADR 0407).
 		return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since)
 	}
+	if waitingOnBackups(p) {
+		return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since)
+	}
 	if waiting, paused := pausedWaiting(p, pause, now); paused {
 		return fmt.Sprintf("%s · %s, %s", name, where, waiting)
 	}

cmd/mesh-controller/seatverbs.go

@@ -444,6 +444,15 @@ func (a *verbArguments) commandLine() ([]string, error) {
 			argv = append(argv, "--without", w)
 		}
 		return argv, nil
+	case "delivery-backups-again":
+		if err := need("plan", "why"); err != nil {
+			return nil, err
+		}
+		argv := []string{"delivery", "backups-again", str("plan"), "--why", str("why")}
+		if b := str("by"); b != "" {
+			argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b)
+		}
+		return argv, nil
 	case "delivery-walks":
 		argv := []string{"delivery", "walks"}
 		if id := str("plan"); id != "" {

cmd/mesh-controller/watchdogs.go

@@ -538,6 +538,11 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time, b
 		if deferred(p) {
 			continue
 		}
+		// A walk waiting on, or held by, its backups is no tier late either: the backup condition is the one voice
+		// (novox/hq ADR 0404 decision 4, issue 1066).
+		if waitingOnBackups(p) {
+			continue
+		}
 		_, paused := pausedWaiting(p, pause, now)
 		bound := bounds.of(p.Repository)
 		out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier,

internal/broker/nats.go

@@ -245,7 +245,8 @@ var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb:
 // VerbsTheBusStepAsks are the seat verbs the bus's planned step calls (novox/hq to-be 45 §8, ADR 0236):
 // the bus machine's backup holder takes the bus's snapshot now, before the bus is replaced (ADR 0235's
 // dump), and says when it is taken (`backed-up`, granted with the self-check's). The one verb of
-// the controller's grant that acts, and only through the step a person starts.
+// the controller's grant that acts: through the step a person starts, and through the backups a delivery plan
+// takes before it asks its first tier (novox/hq ADR 0404).
 var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}}
 
 // VerbsTheControllerAsksForASecret are the seat verbs `give` calls (novox/hq ADR 0259 §10): the operator's

internal/catalogue/verbs.go

@@ -177,6 +177,12 @@ var ControllerVerbs = []Verb{
 		Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery",
 			"without": "merges to leave out, <repository>@<commit>, comma-separated"},
 			[]string{"plan", "why"})},
+	{Name: "delivery-backups-again", Description: "Ask again the backups a walk was held for (novox/hq ADR 0404 " +
+		"decision 4): a walk whose backup before its send failed or did not answer is held, nothing asked or sent; this " +
+		"clears those failed backups and the next look at the plans asks them again, and only them. Refused for a walk " +
+		"with no failed backup. The delivery's release calls it. The answer names the walk and the backups asked again.",
+		Input: schema(map[string]string{"plan": "the held walk's id", "why": "a person's reason, as release passes it",
+			"by": "who released the delivery"}, []string{"plan", "why"})},
 	{Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " +
 		"last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " +
 		"phases from its merge to every machine running it (novox/hq ADR 0282) — and whether the delivery seat has a " +

internal/inventory/migrations/0103-a-walk-keeps-its-backups.sql

@@ -0,0 +1,6 @@
+-- A walk keeps the backups it takes before it asks its first tier (novox/hq ADR 0404 decisions 2 to 4, issue 1065).
+--
+-- Per machine and module: asked, pending, done or failed, with the backup's own words, and the machines with no backup
+-- holder. Read by whoever says the walk (`plan-moved`). Null for a walk that moves nothing a backup protects, and for
+-- a walk kept before this.
+alter table release_plan add column backups jsonb;

internal/inventory/plan_backups_test.go

@@ -0,0 +1,35 @@
+package inventory
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The plan's backups are the contract mesh-delivery reads from `plan-moved` (issue 1065): their JSON field names
+// are fixed here, and a plan keeps them through a round trip.
+func TestAPlanKeepsItsBackupsThroughItsJSONUnderTheNamesTheDeliveryReads(t *testing.T) {
+	at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
+	p := Plan{ID: "p", Backups: &PlanBackups{State: BackupFailed, NoHolder: []string{"bare"},
+		Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupFailed, AskedAt: &at, Words: "no space left"}}}}
+	raw, err := json.Marshal(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range []string{`"backups":{"state":"failed","asked":[{"machine":"ace","module":"mail","state":"failed",` +
+		`"asked_at":"2026-10-17T12:00:00Z","words":"no space left"}],"no_holder":["bare"]}`} {
+		if !strings.Contains(string(raw), name) {
+			t.Fatalf("plan JSON %s lacks %s", raw, name)
+		}
+	}
+	var back Plan
+	if err := json.Unmarshal(raw, &back); err != nil || !reflect.DeepEqual(back.Backups, p.Backups) {
+		t.Fatalf("round trip: %v %+v", err, back.Backups)
+	}
+	raw, _ = json.Marshal(Plan{ID: "q"})
+	if strings.Contains(string(raw), "backups") {
+		t.Fatalf("a plan with no backups says none: %s", raw)
+	}
+}

internal/inventory/plans.go

@@ -62,12 +62,51 @@ type Plan struct {
 	// Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out
 	// from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`).
 	Phases *WalkPhases `json:"phases,omitempty"`
+	// Backups is the backups the walk takes before it asks its first tier (novox/hq ADR 0404 decisions 2 and 3):
+	// per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a
+	// walk that moves no module with an item protected by a backup, and for a walk kept before it was.
+	Backups *PlanBackups `json:"backups,omitempty"`
 	// WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308
 	// decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every
 	// other save.
 	WalkSaid string `json:"-"`
 }
 
+// The states of a plan's backups, as a whole and each.
+const (
+	BackupPending = "pending"
+	BackupDone    = "done"
+	BackupFailed  = "failed"
+)
+
+// PlanBackups is a walk's record of the backups it takes before it asks anything (novox/hq ADR 0404): read by
+// mesh-delivery from `plan-moved`, so its JSON field names are a contract between the two repositories.
+type PlanBackups struct {
+	// State is the whole: pending while any backup is asked or unasked and none failed, failed as soon as one
+	// failed, done when every one is.
+	State string `json:"state"`
+	// Asked is one entry per machine and module, sorted by machine then module.
+	Asked []PlanBackup `json:"asked"`
+	// NoHolder are the machines that run a module with an item protected by a backup and hold no node-backup:
+	// nothing is asked of them, and the plan says so.
+	NoHolder []string `json:"no_holder,omitempty"`
+}
+
+// PlanBackup is one backup asked of one machine's backup holder for one module.
+type PlanBackup struct {
+	Machine string `json:"machine"`
+	Module  string `json:"module"`
+	// State is pending, done or failed.
+	State string `json:"state"`
+	// AskedAt is when the holder was asked; nil while it is not asked yet.
+	AskedAt *time.Time `json:"asked_at,omitempty"`
+	// TriedAt is when the controller first tried to ask, kept while the holder cannot be reached: the bound of a
+	// backup that is never asked counts from it. Nil once asked.
+	TriedAt *time.Time `json:"tried_at,omitempty"`
+	// Words are the backup's own: the holder's error for a failed one, "did not answer in ten minutes" for a silent one.
+	Words string `json:"words,omitempty"`
+}
+
 // PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6).
 type PlanTimes struct {
 	// WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum.
@@ -707,7 +746,12 @@ func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, ti
 	if err != nil {
 		return 0, time.Time{}, err
 	}
-	var release, delivery, commits, times []byte
+	var release, delivery, commits, times, backups []byte
+	if p.Backups != nil {
+		if backups, err = json.Marshal(p.Backups); err != nil {
+			return 0, time.Time{}, err
+		}
+	}
 	if p.Times != nil {
 		if times, err = json.Marshal(p.Times); err != nil {
 			return 0, time.Time{}, err
@@ -735,18 +779,18 @@ func writePlan(ctx context.Context, tx pgx.Tx, p *Plan, epoch *int64) (int64, ti
 	var revision int64
 	err = tx.QueryRow(ctx,
 		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
-		                           branch, tier_entered, revision, epoch, release, delivery, merged_at, commits, times)
-		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17, $18)
+		                           branch, tier_entered, revision, epoch, release, delivery, merged_at, commits, times, backups)
+		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13, $14, $15, $16, $17, $18, $19)
 		 on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
 		        tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
 		        tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch,
 		        release = excluded.release, delivery = excluded.delivery, repository = excluded.repository,
 		        commit_hash = excluded.commit_hash, merged_at = excluded.merged_at, commits = excluded.commits,
-		        created = excluded.created, times = excluded.times
+		        created = excluded.created, times = excluded.times, backups = excluded.backups
 		  where release_plan.revision = $12
 		 returning revision`,
 		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
-		p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits, times).Scan(&revision)
+		p.Revision, epoch, release, delivery, mergedAt(p.Merged), commits, times, backups).Scan(&revision)
 	if errors.Is(err, pgx.ErrNoRows) {
 		// The row is there and at another revision — moved since this was read, or there already
 		// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
@@ -815,7 +859,7 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
 func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
-		        coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits, times
+		        coalesce(tier_entered, created), revision, coalesce(epoch, 0), release, delivery, merged_at, commits, times, backups
 		   from release_plan `+tail, args...)
 	if err != nil {
 		return nil, err
@@ -824,14 +868,19 @@ func (i *Inventory) plans(ctx context.Context, tail string, args ...any) ([]Plan
 	var out []Plan
 	for rows.Next() {
 		var p Plan
-		var tiers, modules, release, delivery, commits, times []byte
+		var tiers, modules, release, delivery, commits, times, backups []byte
 		var epoch int64
 		var merged *time.Time
 		if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
 			&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch, &release,
-			&delivery, &merged, &commits, &times); err != nil {
+			&delivery, &merged, &commits, &times, &backups); err != nil {
 			return nil, err
 		}
+		if len(backups) > 0 {
+			if err := json.Unmarshal(backups, &p.Backups); err != nil {
+				return nil, err
+			}
+		}
 		if len(times) > 0 {
 			if err := json.Unmarshal(times, &p.Times); err != nil {
 				return nil, err

internal/inventory/plans_test.go

@@ -198,3 +198,28 @@ func TestASchemaReachIsKeptByBuild(t *testing.T) {
 		t.Fatalf("reach %d %v %v", reach, known, err)
 	}
 }
+
+// A plan keeps its backups through the store, timestamps to the nanosecond (novox/hq ADR 0404, issue 1065).
+func TestAPlanKeepsItsBackupsInTheStore(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	at := time.Now().UTC()
+	p := Plan{ID: "plan-backups", Repository: "novox/mesh-catalog", Commit: "abc", Created: at, State: PlanBuilding,
+		Tiers: [][]string{{"mail"}}, Modules: map[string]*PlanModule{"mail": {}},
+		Backups: &PlanBackups{State: BackupPending, NoHolder: []string{"bare"},
+			Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupPending, AskedAt: &at}}}}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	got, err := inv.PlanByID(ctx, p.ID)
+	if err != nil || got.Backups == nil || !reflect.DeepEqual(got.Backups, p.Backups) {
+		t.Fatalf("kept %+v (%v), wrote %+v", got.Backups, err, p.Backups)
+	}
+	got.Backups.State = BackupDone
+	if err := inv.SavePlan(ctx, &got); err != nil {
+		t.Fatal(err)
+	}
+	if again, _ := inv.PlanByID(ctx, p.ID); again.Backups == nil || again.Backups.State != BackupDone {
+		t.Fatalf("the change was not kept: %+v", again.Backups)
+	}
+}

module.json

@@ -57,6 +57,7 @@
     "delivery-check",
     "deliver",
     "delivery-stop",
+    "delivery-backups-again",
     "delivery-walks",
     "plan",
     "assign",