Ask the backup seats before the first tier (issue 1065)

cmd/mesh-controller/plan_backups.go

@@ -0,0 +1,294 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"slices"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **A backup before the send** (novox/hq ADR 0404 decisions 2 and 3, design 47, issue 1065). Once a delivery plan
+// has started and before it asks its first tier, the controller asks the node-backup holder of every machine that
+// runs a moved module with an item protected by a backup to back that module up now, and asks nothing of a builder
+// and sends nothing until every one of them answered done. The plan keeps the record (inventory.PlanBackups).
+
+// backupsWithin is how long a backup is given from its ask to say it is done, and backupsPollEvery how soon the plan
+// is looked at again while it waits: variables so a test waits for neither.
+var (
+	backupsWithin    = 10 * time.Minute
+	backupsPollEvery = 10 * time.Second
+)
+
+// night is one module's last night as a holder's `backed-up` says it (lastNight).
+type night struct {
+	OK    bool      `json:"ok"`
+	At    time.Time `json:"at"`
+	Error string    `json:"error,omitempty"`
+}
+
+// askBackupNow asks a machine's backup holder to back a module up now; the holder answers when it has started. A
+// variable so a test needs no bus. An error is the holder's own words when it refused, or why it could not be asked.
+var askBackupNow = func(ctx context.Context, machine, module string) error {
+	return onTheBus(func(conn *nats.Conn) error {
+		answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", machine,
+			map[string]any{"module": module}, 30*time.Second)
+		if err != nil {
+			return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: %w", machine, err)}
+		}
+		if answer.Error != "" {
+			return fmt.Errorf("%s", answer.Error)
+		}
+		return nil
+	})
+}
+
+// backupNotAsked is an ask that could not be made, as opposed to one the holder refused: the step is tried again.
+type backupNotAsked struct{ error }
+
+func (e *backupNotAsked) Unwrap() error { return e.error }
+
+// readLastNights reads a machine's holder's `backed-up` by module: how each module's last backup went. A variable
+// so a test needs no bus.
+var readLastNights = func(ctx context.Context, machine string) (map[string]*night, error) {
+	var out map[string]*night
+	err := onTheBus(func(conn *nats.Conn) error {
+		answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", machine, map[string]any{}, 10*time.Second)
+		if err != nil {
+			return err
+		}
+		if answer.Error != "" {
+			return fmt.Errorf("%s", answer.Error)
+		}
+		var read []struct {
+			Module    string `json:"module"`
+			LastNight *night `json:"lastNight"`
+		}
+		if err := json.Unmarshal(answer.Result, &read); err != nil {
+			return fmt.Errorf("its answer is not readable: %w", err)
+		}
+		out = map[string]*night{}
+		for _, m := range read {
+			out[m.Module] = m.LastNight
+		}
+		return nil
+	})
+	return out, err
+}
+
+// backupFacts are what the step needs of the mesh: the modules with an item protected by a backup, where each
+// runs, and which machines hold node-backup.
+type backupFacts struct {
+	backedUp map[string][]string // module → machines running it
+	holder   map[string]bool
+}
+
+// backupFactsFrom reads the facts from the catalogue's entries and the seat holdings. A module is backed up when
+// one of its own items is (catalogue.DataItem.BackedUp: never a cache, `backup: none` or redundancy alone). A
+// module whose policy records (ADR 0377) is decided per machine after the build, so every machine running it is
+// asked: a superset costs time and never leaves a send without its restore point.
+func backupFactsFrom(entries []inventory.Entry, held []catalogue.Held) backupFacts {
+	f := backupFacts{backedUp: map[string][]string{}, holder: map[string]bool{}}
+	for _, e := range entries {
+		for _, it := range e.Manifest.DataItems() {
+			if it.BackedUp() {
+				f.backedUp[e.Manifest.Module] = slices.Clone(e.On)
+				break
+			}
+		}
+	}
+	for _, h := range held {
+		if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat {
+			f.holder[h.Node] = true
+		}
+	}
+	return f
+}
+
+// loadBackupFacts reads the facts from the inventory.
+func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts, error) {
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return backupFacts{}, err
+	}
+	held, err := inv.Holdings(ctx)
+	if err != nil {
+		return backupFacts{}, err
+	}
+	return backupFactsFrom(entries, held), nil
+}
+
+// stepBackups takes one step of a plan's backups and says whether the plan may go on to ask its first tier. It
+// asks what is not asked, reads how the asked ones went, and records it all in p.Backups; it waits (false, nil)
+// while any is pending, and stays stopped (false, nil) once one failed, which part B holds and says. A plan whose
+// moved modules have no backed-up item keeps no record and goes on at once. An error is an ask that could not be
+// made or a read the store refused: the step is tried again, the asks already made kept.
+func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) {
+	if p.Backups == nil {
+		b := wantedBackups(*p, facts)
+		if b == nil {
+			return true, nil
+		}
+		p.Backups = b
+	}
+	b := p.Backups
+	switch b.State {
+	case inventory.BackupDone:
+		return true, nil
+	case inventory.BackupFailed:
+		return false, nil
+	}
+	nights := map[string]map[string]*night{}
+	for i := range b.Asked {
+		a := &b.Asked[i]
+		if a.State != inventory.BackupPending {
+			continue
+		}
+		if a.AskedAt == nil {
+			asked := now
+			if err := askBackupNow(ctx, a.Machine, a.Module); err != nil {
+				var unasked *backupNotAsked
+				if errors.As(err, &unasked) {
+					return false, err
+				}
+				a.State, a.Words = inventory.BackupFailed, err.Error()
+				a.AskedAt = &asked
+				continue
+			}
+			a.AskedAt = &asked
+			continue
+		}
+		if _, read := nights[a.Machine]; !read {
+			n, err := readLastNights(ctx, a.Machine)
+			if err != nil {
+				n = nil // not read now: the bound below still ends the wait
+			}
+			nights[a.Machine] = n
+		}
+		if n := nights[a.Machine][a.Module]; n != nil && n.At.After(*a.AskedAt) {
+			if n.OK {
+				a.State, a.Words = inventory.BackupDone, ""
+			} else {
+				a.State, a.Words = inventory.BackupFailed, n.Error
+				if a.Words == "" {
+					a.Words = "the backup failed and did not say why"
+				}
+			}
+			continue
+		}
+		if now.Sub(*a.AskedAt) >= backupsWithin {
+			a.State = inventory.BackupFailed
+			a.Words = "did not answer in " + wordsOfBound(backupsWithin)
+		}
+	}
+	b.State = backupsStateOf(b.Asked)
+	switch b.State {
+	case inventory.BackupDone:
+		return true, nil
+	case inventory.BackupPending:
+		lookAgainForBackups(p.ID)
+	}
+	return false, nil
+}
+
+// wantedBackups is the record of backups a plan has to take, or nil for none: one entry per machine with a holder
+// running a moved module that has a backed-up item, and the machines with none.
+func wantedBackups(p inventory.Plan, facts backupFacts) *inventory.PlanBackups {
+	var asked []inventory.PlanBackup
+	noHolder := map[string]bool{}
+	for _, tier := range p.Tiers {
+		for _, module := range tier {
+			for _, machine := range facts.backedUp[module] {
+				if !facts.holder[machine] {
+					noHolder[machine] = true
+					continue
+				}
+				asked = append(asked, inventory.PlanBackup{Machine: machine, Module: module, State: inventory.BackupPending})
+			}
+		}
+	}
+	if len(asked) == 0 && len(noHolder) == 0 {
+		return nil
+	}
+	sort.SliceStable(asked, func(i, j int) bool {
+		if asked[i].Machine != asked[j].Machine {
+			return asked[i].Machine < asked[j].Machine
+		}
+		return asked[i].Module < asked[j].Module
+	})
+	b := &inventory.PlanBackups{Asked: asked}
+	for m := range noHolder {
+		b.NoHolder = append(b.NoHolder, m)
+	}
+	sort.Strings(b.NoHolder)
+	b.State = backupsStateOf(asked)
+	return b
+}
+
+// backupsStateOf is the whole of the backups' states: failed if any failed, done if all are (or none are asked).
+func backupsStateOf(asked []inventory.PlanBackup) string {
+	state := inventory.BackupDone
+	for _, a := range asked {
+		switch a.State {
+		case inventory.BackupFailed:
+			return inventory.BackupFailed
+		case inventory.BackupPending:
+			state = inventory.BackupPending
+		}
+	}
+	return state
+}
+
+func wordsOfBound(d time.Duration) string {
+	if d == 10*time.Minute {
+		return "ten minutes"
+	}
+	if d%time.Minute == 0 {
+		return fmt.Sprintf("%d minutes", int(d/time.Minute))
+	}
+	return d.String()
+}
+
+var (
+	backupLookMu sync.Mutex
+	backupLooks  = map[string]bool{}
+)
+
+// lookAgainForBackups has the serving controller look at the plans again soon, once per waiting plan: a backup ending
+// is no event the controller hears, so the wait is looked at by a timer, never by blocking the hold on the plans.
+func lookAgainForBackups(plan string) {
+	backupLookMu.Lock()
+	if backupLooks[plan] {
+		backupLookMu.Unlock()
+		return
+	}
+	backupLooks[plan] = true
+	backupLookMu.Unlock()
+	time.AfterFunc(backupsPollEvery, func() {
+		backupLookMu.Lock()
+		delete(backupLooks, plan)
+		backupLookMu.Unlock()
+		askToLook()
+	})
+}
+
+// backupsFirst is the step advanceOnce takes before it asks a walk's first tier: whether the plan may go on.
+func backupsFirst(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) (bool, error) {
+	facts := backupFacts{}
+	if p.Backups == nil {
+		f, err := loadBackupFacts(ctx, inv)
+		if err != nil {
+			return false, err
+		}
+		facts = f
+	}
+	return stepBackups(ctx, p, facts, time.Now().UTC())
+}

cmd/mesh-controller/plan_backups_test.go

@@ -0,0 +1,224 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// The backups before a delivery plan asks anything (novox/hq ADR 0404 decisions 2 and 3, issue 1065).
+
+// aModuleWith is a catalogue entry for a module keeping the given data items, on the given machines.
+func aModuleWith(t *testing.T, name string, on []string, items ...string) inventory.Entry {
+	t.Helper()
+	raw := `{"module":"` + name + `","version":"1"`
+	if len(items) > 0 {
+		raw += `,"data":{"own":[` + strings.Join(items, ",") + `]}`
+	}
+	raw += `}`
+	var m catalogue.Manifest
+	if err := json.Unmarshal([]byte(raw), &m); err != nil {
+		t.Fatal(err)
+	}
+	return inventory.Entry{Manifest: m, On: on}
+}
+
+func anItem(id, extra string) string {
+	if extra != "" {
+		extra = "," + extra
+	}
+	return `{"id":"` + id + `","path":"/x/` + id + `","class":"valuable","why":"w"` + extra + `}`
+}
+
+func holders(nodes ...string) []catalogue.Held {
+	var out []catalogue.Held
+	for _, n := range nodes {
+		out = append(out, catalogue.Held{Claim: catalogue.BackupSeat, Node: n, Module: "restic"})
+	}
+	return out
+}
+
+// aFakeBackupSeat stands in for the machines' node-backup holders.
+type aFakeBackupSeat struct {
+	asked  []string // "machine/module", in order
+	nights map[string]*night
+	refuse map[string]string
+}
+
+func (f *aFakeBackupSeat) install(t *testing.T) {
+	t.Helper()
+	was, wasRead := askBackupNow, readLastNights
+	t.Cleanup(func() { askBackupNow, readLastNights = was, wasRead })
+	askBackupNow = func(_ context.Context, machine, module string) error {
+		f.asked = append(f.asked, machine+"/"+module)
+		if why := f.refuse[machine+"/"+module]; why != "" {
+			return fmt.Errorf("%s", why)
+		}
+		return nil
+	}
+	readLastNights = func(_ context.Context, machine string) (map[string]*night, error) {
+		out := map[string]*night{}
+		for k, n := range f.nights {
+			if m, mod, _ := strings.Cut(k, "/"); m == machine {
+				out[mod] = n
+			}
+		}
+		return out, nil
+	}
+}
+
+func aPlanOf(modules ...string) *inventory.Plan {
+	p := &inventory.Plan{ID: "plan-b", State: inventory.PlanBuilding, Tiers: [][]string{modules},
+		Modules: map[string]*inventory.PlanModule{}}
+	for _, m := range modules {
+		p.Modules[m] = &inventory.PlanModule{}
+	}
+	return p
+}
+
+var backupT0 = time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
+
+// Only a module with an item protected by a backup is asked, and on the machines that run it.
+func TestOnlyModulesWithAnItemProtectedByABackupAreAsked(t *testing.T) {
+	seat := &aFakeBackupSeat{}
+	seat.install(t)
+	entries := []inventory.Entry{
+		aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", "")),
+		aModuleWith(t, "dumped", []string{"ace"}, anItem("b", `"backup":{"dump":"pg_dump","into":"d"}`), anItem("d", "")),
+		aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)),
+		aModuleWith(t, "unbacked", []string{"ace"}, anItem("e", `"backup":"none"`)),
+		aModuleWith(t, "cached", []string{"ace"}, `{"id":"f","path":"/x/f","class":"cache"}`),
+		aModuleWith(t, "dataless", []string{"ace"}),
+	}
+	p := aPlanOf("copied", "dumped", "mirrored", "unbacked", "cached", "dataless")
+	facts := backupFactsFrom(entries, holders("ace", "g14"))
+	proceed, err := stepBackups(t.Context(), p, facts, backupT0)
+	if err != nil || proceed {
+		t.Fatalf("proceed=%v err=%v; the plan must wait for the backups it asked", proceed, err)
+	}
+	want := "ace/copied ace/dumped g14/copied"
+	if got := strings.Join(seat.asked, " "); got != want {
+		t.Fatalf("asked %q, want %q", got, want)
+	}
+	if p.Backups == nil || p.Backups.State != inventory.BackupPending || len(p.Backups.Asked) != 3 {
+		t.Fatalf("the plan's record: %+v", p.Backups)
+	}
+}
+
+// A machine with no backup holder is asked nothing, the plan says so, and it does not hold the plan.
+func TestANodeWithNoBackupHolderAsksNothingAndThePlanSaysSo(t *testing.T) {
+	seat := &aFakeBackupSeat{}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "bare"}, anItem("a", ""))}
+	p := aPlanOf("copied")
+	proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0)
+	if err != nil || proceed {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	if strings.Join(seat.asked, " ") != "ace/copied" {
+		t.Fatalf("asked %v", seat.asked)
+	}
+	if p.Backups == nil || len(p.Backups.NoHolder) != 1 || p.Backups.NoHolder[0] != "bare" {
+		t.Fatalf("the plan must say bare has no holder: %+v", p.Backups)
+	}
+	// Only the no-holder machine: nothing to wait for, the plan goes on, and still says it.
+	only := []inventory.Entry{aModuleWith(t, "copied", []string{"bare"}, anItem("a", ""))}
+	q := aPlanOf("copied")
+	proceed, err = stepBackups(t.Context(), q, backupFactsFrom(only, holders("ace")), backupT0)
+	if err != nil || !proceed || q.Backups == nil || q.Backups.NoHolder[0] != "bare" || q.Backups.State != inventory.BackupDone {
+		t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, q.Backups)
+	}
+}
+
+// Nothing of the plan goes on until every backup answered done; a backup is not asked twice.
+func TestNothingIsAskedOfTheBuildOrSentUntilEveryBackupAnsweredDone(t *testing.T) {
+	seat := &aFakeBackupSeat{nights: map[string]*night{}}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))}
+	facts := backupFactsFrom(entries, holders("ace", "g14"))
+	p := aPlanOf("copied")
+	if proceed, _ := stepBackups(t.Context(), p, facts, backupT0); proceed {
+		t.Fatal("went on before any backup answered")
+	}
+	// ace is done, g14 still running (its last night is from before the ask).
+	seat.nights["ace/copied"] = &night{OK: true, At: backupT0.Add(time.Minute)}
+	seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(-time.Hour)}
+	if proceed, _ := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed {
+		t.Fatal("went on while g14's backup had not answered")
+	}
+	seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(3 * time.Minute)}
+	proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(4*time.Minute))
+	if err != nil || !proceed || p.Backups.State != inventory.BackupDone {
+		t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, p.Backups)
+	}
+	if len(seat.asked) != 2 {
+		t.Fatalf("asked %v; each backup is asked once", seat.asked)
+	}
+}
+
+// A failed backup, and a silent one past the bound, are recorded with the backup's own words and the plan stops.
+func TestAFailedOrSilentBackupIsRecordedAndThePlanStopsAsking(t *testing.T) {
+	seat := &aFakeBackupSeat{nights: map[string]*night{}}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))}
+	facts := backupFactsFrom(entries, holders("ace", "g14"))
+	p := aPlanOf("copied")
+	stepBackups(t.Context(), p, facts, backupT0)
+	seat.nights["ace/copied"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"}
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin+time.Minute)); proceed || err != nil {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	got := map[string]inventory.PlanBackup{}
+	for _, b := range p.Backups.Asked {
+		got[b.Machine] = b
+	}
+	if got["ace"].State != inventory.BackupFailed || got["ace"].Words != "no space left on device" {
+		t.Fatalf("ace: %+v", got["ace"])
+	}
+	if got["g14"].State != inventory.BackupFailed || !strings.Contains(got["g14"].Words, "did not answer in ten minutes") {
+		t.Fatalf("g14: %+v", got["g14"])
+	}
+	if p.Backups.State != inventory.BackupFailed {
+		t.Fatalf("state %q", p.Backups.State)
+	}
+	// It stays stopped, asks nothing more, and says no retry in the plan's note.
+	n := len(seat.asked)
+	if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(time.Hour)); proceed || err != nil || len(seat.asked) != n {
+		t.Fatalf("a failed backup must stay stopped: proceed=%v err=%v asked=%v", proceed, err, seat.asked)
+	}
+}
+
+// A holder that refuses the ask says why, in the record.
+func TestAHolderThatRefusesTheAskIsRecordedFailedWithItsWords(t *testing.T) {
+	seat := &aFakeBackupSeat{refuse: map[string]string{"ace/copied": "copied declares nothing to back up"}}
+	seat.install(t)
+	entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace"}, anItem("a", ""))}
+	p := aPlanOf("copied")
+	if proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0); proceed || err != nil {
+		t.Fatalf("proceed=%v err=%v", proceed, err)
+	}
+	if b := p.Backups.Asked[0]; b.State != inventory.BackupFailed || b.Words != "copied declares nothing to back up" {
+		t.Fatalf("%+v", b)
+	}
+}
+
+// A plan moving no backed-up item asks nothing and keeps no record (design 47: costs nothing).
+func TestAPlanThatMovesNoBackedUpItemAsksNothing(t *testing.T) {
+	seat := &aFakeBackupSeat{}
+	seat.install(t)
+	entries := []inventory.Entry{
+		aModuleWith(t, "dataless", []string{"ace"}),
+		aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)),
+	}
+	p := aPlanOf("dataless", "mirrored")
+	proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0)
+	if err != nil || !proceed || p.Backups != nil || len(seat.asked) != 0 {
+		t.Fatalf("proceed=%v err=%v record=%+v asked=%v", proceed, err, p.Backups, seat.asked)
+	}
+}

cmd/mesh-controller/release_plan.go

@@ -807,6 +807,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 				return changed, nil
 			}
 		}
+		// **A backup before the send** (novox/hq ADR 0404, issue 1065): a walk that starts asks the backup holders of
+		// the machines running a moved module that keeps data, and asks its first tier only when every one is done.
+		if p.Tier == 0 && !askedInTier(*p) {
+			if proceed, err := backupsFirst(ctx, inv, p); err != nil {
+				return false, err
+			} else if !proceed {
+				return false, nil
+			}
+		}
 		if len(unasked) < len(tier) {
 			fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", "))
 		}

internal/broker/nats.go

@@ -245,7 +245,8 @@ var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb:
 // VerbsTheBusStepAsks are the seat verbs the bus's planned step calls (novox/hq to-be 45 §8, ADR 0236):
 // the bus machine's backup holder takes the bus's snapshot now, before the bus is replaced (ADR 0235's
 // dump), and says when it is taken (`backed-up`, granted with the self-check's). The one verb of
-// the controller's grant that acts, and only through the step a person starts.
+// the controller's grant that acts: through the step a person starts, and through the backups a delivery plan
+// takes before it asks its first tier (novox/hq ADR 0404).
 var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}}
 
 // VerbsTheControllerAsksForASecret are the seat verbs `give` calls (novox/hq ADR 0259 §10): the operator's

internal/inventory/plan_backups_test.go

@@ -0,0 +1,35 @@
+package inventory
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The plan's backups are the contract mesh-delivery reads from `plan-moved` (issue 1065): their JSON field names
+// are fixed here, and a plan keeps them through a round trip.
+func TestAPlanKeepsItsBackupsThroughItsJSONUnderTheNamesTheDeliveryReads(t *testing.T) {
+	at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
+	p := Plan{ID: "p", Backups: &PlanBackups{State: BackupFailed, NoHolder: []string{"bare"},
+		Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupFailed, AskedAt: &at, Words: "no space left"}}}}
+	raw, err := json.Marshal(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range []string{`"backups":{"state":"failed","asked":[{"machine":"ace","module":"mail","state":"failed",` +
+		`"asked_at":"2026-10-17T12:00:00Z","words":"no space left"}],"no_holder":["bare"]}`} {
+		if !strings.Contains(string(raw), name) {
+			t.Fatalf("plan JSON %s lacks %s", raw, name)
+		}
+	}
+	var back Plan
+	if err := json.Unmarshal(raw, &back); err != nil || !reflect.DeepEqual(back.Backups, p.Backups) {
+		t.Fatalf("round trip: %v %+v", err, back.Backups)
+	}
+	raw, _ = json.Marshal(Plan{ID: "q"})
+	if strings.Contains(string(raw), "backups") {
+		t.Fatalf("a plan with no backups says none: %s", raw)
+	}
+}

internal/inventory/plans.go

@@ -62,12 +62,48 @@ type Plan struct {
 	// Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out
 	// from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`).
 	Phases *WalkPhases `json:"phases,omitempty"`
+	// Backups is the backups the walk takes before it asks its first tier (novox/hq ADR 0404 decisions 2 and 3):
+	// per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a
+	// walk that moves no module with an item protected by a backup, and for a walk kept before it was.
+	Backups *PlanBackups `json:"backups,omitempty"`
 	// WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308
 	// decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every
 	// other save.
 	WalkSaid string `json:"-"`
 }
 
+// The states of a plan's backups, as a whole and each.
+const (
+	BackupPending = "pending"
+	BackupDone    = "done"
+	BackupFailed  = "failed"
+)
+
+// PlanBackups is a walk's record of the backups it takes before it asks anything (novox/hq ADR 0404): read by
+// mesh-delivery from `plan-moved`, so its JSON field names are a contract between the two repositories.
+type PlanBackups struct {
+	// State is the whole: pending while any backup is asked or unasked and none failed, failed as soon as one
+	// failed, done when every one is.
+	State string `json:"state"`
+	// Asked is one entry per machine and module, sorted by machine then module.
+	Asked []PlanBackup `json:"asked"`
+	// NoHolder are the machines that run a module with an item protected by a backup and hold no node-backup:
+	// nothing is asked of them, and the plan says so.
+	NoHolder []string `json:"no_holder,omitempty"`
+}
+
+// PlanBackup is one backup asked of one machine's backup holder for one module.
+type PlanBackup struct {
+	Machine string `json:"machine"`
+	Module  string `json:"module"`
+	// State is pending, done or failed.
+	State string `json:"state"`
+	// AskedAt is when the holder was asked; nil while it is not asked yet.
+	AskedAt *time.Time `json:"asked_at,omitempty"`
+	// Words are the backup's own: the holder's error for a failed one, "did not answer in ten minutes" for a silent one.
+	Words string `json:"words,omitempty"`
+}
+
 // PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6).
 type PlanTimes struct {
 	// WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum.