W13.5: the node agent asks mesh-recovery for a restore point before a form change and restores it at a revert

Before a move across a kept data's form change the node agent asks snapshot-kept and waits for the ended of its own act; at a revert it stops the newer build (Env.Stop), asks kept-points, asks restore-kept with the snapshot of the point of the form and fingerprint it reverts from, waits for restored, applies the reverted build, and calls the point's catch-up as the module's tool through the tool runner. The wait is in the have (ask-at, act), bounded by a bounds key of the state bounds (recovery-wait-minutes, 90), and each ending is read as the table of 560 says. The manifest consumes mesh-recovery's ended and restored and invokes its three verbs.

Builds: design 47 G §296, §297, §554, §555, §560, §630, §631, §632, §633, A-77, A-92, A-98, A-108
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h

internal/agent/agent.go

@@ -75,6 +75,8 @@ type Have struct {
 	// PullWait is, on a failed have, the pull from a store its apply failed on and waits on the holder of (issue 1966
 	// ENG-9, nodeengine.HaveByNodeModuleValue.PullWait): read through the have, as node_haves shows it.
 	PullWait *nodeengine.PullWait
+	// Wait is the hold or wait the have says, in the recovery wait with its ask-at and act (design 47 G §560).
+	Wait *nodeengine.Wait
 }
 
 // Haves is the node's have bucket as the agent uses it.
@@ -140,6 +142,9 @@ type Env struct {
 	// build with it, restores the module's kept data, and then applies the reverted build (design 47 G A-98, §297). Nil
 	// stops nothing.
 	Stop func(ctx context.Context, module string) Result
+	// Recovery takes a module's kept data as a restore point before a form change moves it and restores it at a revert
+	// (design 47 G §296, §297, §554); nil moves none.
+	Recovery *Recovery
 	// Backup backs a module up before a new composition of it is applied; nil backs up nothing.
 	Backup func(ctx context.Context, module string) Backup
 	// Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539
@@ -1632,6 +1637,12 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
 		return
 	}
 
+	// A kept data on the bus whose form changes is taken as a restore point first, or restored at a revert (design 47 G §296, §297).
+	held, catchUps := a.keptStep(ctx, module, w, comp, ranBefore)
+	if held {
+		return
+	}
+
 	// A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on
 	// without one (ADR 0539 §16).
 	if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != meshwants.WantsByWantNodeModuleValueBackupSkip {
@@ -1736,6 +1747,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
 		a.env.Settled(module)
 	}
 	a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseRunning), Detail: res.Detail})
+	a.catchUp(ctx, module, catchUps)
 }
 
 // ranFailed marks what runs of a module as failed once an apply of it failed (novox/hq issue 1966, 2026-11-08): a failed

internal/agent/agent_test.go

@@ -930,6 +930,7 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
 
 // A data module with a good backup is applied after it; one that never ran has nothing to back up.
 func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
+	t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange)
 	b := newBed(t)
 	stop := b.run()
 	defer stop()

internal/agent/kept.go

@@ -0,0 +1,323 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
+	"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
+	"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
+	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+)
+
+// A module's kept data on the bus, taken as a restore point before a form change moves it and restored at a revert (design 47 G
+// §296, §297, §554, §555, §560, §630 to §632, the lead's answers A-77, A-98 and A-108): the node agent asks the seat
+// `mesh-recovery`'s `snapshot-kept` and waits for the `ended` of its own act, or, at a revert, stops the newer build, asks
+// `kept-points`, asks `restore-kept` with the snapshot of the point of the form it reverts from, waits for its `restored`, and
+// after the reverted build's apply calls the point's catch-up as the module's tool.
+
+// Ending is the ending event of an act of the recovery module as the node agent reads it: `ended` for a restore point taken,
+// `restored` for one restored (design 47 G §560).
+type Ending struct {
+	Act      string
+	Outcome  string
+	Error    string
+	Snapshot string
+}
+
+// RecoveryWaitMinutes is the default of the manifest's config key `recovery-wait-minutes`, the bound of the wait for an
+// ending event, counted from the first ask (design 47 G §554, §633).
+const RecoveryWaitMinutes = 90
+
+// Recovery is what the node agent needs of the recovery module and the manifests to move kept data. Nil moves none.
+type Recovery struct {
+	// FormChanges are the kept data whose form differs between the running build's manifest and the applied build's (the
+	// catalog half's entries by fingerprint); an error says it is not known.
+	FormChanges func(ctx context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error)
+	// The seat's verbs, asked in the node-engine's own name; an error is the ask not answered.
+	SnapshotKept func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error)
+	RestoreKept  func(ctx context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error)
+	KeptPoints   func(ctx context.Context, in meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error)
+	// Hear follows the ending events of kind "ended" or "restored" of one module's kept data: the last one said already first,
+	// then each after it, until stop is called. It is made before the verb is asked (design 47 G §560).
+	Hear func(ctx context.Context, kind, module, kept string) (events <-chan Ending, stop func(), err error)
+	// Bound writes the bound key `<what>.<id>` of a wait in the state bounds with its time to live, so its end is the bus's
+	// marker, heard as expired (design 47 G A-77, A-92); end deletes it. Nil keeps no bound.
+	Bound func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (expired <-chan struct{}, end func(), err error)
+	// CatchUp calls the module's tool on this node through the tool runner (design 47 G A-108).
+	CatchUp func(ctx context.Context, module string, c meshrecovery.CatchUp) error
+	// WaitMinutes is the bound in minutes, the config key recovery-wait-minutes; 0 reads as RecoveryWaitMinutes.
+	WaitMinutes int
+	// Now is the clock; nil reads the machine's.
+	Now func() time.Time
+}
+
+func (r *Recovery) now() time.Time {
+	if r.Now != nil {
+		return r.Now()
+	}
+	return time.Now()
+}
+
+func (r *Recovery) bound() time.Duration {
+	if r.WaitMinutes > 0 {
+		return time.Duration(r.WaitMinutes) * time.Minute
+	}
+	return RecoveryWaitMinutes * time.Minute
+}
+
+// KeptMove is one kept data a want moves across a change of its form, as the seat mesh-recovery's verbs name it.
+type KeptMove struct {
+	// Name is the kept data's local name, Form the form the restore point holds (the older one), and Fingerprint the
+	// fingerprint the restore point is taken before (the newer build's) or reverted from.
+	Name        string
+	Form        int
+	Fingerprint string
+	// CatchUp is, when taking the point, the tool the newer manifest declares beside the kept data.
+	CatchUp *meshrecovery.CatchUp
+}
+
+// keptMoves are the restore points a move needs: a point to take before the newer build starts (the kept data's form rises,
+// or the newer manifest declares it no more) and a point to restore before the older build starts (the kept data goes back to
+// a lower form). A kept data the running build does not declare is neither: nothing was written before it.
+func keptMoves(changes []catalogue.FormChange, runningFp, appliedFp string) (take, restore []KeptMove) {
+	for _, c := range changes {
+		switch {
+		case c.Running == nil:
+		case c.Applied == nil || c.Applied.Form > c.Running.Form:
+			m := KeptMove{Name: c.Name, Form: c.Running.Form, Fingerprint: appliedFp}
+			if c.Applied != nil && c.Applied.CatchUp != nil {
+				m.CatchUp = &meshrecovery.CatchUp{Tool: c.Applied.CatchUp.Tool, Arguments: c.Applied.CatchUp.Arguments}
+			}
+			take = append(take, m)
+		case c.Applied.Form < c.Running.Form:
+			restore = append(restore, KeptMove{Name: c.Name, Form: c.Applied.Form, Fingerprint: runningFp})
+		}
+	}
+	return take, restore
+}
+
+// recoveryHeld is how a wait for the recovery module ended when the move cannot go on: the words are the reason of the hold.
+type recoveryHeld struct {
+	Words       string
+	Interrupted bool
+}
+
+// askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the
+// ending of the act it names and asks again, `failed` with `stopped` asks again within the same bound, a failure with any
+// other error holds. The wait is written in the have with its first ask's time and the act, so a restart waits for that act
+// and asks nothing again; past the bound the have says the wait, and the wait goes on (the watchman raises it).
+func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, ask func() (accepted, busy, errWords string, err error)) (Ending, *recoveryHeld) {
+	r := a.env.Recovery
+	module := h.Module
+	step := nodeengine.WaitStepCompose
+	wait := nodeengine.Wait{Owner: nodeengine.Seat, Step: step, On: "mesh-recovery." + kind + "." + module + "." + k.Name,
+		Reason: "recovery waits", Bound: r.bound().String()}
+	// A restart: the have names the wait with its ask time and act.
+	last, ok := a.env.Haves.Read(ctx, module)
+	if ok && last.Wait != nil && last.Wait.On == wait.On && last.Wait.AskAt != nil {
+		wait.AskAt, wait.Act, wait.Since = last.Wait.AskAt, last.Wait.Act, last.Wait.Since
+	}
+	now := r.now()
+	if wait.AskAt == nil {
+		wait.AskAt, wait.Since = &now, now
+	}
+	events, stop, err := r.Hear(ctx, kind, module, k.Name)
+	if err != nil {
+		return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act cannot be heard: " + err.Error()}
+	}
+	defer stop()
+	var expired <-chan struct{}
+	if r.Bound != nil {
+		bound := r.bound() - now.Sub(*wait.AskAt)
+		if bound < time.Second {
+			bound = time.Second
+		}
+		exp, end, berr := r.Bound(ctx, "recovery", module+"."+kind+"."+k.Name, module, *wait.AskAt, bound)
+		if berr != nil {
+			return Ending{}, &recoveryHeld{Words: "the bound of the wait for the recovery module cannot be written: " + berr.Error()}
+		}
+		defer end()
+		expired = exp
+	}
+	write := func() {
+		w := wait
+		h.Wait = &w
+		a.writeHave(ctx, h)
+	}
+	act, busyOn := wait.Act, ""
+	for {
+		if act == "" && busyOn == "" {
+			write() // the first ask's time is in the have before the ask
+			accepted, busy, words, err := ask()
+			switch {
+			case err != nil:
+				if ctx.Err() != nil {
+					return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
+				}
+				return Ending{}, &recoveryHeld{Words: "the recovery module could not be asked: " + err.Error()}
+			case words != "":
+				return Ending{}, &recoveryHeld{Words: words}
+			case accepted != "":
+				act, wait.Act = accepted, accepted
+				write()
+			case busy != "":
+				busyOn = busy
+			default:
+				return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy"}
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
+		case <-expired:
+			expired = nil
+			wait.Words = fmt.Sprintf("the recovery module has not ended its act within %s", r.bound())
+			write()
+		case ev, open := <-events:
+			if !open {
+				return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act is no longer heard"}
+			}
+			switch {
+			case busyOn != "" && ev.Act == busyOn:
+				busyOn = ""
+			case act != "" && ev.Act == act:
+				switch {
+				case ev.Outcome == meshrecovery.EndedEventOutcomeDone:
+					return ev, nil
+				case ev.Error == "stopped":
+					act, wait.Act = "", ""
+				case ev.Error == "cancelled":
+					return ev, &recoveryHeld{Words: "cancelled"}
+				default:
+					words := ev.Error
+					if words == "" {
+						words = "the recovery module's act failed"
+					}
+					return ev, &recoveryHeld{Words: words}
+				}
+			}
+		}
+	}
+}
+
+// keptStep takes the restore points a move of a module needs before it applies and restores the ones a revert needs: it
+// answers whether the module is held (its have says why, and the running build is kept) and, after a restore, the catch-up
+// of the point to call once the reverted build runs. A want that says to go on without a backup asks neither.
+func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool) (held bool, catchUps []meshrecovery.CatchUp) {
+	r := a.env.Recovery
+	if !ranBefore || r == nil || r.FormChanges == nil || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip {
+		return false, nil
+	}
+	fp, dig := w.Want.Fp, w.Want.Composed
+	a.mu.Lock()
+	prev, runs := a.running[module]
+	a.mu.Unlock()
+	if !runs {
+		if h, found, _ := a.readHave(ctx, module); found {
+			prev = nodeengine.HaveByNodeModuleValue{Fp: h.FP, Composed: h.Composed}
+		}
+	}
+	if prev.Fp == "" || prev.Fp == fp {
+		return false, nil
+	}
+	changes, err := r.FormChanges(ctx, module, prev.Fp, fp)
+	if err != nil {
+		// Not known whether the move changes a form: said, and the move goes on as before, since holding every module for a
+		// build record that is gone would stop deliveries a form never changed.
+		a.sayOnce("kept:"+module+":"+fp, fmt.Sprintf("%s: whether its kept data changes form between %s and %s is not known: %v", module, prev.Fp, fp, err))
+		return false, nil
+	}
+	take, restore := keptMoves(changes, prev.Fp, fp)
+	if len(take)+len(restore) == 0 {
+		return false, nil
+	}
+	hold := func(words string, interrupted bool) {
+		if interrupted || ctx.Err() != nil {
+			words = BackupInterrupted
+			a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words))
+		} else {
+			a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words))
+		}
+		a.mu.Lock()
+		a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
+		a.mu.Unlock()
+		since := r.now()
+		a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
+			Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}})
+	}
+	backing := Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseBackingUp)}
+	for _, m := range take {
+		why := fmt.Sprintf("form change of %s's %s to %s", module, m.Name, fp)
+		_, held := a.askAndWait(ctx, backing, "ended", m, func() (string, string, string, error) {
+			got, err := r.SnapshotKept(ctx, meshrecovery.SnapshotKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form),
+				Fingerprint: m.Fingerprint, CatchUp: m.CatchUp, Why: why})
+			return got.Accepted, got.Busy, got.Error, err
+		})
+		if held != nil {
+			hold(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted)
+			return true, nil
+		}
+	}
+	if len(restore) == 0 {
+		return false, nil
+	}
+	// A revert: the newer build stops first (it removes nothing), so the live data is not written while it is set aside.
+	if a.env.Stop != nil {
+		if res := a.env.Stop(ctx, module); res.Failed {
+			hold("the newer build could not be stopped before the restore: "+res.Detail, false)
+			return true, nil
+		}
+	}
+	for _, m := range restore {
+		pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: module, Kept: m.Name})
+		if err != nil {
+			hold(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false)
+			return true, nil
+		}
+		var point *meshrecovery.Point
+		for i := range pts.Points {
+			p := pts.Points[i]
+			if p.Module == module && p.Kept == m.Name && int(p.Form) == m.Form && p.Fingerprint == m.Fingerprint {
+				if point == nil || p.At.After(point.At) {
+					point = &pts.Points[i]
+				}
+			}
+		}
+		if point == nil {
+			hold(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false)
+			return true, nil
+		}
+		why := fmt.Sprintf("revert of %s's %s from %s", module, m.Name, m.Fingerprint)
+		_, held := a.askAndWait(ctx, backing, "restored", m, func() (string, string, string, error) {
+			got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form),
+				Fingerprint: m.Fingerprint, Snapshot: point.Snapshot, Why: why})
+			return got.Accepted, got.Busy, got.Error, err
+		})
+		if held != nil {
+			hold(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted)
+			return true, nil
+		}
+		if point.CatchUp != nil {
+			catchUps = append(catchUps, *point.CatchUp)
+		}
+	}
+	return false, catchUps
+}
+
+// catchUp calls, once the reverted build of a module runs, the catch-up of each restore point restored, as the module's own
+// tool through the tool runner (design 47 G A-108). A tool that fails is said: the build runs, and the tool can be called by hand.
+func (a *Agent) catchUp(ctx context.Context, module string, calls []meshrecovery.CatchUp) {
+	if a.env.Recovery == nil || a.env.Recovery.CatchUp == nil {
+		return
+	}
+	for _, c := range calls {
+		if err := a.env.Recovery.CatchUp(ctx, module, c); err != nil {
+			a.env.Say(fmt.Sprintf("%s runs the reverted build, and its catch-up tool %s failed: %v", module, c.Tool, err))
+			continue
+		}
+		a.env.Say(fmt.Sprintf("%s's catch-up tool %s ran after the restore", module, c.Tool))
+	}
+}

internal/agent/kept_test.go

@@ -0,0 +1,257 @@
+package agent
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
+	"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"github.com/novox/mesh-host/internal/compositions"
+)
+
+// A recovery module that answers as scripted: each ask takes the next answer and says the events it carries after it.
+type recoveryFake struct {
+	mu      sync.Mutex
+	asks    []string // verb and its input, in order
+	answers []fakeAnswer
+	points  []meshrecovery.Point
+	events  chan Ending
+	changes []catalogue.FormChange
+	calls   []string // stop, ask, apply, catch-up, in order
+	catchUp []meshrecovery.CatchUp
+	expired chan struct{}
+}
+
+type fakeAnswer struct {
+	accepted, busy, errWords string
+	then                     []Ending
+}
+
+func (f *recoveryFake) next(verb string, in any) (string, string, string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.asks = append(f.asks, fmt.Sprintf("%s %s", verb, asJSON(in)))
+	f.calls = append(f.calls, verb)
+	if len(f.answers) == 0 {
+		return "", "", "no answer scripted"
+	}
+	a := f.answers[0]
+	f.answers = f.answers[1:]
+	for _, e := range a.then {
+		f.events <- e
+	}
+	return a.accepted, a.busy, a.errWords
+}
+
+func (f *recoveryFake) env() *Recovery {
+	return &Recovery{
+		FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { return f.changes, nil },
+		SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) {
+			a, b, e := f.next("snapshot-kept", in)
+			return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
+		},
+		RestoreKept: func(_ context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error) {
+			a, b, e := f.next("restore-kept", in)
+			return meshrecovery.RestoreKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
+		},
+		KeptPoints: func(context.Context, meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error) {
+			f.mu.Lock()
+			defer f.mu.Unlock()
+			f.calls = append(f.calls, "kept-points")
+			return meshrecovery.KeptPointsAnswer{Points: f.points}, nil
+		},
+		Hear: func(context.Context, string, string, string) (<-chan Ending, func(), error) {
+			return f.events, func() {}, nil
+		},
+		Bound: func(context.Context, string, string, string, time.Time, time.Duration) (<-chan struct{}, func(), error) {
+			return f.expired, func() {}, nil
+		},
+		CatchUp: func(_ context.Context, _ string, c meshrecovery.CatchUp) error {
+			f.mu.Lock()
+			defer f.mu.Unlock()
+			f.calls = append(f.calls, "catch-up")
+			f.catchUp = append(f.catchUp, c)
+			return nil
+		},
+		Now: time.Now,
+	}
+}
+
+func (f *recoveryFake) seen() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return slices.Clone(f.calls)
+}
+
+// moveDB runs a data module at fp "a", then wants it at fp "b", and answers the bed once the second want is settled or held.
+func moveDB(t *testing.T, f *recoveryFake, ranFrom, to string, settle func(b *bed) bool, before ...func(b *bed, dig string)) *bed {
+	t.Helper()
+	b := newBed(t)
+	inner, applies := b.env.Apply, 0
+	b.env.Apply = func(ctx context.Context, c compositions.Composed, others []compositions.Composed) Result {
+		if applies++; applies >= 2 {
+			f.mu.Lock()
+			f.calls = append(f.calls, "apply")
+			f.mu.Unlock()
+		}
+		return inner(ctx, c, others)
+	}
+	f.events, f.expired = make(chan Ending, 16), make(chan struct{})
+	b.env.Recovery = f.env()
+	b.env.Stop = func(context.Context, string) Result {
+		f.mu.Lock()
+		f.calls = append(f.calls, "stop")
+		f.mu.Unlock()
+		return Result{}
+	}
+	stop := b.run()
+	t.Cleanup(stop)
+	fp1, fp2 := fpOf(ranFrom), fpOf(to)
+	ck, craw, dig := composition(t, "db", fp1, true, file("db", "x", "1"))
+	wk, wraw := wantFor(t, "db", fp1, dig, "")
+	b.comps.put(ck, craw)
+	b.wants.put(wk, wraw)
+	eventually(t, "the first build is not applied", func() bool { return len(b.appliedList()) == 1 })
+	b.idle()
+	ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2"))
+	for _, h := range before {
+		h(b, dig)
+	}
+	wk2, wraw2 := wantFor(t, "db", fp2, dig2, "")
+	b.comps.put(ck2, craw2)
+	b.wants.put(wk2, wraw2)
+	eventually(t, "the move is neither applied nor held", func() bool { return settle(b) })
+	return b
+}
+
+// testAMoveAcrossAFormChange is design 47 G §296, §297, §554, §560, §630 to §632 and A-98, A-108 at the node agent.
+func testAMoveAcrossAFormChange(t *testing.T) {
+	up := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1},
+		Applied: &catalogue.KeptLayout{Form: 2, CatchUp: &catalogue.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}}
+	down := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 2},
+		Applied: &catalogue.KeptLayout{Form: 1}}}
+	fp := func(c string) string { return fpOf(c) }
+	done := func(act string) Ending { return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeDone} }
+	failed := func(act, why string) Ending {
+		return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeFailed, Error: why}
+	}
+	applied := func(b *bed) bool { return len(b.appliedList()) == 2 }
+	heldHave := func(b *bed) bool { return b.haves.last("db").Phase == "held" }
+
+	t.Run("a form that rises is taken as a restore point before the move, with the why and the catch-up, and then applied", func(t *testing.T) {
+		f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{done("a1")}}}}
+		moveDB(t, f, "a", "b", applied)
+		if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) {
+			t.Fatalf("calls %v", got)
+		}
+		want := fmt.Sprintf("snapshot-kept %s", asJSON(meshrecovery.SnapshotKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("b"),
+			Why: "form change of db's cache to " + fp("b"), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}))
+		if f.asks[0] != want {
+			t.Fatalf("asked %s, want %s", f.asks[0], want)
+		}
+	})
+
+	t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
+		f := &recoveryFake{changes: up, answers: []fakeAnswer{
+			{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
+			{accepted: "a2", then: []Ending{failed("a2", "stopped")}},
+			{accepted: "a3", then: []Ending{done("a3")}},
+		}}
+		moveDB(t, f, "a", "b", applied)
+		if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "snapshot-kept", "snapshot-kept", "apply"}) {
+			t.Fatalf("calls %v", got)
+		}
+	})
+
+	for _, row := range []struct{ error, held string }{
+		{"cancelled", "cancelled"},
+		{"cap", "cap"},
+		{"restore point damaged", "restore point damaged"},
+		{"disk full", "disk full"},
+	} {
+		t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) {
+			f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}}
+			b := moveDB(t, f, "a", "b", heldHave)
+			if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" {
+				t.Fatalf("the have %+v", h)
+			}
+			if got := f.seen(); len(got) != 1 {
+				t.Fatalf("asked again: %v", got)
+			}
+			if n := len(b.appliedList()); n != 1 {
+				t.Fatalf("applied %d", n)
+			}
+		})
+	}
+
+	t.Run("the bound passed says the wait in the have and the wait goes on to the ending", func(t *testing.T) {
+		f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1"}}}
+		b := moveDB(t, f, "a", "b", func(b *bed) bool {
+			b.haves.mu.Lock()
+			defer b.haves.mu.Unlock()
+			for _, h := range b.haves.log {
+				if h.Wait != nil && h.Wait.Words != "" && h.Phase == "backing-up" {
+					return true
+				}
+			}
+			return false
+		}, func(*bed, string) { close(f.expired) })
+		f.events <- done("a1")
+		eventually(t, "the move went on after the ending", func() bool { return applied(b) })
+	})
+
+	t.Run("a revert stops the newer build, restores the point of the form it reverts from, applies, then calls its catch-up", func(t *testing.T) {
+		f := &recoveryFake{changes: down,
+			points: []meshrecovery.Point{
+				{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("z"), Snapshot: "other", At: time.Now()},
+				{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now(), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild"}}},
+			answers: []fakeAnswer{{accepted: "r1", then: []Ending{done("r1")}}}}
+		moveDB(t, f, "a", "b", func(b *bed) bool { return len(f.seen()) >= 5 })
+		if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply", "catch-up"}) {
+			t.Fatalf("calls %v", got)
+		}
+		if f.catchUp[0].Tool != "rebuild" {
+			t.Fatalf("catch-up %+v", f.catchUp)
+		}
+		want := fmt.Sprintf("restore-kept %s", asJSON(meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"),
+			Snapshot: "snap-1", Why: "revert of db's cache from " + fp("a")}))
+		if f.asks[0] != want {
+			t.Fatalf("asked %s, want %s", f.asks[0], want)
+		}
+	})
+
+	t.Run("a revert with no point of its form holds as a restore point missing", func(t *testing.T) {
+		f := &recoveryFake{changes: down}
+		b := moveDB(t, f, "a", "b", heldHave)
+		if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") {
+			t.Fatalf("the have %+v", h)
+		}
+	})
+
+	t.Run("a restart waits for the act its have names and asks nothing again", func(t *testing.T) {
+		f := &recoveryFake{changes: up}
+		b := moveDB(t, f, "a", "b", applied, func(b *bed, dig string) {
+			at := time.Now().Add(-time.Minute)
+			b.haves.mu.Lock()
+			b.haves.stored["db"] = Have{Module: "db", FP: fp("a"), Phase: "backing-up", Wait: &nodeengine.Wait{
+				On: "mesh-recovery.ended.db.cache", AskAt: &at, Act: "a1", Since: at}}
+			b.haves.mu.Unlock()
+			f.events <- done("a1")
+		})
+		_ = b
+		if got := f.seen(); !slices.Equal(got, []string{"apply"}) {
+			t.Fatalf("calls %v", got)
+		}
+	})
+}
+
+func contains(s, sub string) bool { return strings.Contains(s, sub) }
+
+func asJSON(v any) string { raw, _ := json.Marshal(v); return string(raw) }

internal/link/have.go

@@ -409,6 +409,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
 		if h.Phase == nodeengineclient.PhaseFailed {
 			h.PullWait = f.PullWait
 		}
+		h.Wait = f.Wait
 		h.Health = s.healthOf(module, f)
 		if h.Detail == "" {
 			h.Detail = s.wordsOf(module, h.Health)

internal/link/messages.go

@@ -647,4 +647,6 @@ type ModuleFact struct {
 	Sealed compositions.Seals
 	// PullWait is, on a failed fact, the pull from a store its apply waits on (nodeengine.HaveByNodeModuleValue.PullWait, issue 1966 ENG-9).
 	PullWait *nodeengine.PullWait
+	// Wait is the hold or wait the module has, with the ask-at and act of a recovery wait (nodeengine.HaveByNodeModuleValue.Wait, design 47 G §560).
+	Wait *nodeengine.Wait
 }

internal/link/recovery.go

@@ -0,0 +1,241 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// The node agent's wait for the recovery module (design 47 G §554, §560, A-77, A-92): the ending events of one module's kept data
+// are heard through an ordered consumer that begins at the last message of the subject and follows the rest, and a wait's bound
+// is a key of the state bounds with its own time to live, ended by the bus's marker.
+
+// BoundsBucket is the node-engine's state `bounds`, named by its holder as every state is.
+const BoundsBucket = "mesh-host_bounds"
+
+// RecoveryBus is a link that hears the recovery module's events and keeps a bound.
+type RecoveryBus interface {
+	// HearEnding calls heard with the last message of the subject on the stream, then each after it, until stop is called.
+	HearEnding(stream, subject string, heard func([]byte)) (stop func(), err error)
+	// BoundCreate writes the bound key with its time to live; one that stands is left as it is. BoundWatch calls ended once the
+	// bus's marker says the key's time ended; BoundDelete ends it.
+	BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error
+	BoundWatch(ctx context.Context, key string, ended func()) (stop func(), err error)
+	BoundDelete(ctx context.Context, key string) error
+}
+
+func hearEnding(js nats.JetStreamContext, stream, subject string, heard func([]byte)) (func(), error) {
+	sub, err := js.Subscribe(subject, func(m *nats.Msg) { heard(m.Data) }, nats.OrderedConsumer(), nats.DeliverLast(), nats.BindStream(stream))
+	if err != nil {
+		return nil, fmt.Errorf("the stream %s cannot be followed: %w", stream, err)
+	}
+	return func() { _ = sub.Unsubscribe() }, nil
+}
+
+func boundKV(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
+	js, err := jetstream.New(conn)
+	if err != nil {
+		return nil, err
+	}
+	kv, err := js.KeyValue(ctx, BoundsBucket)
+	if err != nil {
+		return nil, fmt.Errorf("the state %s cannot be bound: %w", BoundsBucket, err)
+	}
+	return kv, nil
+}
+
+func boundCreate(ctx context.Context, conn *nats.Conn, key string, value []byte, ttl time.Duration) error {
+	kv, err := boundKV(ctx, conn)
+	if err != nil {
+		return err
+	}
+	if _, err := kv.Create(ctx, key, value, jetstream.KeyTTL(ttl)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
+		return err
+	}
+	return nil
+}
+
+func boundWatch(ctx context.Context, conn *nats.Conn, key string, ended func()) (func(), error) {
+	kv, err := boundKV(ctx, conn)
+	if err != nil {
+		return nil, err
+	}
+	w, err := kv.Watch(ctx, key, jetstream.UpdatesOnly())
+	if err != nil {
+		return nil, err
+	}
+	go func() {
+		for e := range w.Updates() {
+			if e != nil && e.Operation() == jetstream.KeyValuePurge {
+				ended()
+			}
+		}
+	}()
+	return func() { _ = w.Stop() }, nil
+}
+
+func boundDelete(ctx context.Context, conn *nats.Conn, key string) error {
+	kv, err := boundKV(ctx, conn)
+	if err != nil {
+		return err
+	}
+	if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		return err
+	}
+	return nil
+}
+
+func (l *natsLink) HearEnding(stream, subject string, heard func([]byte)) (func(), error) {
+	return hearEnding(l.js, stream, subject, heard)
+}
+func (l *natsLink) BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	return boundCreate(ctx, l.conn, key, value, ttl)
+}
+func (l *natsLink) BoundWatch(ctx context.Context, key string, ended func()) (func(), error) {
+	return boundWatch(ctx, l.conn, key, ended)
+}
+func (l *natsLink) BoundDelete(ctx context.Context, key string) error {
+	return boundDelete(ctx, l.conn, key)
+}
+
+func (b OverNATS) HearEnding(stream, subject string, heard func([]byte)) (func(), error) {
+	return hearEnding(b.JS, stream, subject, heard)
+}
+func (b OverNATS) BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error {
+	return boundCreate(ctx, b.Conn, key, value, ttl)
+}
+func (b OverNATS) BoundWatch(ctx context.Context, key string, ended func()) (func(), error) {
+	return boundWatch(ctx, b.Conn, key, ended)
+}
+func (b OverNATS) BoundDelete(ctx context.Context, key string) error {
+	return boundDelete(ctx, b.Conn, key)
+}
+
+var (
+	_ RecoveryBus = (*natsLink)(nil)
+	_ RecoveryBus = OverNATS{}
+)
+
+func (q *Session) recoveryBus() (RecoveryBus, error) {
+	q.init()
+	q.mu.Lock()
+	bus := q.bus
+	q.mu.Unlock()
+	h, ok := bus.(RecoveryBus)
+	if bus == nil || !ok {
+		return nil, ErrNoLink
+	}
+	return h, nil
+}
+
+// RecoveryEnding is the ending event of an act as the node agent reads it, the fields every ending carries.
+type RecoveryEnding struct {
+	Act      string `json:"act"`
+	Outcome  string `json:"outcome"`
+	Error    string `json:"error,omitempty"`
+	Snapshot string `json:"snapshot,omitempty"`
+}
+
+// HearRecoveryEnding follows the events `ended` (kind "ended") or `restored` (kind "restored") of one module's kept data: the last
+// said first, then each after it, until stop is called.
+func (q *Session) HearRecoveryEnding(kind, module, kept string) (<-chan RecoveryEnding, func(), error) {
+	bus, err := q.recoveryBus()
+	if err != nil {
+		return nil, nil, err
+	}
+	stream, subject := meshrecovery.StreamEnded, meshrecovery.SubjectEnded(module, kept)
+	if kind == "restored" {
+		stream, subject = meshrecovery.StreamRestored, meshrecovery.SubjectRestored(module, kept)
+	}
+	out := make(chan RecoveryEnding, 64)
+	stop, err := bus.HearEnding(stream, subject, func(raw []byte) {
+		var ev RecoveryEnding
+		if json.Unmarshal(raw, &ev) == nil && ev.Act != "" {
+			select {
+			case out <- ev:
+			default:
+			}
+		}
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	return out, stop, nil
+}
+
+// KeepBound writes the bound key `<what>.<id>` of a wait with its time to live and watches it: expired is closed when the bus's
+// marker says its time ended, and end deletes it.
+func (q *Session) KeepBound(ctx context.Context, what, id, module string, since time.Time, ttl time.Duration) (<-chan struct{}, func(), error) {
+	bus, err := q.recoveryBus()
+	if err != nil {
+		return nil, nil, err
+	}
+	key := what + "." + id
+	value, err := json.Marshal(struct {
+		Module string `json:"module,omitempty"`
+		Since  string `json:"since"`
+		V      int64  `json:"v"`
+	}{module, since.UTC().Format(time.RFC3339), 1})
+	if err != nil {
+		return nil, nil, err
+	}
+	expired := make(chan struct{})
+	stopWatch, err := bus.BoundWatch(ctx, key, func() {
+		select {
+		case <-expired:
+		default:
+			close(expired)
+		}
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := bus.BoundCreate(ctx, key, value, ttl); err != nil {
+		stopWatch()
+		return nil, nil, err
+	}
+	return expired, func() {
+		stopWatch()
+		_ = bus.BoundDelete(context.WithoutCancel(ctx), key)
+	}, nil
+}
+
+// CallTool calls a module's tool on this machine's own node tools with its arguments and answers its refusal as an error (design 47 G
+// A-108: the catch-up of a restore point is called so, as the node-engine is granted to).
+func (q *Session) CallTool(ctx context.Context, module, tool string, args map[string]any) error {
+	q.init()
+	q.mu.Lock()
+	bus := q.bus
+	q.mu.Unlock()
+	asker, ok := bus.(ToolBus)
+	if bus == nil || !ok {
+		return ErrNoLink
+	}
+	if args == nil {
+		args = map[string]any{}
+	}
+	body, err := json.Marshal(args)
+	if err != nil {
+		return err
+	}
+	reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), body)
+	if err != nil {
+		return err
+	}
+	var env struct {
+		Error string `json:"error"`
+	}
+	if err := json.Unmarshal(reply, &env); err != nil {
+		return fmt.Errorf("its answer is not a tool's answer: %w", err)
+	}
+	if env.Error != "" {
+		return errors.New(env.Error)
+	}
+	return nil
+}

internal/selfcompose/entries.go

@@ -15,6 +15,7 @@ import (
 	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
 	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
 	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
 
 	"github.com/novox/mesh-host/internal/agent"
 )
@@ -514,3 +515,32 @@ func EntryCall(ask func(ctx context.Context, seat, verb string, args any) (json.
 
 // noEntryWords are the words the seat's holder refuses an unknown fingerprint in.
 const noEntryWords = "no entry of fingerprint"
+
+// FormChanges are the kept data whose form differs between the manifests of the build a module runs and the one a want moves it
+// to, each read from the entry the composer holds of its fingerprint (design 47 G §296, §297). An entry not held is an error: the
+// node agent then says it is not known and the move goes on.
+func (c *Composer) FormChanges(_ context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error) {
+	if !c.byEntries() {
+		return nil, fmt.Errorf("this engine reads no entry by fingerprint")
+	}
+	read := func(fp string) (catalogue.Manifest, error) {
+		e, state := c.entryOf(fp)
+		if state != meshcatalogclient.EntryAnswered {
+			return catalogue.Manifest{}, fmt.Errorf("the entry of %s is %s", fp, state)
+		}
+		_, raw, err := entryManifest(e)
+		if err != nil {
+			return catalogue.Manifest{}, err
+		}
+		return catalogue.Decode(raw)
+	}
+	running, err := read(runningFp)
+	if err != nil {
+		return nil, fmt.Errorf("%s's running manifest: %w", module, err)
+	}
+	applied, err := read(appliedFp)
+	if err != nil {
+		return nil, fmt.Errorf("%s's wanted manifest: %w", module, err)
+	}
+	return catalogue.FormChanges(running, applied), nil
+}