Ask a data module's backup again while nothing on the machine answers it, within three minutes, so a tool-runner restart no longer holds the module silently (issue 774)

On 2026-10-30 ace's redis, postgres and restic (14:03), step-ca (13:55) and mesh-vault on novox (15:22)
were held on a single backup ask that met no responder while the tool runner restarted.

Quick fix: ask node-backup.now twice. Thorough fix (chosen): every node-backup ask (backed-up and now)
is asked again with the engine's backoff while it is known never to have been delivered (ErrNotAsked),
bounded at three minutes; an ask that may have been delivered (a timeout, a refusal) is never asked
twice, so no backup starts twice. Past the bound the backup fails saying how often and how long it was
asked, the module is held and its have carries those words for the assigner's ask and condition. This
was the engine's only one-shot seat ask.

cmd/mesh-host/backup_event_test.go

@@ -3,9 +3,13 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/novox/mesh-host/internal/link"
 )
 
 func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
@@ -39,4 +43,34 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
 	if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
 		t.Errorf("silence within the bound: done=%v words=%q", done, words)
 	}
+
+	// The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it
+	// while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have
+	// reached it is never asked twice.
+	notAsked := fmt.Errorf("node-backup.now could not be asked: %w: nothing on this machine answers "+
+		"mesh.seat.node-backup.tool.now.ace", link.ErrNotAsked)
+	for _, row := range []struct {
+		name    string
+		answers []error
+		within  time.Duration
+		asked   int
+		words   string
+	}{
+		{"heard after the tool runner is back", []error{notAsked, notAsked, nil}, time.Second, 3, ""},
+		// Asked again until the bound, however slow the machine: at least twice, and fewer than it would answer.
+		{"never heard within the bound", []error{notAsked, notAsked, notAsked, notAsked, notAsked, nil}, 25 * time.Millisecond,
+			0, "times over"},
+		{"a timeout may have started it", []error{errors.New("no answer from x: context deadline exceeded"), nil},
+			time.Second, 1, "deadline exceeded"},
+		{"a refusal in words", []error{errors.New("restic is not on this machine"), nil}, time.Second, 1, "not on this"},
+	} {
+		asked := 0
+		_, err := askUntilHeard(context.Background(), func(context.Context) (json.RawMessage, error) {
+			asked++
+			return json.RawMessage(`{}`), row.answers[asked-1]
+		}, row.within, func(int) time.Duration { return 10 * time.Millisecond })
+		if row.asked == 0 && (asked < 2 || asked >= len(row.answers)) || row.asked != 0 && asked != row.asked || (row.words == "") != (err == nil) || err != nil && !strings.Contains(err.Error(), row.words) {
+			t.Errorf("%s: asked %d times, %v", row.name, asked, err)
+		}
+	}
 }

cmd/mesh-host/nodeagent.go

@@ -165,19 +165,54 @@ func backupNode(session *link.Session) func(ctx context.Context, module string)
 		defer stop()
 		// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
 		// 1945). An answer that cannot be had or read asks for the backup as before.
-		if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil {
+		ask := func(verb string) func(context.Context) (json.RawMessage, error) {
+			return func(ctx context.Context) (json.RawMessage, error) {
+				return session.AskSeat(ctx, "node-backup", verb, map[string]any{"module": module}, 30*time.Second)
+			}
+		}
+		if said, err := askUntilHeard(ctx, ask("backed-up"), backupAskFor, link.Backoff); err == nil {
 			if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
 				return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
 			}
 		}
 		asked := time.Now()
-		if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil {
+		if _, err := askUntilHeard(ctx, ask("now"), backupAskFor, link.Backoff); err != nil {
 			return agent.Backup{Words: err.Error()}
 		}
 		return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin)
 	}
 }
 
+// backupAskFor is how long the backup's ask is asked again while it is known never to have reached the holder of
+// node-backup: nothing on this machine answers it (the tool runner restarting, its subscriptions not made yet) or no
+// link is open. The sweep of 2026-10-30 met it on ace's redis, postgres and restic (14:03), step-ca (13:55) and
+// mesh-vault on novox (15:22): each was held on one unanswered ask. Past the bound the backup failed, said with how
+// often and how long it was asked: the module is held, its have says so, and the assigner asks the operator.
+var backupAskFor = 3 * time.Minute
+
+// askUntilHeard asks, and asks again after wait(try) while the answer says the ask was never delivered (link.ErrNotAsked),
+// for at most within: an ask that may have reached its holder (a timeout, a refusal in words) is never asked twice, so a
+// backup is never started twice.
+func askUntilHeard(ctx context.Context, ask func(context.Context) (json.RawMessage, error), within time.Duration,
+	wait func(int) time.Duration) (json.RawMessage, error) {
+	began := time.Now()
+	for try := 1; ; try++ {
+		said, err := ask(ctx)
+		if err == nil || !errors.Is(err, link.ErrNotAsked) {
+			return said, err
+		}
+		pause := wait(try)
+		if time.Since(began)+pause > within {
+			return nil, fmt.Errorf("asked %d times over %s and never heard: %w", try, time.Since(began).Round(time.Second), err)
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("the node agent was stopped while it asked again: %w", err)
+		case <-time.After(pause):
+		}
+	}
+}
+
 // awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it
 // says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node
 // or module, or from before the ask, is not the answer.