diff --git a/cmd/mesh-host/backup_event_test.go b/cmd/mesh-host/backup_event_test.go index 0a773cc..5871dc1 100644 --- a/cmd/mesh-host/backup_event_test.go +++ b/cmd/mesh-host/backup_event_test.go @@ -3,9 +3,13 @@ package main import ( "context" "encoding/json" + "errors" + "fmt" "strings" "testing" "time" + + "github.com/novox/mesh-host/internal/link" ) func backupEvent(node, module string, ok bool, at time.Time, why string) []byte { @@ -39,4 +43,34 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) { if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") { t.Errorf("silence within the bound: done=%v words=%q", done, words) } + + // The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it + // while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have + // reached it is never asked twice. + notAsked := fmt.Errorf("node-backup.now could not be asked: %w: nothing on this machine answers "+ + "mesh.seat.node-backup.tool.now.ace", link.ErrNotAsked) + for _, row := range []struct { + name string + answers []error + within time.Duration + asked int + words string + }{ + {"heard after the tool runner is back", []error{notAsked, notAsked, nil}, time.Second, 3, ""}, + // Asked again until the bound, however slow the machine: at least twice, and fewer than it would answer. + {"never heard within the bound", []error{notAsked, notAsked, notAsked, notAsked, notAsked, nil}, 25 * time.Millisecond, + 0, "times over"}, + {"a timeout may have started it", []error{errors.New("no answer from x: context deadline exceeded"), nil}, + time.Second, 1, "deadline exceeded"}, + {"a refusal in words", []error{errors.New("restic is not on this machine"), nil}, time.Second, 1, "not on this"}, + } { + asked := 0 + _, err := askUntilHeard(context.Background(), func(context.Context) (json.RawMessage, error) { + asked++ + return json.RawMessage(`{}`), row.answers[asked-1] + }, row.within, func(int) time.Duration { return 10 * time.Millisecond }) + if row.asked == 0 && (asked < 2 || asked >= len(row.answers)) || row.asked != 0 && asked != row.asked || (row.words == "") != (err == nil) || err != nil && !strings.Contains(err.Error(), row.words) { + t.Errorf("%s: asked %d times, %v", row.name, asked, err) + } + } } diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index a67f6a0..7d3ba5f 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -165,19 +165,54 @@ func backupNode(session *link.Session) func(ctx context.Context, module string) defer stop() // A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue // 1945). An answer that cannot be had or read asks for the backup as before. - if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil { + ask := func(verb string) func(context.Context) (json.RawMessage, error) { + return func(ctx context.Context) (json.RawMessage, error) { + return session.AskSeat(ctx, "node-backup", verb, map[string]any{"module": module}, 30*time.Second) + } + } + if said, err := askUntilHeard(ctx, ask("backed-up"), backupAskFor, link.Backoff); err == nil { if nothing, read := declaresNothingToBackUp(said, module); read && nothing { return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"} } } asked := time.Now() - if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil { + if _, err := askUntilHeard(ctx, ask("now"), backupAskFor, link.Backoff); err != nil { return agent.Backup{Words: err.Error()} } return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin) } } +// backupAskFor is how long the backup's ask is asked again while it is known never to have reached the holder of +// node-backup: nothing on this machine answers it (the tool runner restarting, its subscriptions not made yet) or no +// link is open. The sweep of 2026-10-30 met it on ace's redis, postgres and restic (14:03), step-ca (13:55) and +// mesh-vault on novox (15:22): each was held on one unanswered ask. Past the bound the backup failed, said with how +// often and how long it was asked: the module is held, its have says so, and the assigner asks the operator. +var backupAskFor = 3 * time.Minute + +// askUntilHeard asks, and asks again after wait(try) while the answer says the ask was never delivered (link.ErrNotAsked), +// for at most within: an ask that may have reached its holder (a timeout, a refusal in words) is never asked twice, so a +// backup is never started twice. +func askUntilHeard(ctx context.Context, ask func(context.Context) (json.RawMessage, error), within time.Duration, + wait func(int) time.Duration) (json.RawMessage, error) { + began := time.Now() + for try := 1; ; try++ { + said, err := ask(ctx) + if err == nil || !errors.Is(err, link.ErrNotAsked) { + return said, err + } + pause := wait(try) + if time.Since(began)+pause > within { + return nil, fmt.Errorf("asked %d times over %s and never heard: %w", try, time.Since(began).Round(time.Second), err) + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("the node agent was stopped while it asked again: %w", err) + case <-time.After(pause): + } + } +} + // awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it // says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node // or module, or from before the ask, is not the answer.