jschoubben
39365c62c9
On novox (2026-10-31) postgres's repeat backup was interrupted by the node-engine's restart: its held have named no build, so the node's own composition took postgres in and was refused for its secret reader, and nothing ran the backup again. Issue 1289 (R4c, its delivery held). - A held have names what runs: read from the module's have before backing-up when the agent has no running entry, as the not-composed mark does. - A have in any phase but running that names nothing is not run inside the node's own. - A backup interrupted by the agent's stop is no failure: the next start backs the module up again and applies while the want asks a backup. A real failure stays held for the operator's answer. Quick fix: a new backup ask by hand. Thorough, chosen: the fault fixed where it lives in the node agent, so every data module's interrupted backup resumes by itself. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
89 lines
4.0 KiB
Go
89 lines
4.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/agent"
|
|
"github.com/novox/mesh-host/internal/link"
|
|
)
|
|
|
|
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
|
raw, _ := json.Marshal(map[string]any{"node": node, "module": module, "ok": ok, "at": at, "error": why})
|
|
return raw
|
|
}
|
|
|
|
// A backup is done when the holder says so for this node and module, held on its words when it says it failed, and held
|
|
// when it says nothing within the bound; another node's or module's event, or one from before the ask, is not the answer
|
|
// (R6-A A5, novox/hq ADR 0429 §14).
|
|
func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
|
|
since := time.Now()
|
|
run := func(within time.Duration, events ...[]byte) (done bool, words string) {
|
|
ch := make(chan []byte, len(events))
|
|
for _, e := range events {
|
|
ch <- e
|
|
}
|
|
got := awaitBackup(context.Background(), ch, "n1", "db", since, within)
|
|
return got.Done, got.Words
|
|
}
|
|
if done, _ := run(time.Second,
|
|
backupEvent("n2", "db", true, since, ""), backupEvent("n1", "web", true, since, ""),
|
|
backupEvent("n1", "db", false, since.Add(-time.Hour), "an old night"),
|
|
backupEvent("n1", "db", true, since.Add(time.Second), "")); !done {
|
|
t.Error("the event of this node and module was not the answer")
|
|
}
|
|
if done, words := run(time.Second, backupEvent("n1", "db", false, since, "the repository is locked")); done ||
|
|
!strings.Contains(words, "repository is locked") {
|
|
t.Errorf("a failed backup: done=%v words=%q", done, words)
|
|
}
|
|
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
|
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
|
}
|
|
// The node agent stopping while it waits is no failure of the backup: said as interrupted, so its next start backs
|
|
// the module up again (postgres on novox, 2026-10-31).
|
|
stopped, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if got := awaitBackup(stopped, make(chan []byte), "n1", "db", since, time.Second); got.Done || !got.Interrupted ||
|
|
got.Words != agent.BackupInterrupted {
|
|
t.Errorf("a stopped node agent: %+v", got)
|
|
}
|
|
if got := awaitBackup(context.Background(), make(chan []byte), "n1", "db", since, 10*time.Millisecond); got.Interrupted {
|
|
t.Errorf("silence within the bound taken as interrupted: %+v", got)
|
|
}
|
|
|
|
// The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it
|
|
// while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have
|
|
// reached it is never asked twice.
|
|
notAsked := fmt.Errorf("node-backup.now could not be asked: %w: nothing on this machine answers "+
|
|
"mesh.seat.node-backup.tool.now.ace", link.ErrNotAsked)
|
|
for _, row := range []struct {
|
|
name string
|
|
answers []error
|
|
within time.Duration
|
|
asked int
|
|
words string
|
|
}{
|
|
{"heard after the tool runner is back", []error{notAsked, notAsked, nil}, time.Second, 3, ""},
|
|
// Asked again until the bound, however slow the machine: at least twice, and fewer than it would answer.
|
|
{"never heard within the bound", []error{notAsked, notAsked, notAsked, notAsked, notAsked, nil}, 25 * time.Millisecond,
|
|
0, "times over"},
|
|
{"a timeout may have started it", []error{errors.New("no answer from x: context deadline exceeded"), nil},
|
|
time.Second, 1, "deadline exceeded"},
|
|
{"a refusal in words", []error{errors.New("restic is not on this machine"), nil}, time.Second, 1, "not on this"},
|
|
} {
|
|
asked := 0
|
|
_, err := askUntilHeard(context.Background(), func(context.Context) (json.RawMessage, error) {
|
|
asked++
|
|
return json.RawMessage(`{}`), row.answers[asked-1]
|
|
}, row.within, func(int) time.Duration { return 10 * time.Millisecond })
|
|
if row.asked == 0 && (asked < 2 || asked >= len(row.answers)) || row.asked != 0 && asked != row.asked || (row.words == "") != (err == nil) || err != nil && !strings.Contains(err.Error(), row.words) {
|
|
t.Errorf("%s: asked %d times, %v", row.name, asked, err)
|
|
}
|
|
}
|
|
}
|