novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Resume a backup the node agent's own stop interrupted, and keep what a held module runs

Browse Source 
On novox (2026-10-31) postgres's repeat backup was interrupted by the node-engine's
restart: its held have named no build, so the node's own composition took postgres in
and was refused for its secret reader, and nothing ran the backup again. Issue 1289
(R4c, its delivery held).

- A held have names what runs: read from the module's have before backing-up when
  the agent has no running entry, as the not-composed mark does.
- A have in any phase but running that names nothing is not run inside the node's own.
- A backup interrupted by the agent's stop is no failure: the next start backs the
  module up again and applies while the want asks a backup. A real failure stays
  held for the operator's answer.

Quick fix: a new backup ask by hand. Thorough, chosen: the fault fixed where it lives
in the node agent, so every data module's interrupted backup resumes by itself.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
jschoubben
2026-10-31 16:33:49 +01:00
parent a4713e2d28
commit 39365c62c9
6 changed files with 94 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/agent/agent.go
+35 -3
View File
@@ -84,9 +84,15 @@ type Result struct {
}
// Backup is the answer of the backup asked before a module moves.
// BackupInterrupted is the words of a backup the node agent itself stopped waiting for (its context ended): no failure, so
// the next start backs the module up again rather than holding it for the operator.
const BackupInterrupted = "the node agent was stopped while the backup ran"
type Backup struct {
Done bool
Words string
// Interrupted says the node agent stopped waiting for it (its context ended): not a failure of the backup.
Interrupted bool
}
// Env is everything the agent needs of the node.
@@ -1325,6 +1331,15 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
a.mu.Lock()
a.prior[module] = true
a.mu.Unlock()
if h.Phase == string(delivery.PhaseHeld) && h.Detail == BackupInterrupted && w.Want.Backup != delivery.BackupSkip {
// Held only because this node agent stopped while the backup ran (postgres on novox, 2026-10-31): no
// failure and nothing for the operator to answer. It is backed up again and applied (a.prior makes the
// apply back it up first); the want still asking a backup is all the answer needed.
a.mu.Lock()
a.sayOnce("interrupted:"+module, fmt.Sprintf("%s's backup was interrupted by this node agent's stop: it is backed up again", module))
a.mu.Unlock()
return false
}
if h.Phase == string(delivery.PhaseHeld) {
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4).
@@ -1414,16 +1429,33 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
// A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on
// without one (ADR 0539 §16).
if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != delivery.BackupSkip {
// What runs, read before the have says backing-up: a module known only from its have (a.prior) has no running
// entry, and its held have kept naming nothing (postgres on novox, 2026-10-31), as the not-composed mark keeps it.
a.mu.Lock()
prev, runs := a.running[module]
a.mu.Unlock()
if !runs {
if h, found, _ := a.readHave(ctx, module); found {
prev = delivery.Have{FP: h.FP, Composed: h.Composed}
}
}
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)})
if got := a.env.Backup(ctx, module); !got.Done {
a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, got.Words))
words := got.Words
if got.Interrupted || ctx.Err() != nil {
// The node agent stopped while it waited: no failure of the backup, so the next start backs it up again
// (baseline) rather than holding it for the operator.
words = BackupInterrupted
a.env.Say(fmt.Sprintf("%s waits: %s; it is backed up again at the next start", module, words))
} else {
a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, words))
}
a.mu.Lock()
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
prev := a.running[module]
a.mu.Unlock()
// It keeps running what it has: the have says what it holds and why it waits.
a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld),
Detail: got.Words})
Detail: words})
return
}
}
internal/agent/agent_test.go
+37
View File
@@ -590,6 +590,38 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
h := b.haves.last("db")
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2
})
// At a start: a backup the node agent's own stop interrupted (postgres on novox, 2026-10-31) is no failure, so it is
// backed up again and applied while the want asks a backup; a backup that failed stays held for the operator.
for _, row := range []struct {
detail, backup string
backups, applied int
phase delivery.Phase
}{
{BackupInterrupted, delivery.BackupAgain, 1, 1, delivery.PhaseRunning},
{BackupInterrupted, "", 1, 1, delivery.PhaseRunning},
{"restic: the repository is locked", delivery.BackupAgain, 0, 0, delivery.PhaseHeld},
} {
r := newBed(t)
r.comps.before(ck1, craw1)
r.comps.before(ck2, craw2)
wk, wraw := wantFor(t, "db", fp2, dig2, row.backup)
r.wants.before(wk, wraw)
r.haves.stored = map[string]Have{"db": {Module: "db", FP: fp1, Composed: dig1, Phase: string(delivery.PhaseHeld), Detail: row.detail}}
stopR := r.run()
if row.applied > 0 {
eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied })
}
time.Sleep(50 * time.Millisecond)
r.mu.Lock()
n := len(r.backups)
r.mu.Unlock()
h := r.haves.last("db")
if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) {
t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h)
}
stopR()
}
}
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
@@ -1640,6 +1672,11 @@ func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone(
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("applied at the start: %v", got)
}
// Held, it keeps naming what runs, which this agent knows only from the have (postgres on novox, 2026-10-31): not
// the build it was to move to, and never nothing.
if h := b.haves.last("db"); h.FP != fpOf("0") || h.Composed != "sha256:"+strings.Repeat("0", 64) {
t.Fatalf("the held have does not name what runs: %+v", h)
}
b.ag.Reconcile("db", "ghost")
time.Sleep(50 * time.Millisecond)
internal/selfcompose/selfcompose.go
+5 -1
View File
@@ -618,7 +618,11 @@ func (c *Composer) takeModuleHave(u agent.Update) bool {
// A have that names nothing it runs, failed (the node could not compose a module it never ran: NotComposed), is
// no module run inside the node's own composition: taken as one, it was composed from its manifest past its
// build (issue 1126, review round 1 of the builds).
runsNothing := h.Phase == delivery.PhaseFailed && h.FP == "" && h.Composed == ""
// Nor is one in any phase but running that names nothing (held by a backup the node agent's restart
// interrupted: postgres on novox, 2026-10-31, refused for its secret reader). Running with nothing named is
// a module of the node's own (mesh-wireguard).
runsNothing := h.FP == "" && h.Composed == "" &&
(h.Phase == delivery.PhaseFailed || h.Build == "" && h.Phase != "" && h.Phase != delivery.PhaseRunning)
in = h.Phase != delivery.PhaseRemoved && !runsNothing && !strings.HasPrefix(h.FP, delivery.SourcePrefix2)
}
}
internal/selfcompose/selfcompose_test.go
+4
View File
@@ -1760,6 +1760,10 @@ func TestAHaveSaysWhetherItsModuleRunsInsideTheNodesOwn(t *testing.T) {
{"running with no fingerprint (mesh-wireguard)", delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, true, false},
{"held today", delivery.Have{FP: "build-0123abcd", Composed: "sha256:" + strings.Repeat("c", 64), Phase: delivery.PhaseHeld,
Health: delivery.HealthWaits, Detail: "held: a required config value is unset"}, true, true},
// postgres on novox (2026-10-31): held by a backup the node agent's restart interrupted, its have naming nothing
// it runs; counted in, it was composed from its manifest past its build and refused for its secret reader.
{"held, naming nothing it runs", delivery.Have{Phase: delivery.PhaseHeld, Health: delivery.HealthWaits,
Detail: "the node agent was stopped while the backup ran"}, false, true},
} {
t.Run(row.name, func(t *testing.T) {
c := New(Env{Node: "n1"})