diff --git a/cmd/mesh-host/backup_event_test.go b/cmd/mesh-host/backup_event_test.go index 5871dc1..51f1bf5 100644 --- a/cmd/mesh-host/backup_event_test.go +++ b/cmd/mesh-host/backup_event_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/link" ) @@ -43,6 +44,17 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) { if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") { t.Errorf("silence within the bound: done=%v words=%q", done, words) } + // The node agent stopping while it waits is no failure of the backup: said as interrupted, so its next start backs + // the module up again (postgres on novox, 2026-10-31). + stopped, cancel := context.WithCancel(context.Background()) + cancel() + if got := awaitBackup(stopped, make(chan []byte), "n1", "db", since, time.Second); got.Done || !got.Interrupted || + got.Words != agent.BackupInterrupted { + t.Errorf("a stopped node agent: %+v", got) + } + if got := awaitBackup(context.Background(), make(chan []byte), "n1", "db", since, 10*time.Millisecond); got.Interrupted { + t.Errorf("silence within the bound taken as interrupted: %+v", got) + } // The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it // while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index d725df7..6b845cf 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -222,7 +222,7 @@ func awaitBackup(ctx context.Context, events <-chan []byte, node, module string, for { select { case <-ctx.Done(): - return agent.Backup{Words: "the node agent was stopped while the backup ran"} + return agent.Backup{Words: agent.BackupInterrupted, Interrupted: true} case <-bound.C: return agent.Backup{Words: fmt.Sprintf("the backup did not say it was done within %s", within)} case raw := <-events: diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 7b59348..5d50426 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -84,9 +84,15 @@ type Result struct { } // Backup is the answer of the backup asked before a module moves. +// BackupInterrupted is the words of a backup the node agent itself stopped waiting for (its context ended): no failure, so +// the next start backs the module up again rather than holding it for the operator. +const BackupInterrupted = "the node agent was stopped while the backup ran" + type Backup struct { Done bool Words string + // Interrupted says the node agent stopped waiting for it (its context ended): not a failure of the backup. + Interrupted bool } // Env is everything the agent needs of the node. @@ -1325,6 +1331,15 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.prior[module] = true a.mu.Unlock() + if h.Phase == string(delivery.PhaseHeld) && h.Detail == BackupInterrupted && w.Want.Backup != delivery.BackupSkip { + // Held only because this node agent stopped while the backup ran (postgres on novox, 2026-10-31): no + // failure and nothing for the operator to answer. It is backed up again and applied (a.prior makes the + // apply back it up first); the want still asking a backup is all the answer needed. + a.mu.Lock() + a.sayOnce("interrupted:"+module, fmt.Sprintf("%s's backup was interrupted by this node agent's stop: it is backed up again", module)) + a.mu.Unlock() + return false + } if h.Phase == string(delivery.PhaseHeld) { // Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is // backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). @@ -1414,16 +1429,33 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo // A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on // without one (ADR 0539 §16). if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != delivery.BackupSkip { + // What runs, read before the have says backing-up: a module known only from its have (a.prior) has no running + // entry, and its held have kept naming nothing (postgres on novox, 2026-10-31), as the not-composed mark keeps it. + a.mu.Lock() + prev, runs := a.running[module] + a.mu.Unlock() + if !runs { + if h, found, _ := a.readHave(ctx, module); found { + prev = delivery.Have{FP: h.FP, Composed: h.Composed} + } + } a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)}) if got := a.env.Backup(ctx, module); !got.Done { - a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, got.Words)) + words := got.Words + if got.Interrupted || ctx.Err() != nil { + // The node agent stopped while it waited: no failure of the backup, so the next start backs it up again + // (baseline) rather than holding it for the operator. + words = BackupInterrupted + a.env.Say(fmt.Sprintf("%s waits: %s; it is backed up again at the next start", module, words)) + } else { + a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, words)) + } a.mu.Lock() a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev} - prev := a.running[module] a.mu.Unlock() // It keeps running what it has: the have says what it holds and why it waits. a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld), - Detail: got.Words}) + Detail: words}) return } } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index e2bb085..6ed291d 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -590,6 +590,59 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { h := b.haves.last("db") return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2 }) + + // The node agent stopped while the backup ran, the module known only from its have: the held have names what runs + // and says it was interrupted (postgres on novox, 2026-10-31, held naming nothing). + ib := newBed(t) + ib.comps.before(ck1, craw1) + ib.comps.before(ck2, craw2) + iwk, iwraw := wantFor(t, "db", fp2, dig2, delivery.BackupAgain) + ib.wants.before(iwk, iwraw) + ib.haves.stored = map[string]Have{"db": {Module: "db", FP: fp1, Composed: dig1, Phase: string(delivery.PhaseRunning)}} + began := make(chan struct{}) + ib.env.Backup = func(ctx context.Context, _ string) Backup { + close(began) + <-ctx.Done() + return Backup{Words: "the backup's ask was cancelled"} + } + stopI := ib.run() + <-began + stopI() + if h := ib.haves.last("db"); h.Phase != string(delivery.PhaseHeld) || h.FP != fp1 || h.Composed != dig1 || h.Detail != BackupInterrupted { + t.Errorf("held after an interrupted backup: %+v", h) + } + + // At a start: a backup the node agent's own stop interrupted (postgres on novox, 2026-10-31) is no failure, so it is + // backed up again and applied while the want asks a backup; a backup that failed stays held for the operator. + for _, row := range []struct { + detail, backup string + backups, applied int + phase delivery.Phase + }{ + {BackupInterrupted, delivery.BackupAgain, 1, 1, delivery.PhaseRunning}, + {BackupInterrupted, "", 1, 1, delivery.PhaseRunning}, + {"restic: the repository is locked", delivery.BackupAgain, 0, 0, delivery.PhaseHeld}, + } { + r := newBed(t) + r.comps.before(ck1, craw1) + r.comps.before(ck2, craw2) + wk, wraw := wantFor(t, "db", fp2, dig2, row.backup) + r.wants.before(wk, wraw) + r.haves.stored = map[string]Have{"db": {Module: "db", FP: fp1, Composed: dig1, Phase: string(delivery.PhaseHeld), Detail: row.detail}} + stopR := r.run() + if row.applied > 0 { + eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied }) + } + time.Sleep(50 * time.Millisecond) + r.mu.Lock() + n := len(r.backups) + r.mu.Unlock() + h := r.haves.last("db") + if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) { + t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h) + } + stopR() + } } // A data module with a good backup is applied after it; one that never ran has nothing to back up. @@ -1640,6 +1693,11 @@ func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone( if got := b.appliedList(); len(got) != 0 { t.Fatalf("applied at the start: %v", got) } + // Held, it keeps naming what runs, which this agent knows only from the have (postgres on novox, 2026-10-31): not + // the build it was to move to, and never nothing. + if h := b.haves.last("db"); h.FP != fpOf("0") || h.Composed != "sha256:"+strings.Repeat("0", 64) { + t.Fatalf("the held have does not name what runs: %+v", h) + } b.ag.Reconcile("db", "ghost") time.Sleep(50 * time.Millisecond)