H-A-180: judge a restore point's hold again at the start and on placement changes

A module held because the recovery module could not be asked kept that hold
across the engine's start as if it waited for the operator's answer, while
its have carried a wait, so the assigner never raised that ask: the hold was
for good (the indexer on the control-node). The engine now judges such a hold
again at its start and at every change of the placements: with no holder of
mesh-recovery placed it moves the module without a point (H-A-177), with one
placed it asks the point once more. A point a holder answered and failed is
held as a failed backup, with no wait, so the assigner raises the operator's
ask (back up again or skip); an older hold of that kind written with a wait
is written again without one at the start.

Builds: design 47 H-A-180, H-A-177, ADR 0568 §4, G-14 (a snapshot-kept that ends failed holds as a failed backup), A-453
This commit is contained in:
2026-11-20 19:31:31 +01:00
parent 1b4ce3ebf6
commit 891df501a7
7 changed files with 196 additions and 27 deletions
+7 -1
View File
@@ -702,7 +702,13 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
composer := selfComposer(session, node, hooks, say) composer := selfComposer(session, node, hooks, say)
composer.AlsoEntryAgain = shadow.EntryAgain composer.AlsoEntryAgain = shadow.EntryAgain
composer.AlsoEntriesAskAgain = shadow.EntriesAskAgain composer.AlsoEntriesAskAgain = shadow.EntriesAskAgain
composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634) composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634)
// A change of the placements judges again a module held because the recovery module could not be asked (design 47 H, H-A-180).
composer.OnPlacements = func() {
if now := own.get(); now != nil {
now.PlacementsChanged()
}
}
machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774)
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs
+45 -2
View File
@@ -290,6 +290,9 @@ type Agent struct {
gen uint64 gen uint64
again chan againAsk again chan againAsk
transient chan struct{} transient chan struct{}
// placements says the assigner's placements changed: a module held because the recovery module could not be asked is
// judged again (design 47 H, H-A-180).
placements chan struct{}
stopped chan struct{} stopped chan struct{}
// quiet is asked by waitIdle: the loop closes the channel it is given once nothing waits to be taken in or applied. // quiet is asked by waitIdle: the loop closes the channel it is given once nothing waits to be taken in or applied.
quiet chan chan struct{} quiet chan chan struct{}
@@ -325,6 +328,9 @@ type attempt struct {
WantRev, CompRev uint64 WantRev, CompRev uint64
Held bool Held bool
AnsweredRev uint64 AnsweredRev uint64
// Unasked marks a hold whose restore point could not be asked of the recovery module: judged again when the placements
// change, never waiting for the operator's answer (design 47 H, H-A-180).
Unasked bool
// Transient marks a failure of the bus, retried by the clock; Gen names the retry that is due for it. // Transient marks a failure of the bus, retried by the clock; Gen names the retry that is due for it.
Transient bool Transient bool
Gen uint64 Gen uint64
@@ -339,7 +345,7 @@ func New(env Env) *Agent {
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, waited: map[string]string{}, notComposed: map[string]notComposedMark{}, attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, waited: map[string]string{}, notComposed: map[string]notComposedMark{},
pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{}, pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{},
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64), retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
transient: make(chan struct{}, 1), stopped: make(chan struct{}), quiet: make(chan chan struct{})} transient: make(chan struct{}, 1), placements: make(chan struct{}, 1), stopped: make(chan struct{}), quiet: make(chan chan struct{})}
} }
// Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition // Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition
@@ -395,7 +401,7 @@ func (a *Agent) Run(ctx context.Context) error {
a.mu.Unlock() a.mu.Unlock()
case done := <-a.quiet: case done := <-a.quiet:
// Idle only when every change already sent is taken in and applied; asked again otherwise, after the rest. // Idle only when every change already sent is taken in and applied; asked again otherwise, after the rest.
if len(dirty) == 0 && len(a.retry)+len(a.reconcile)+len(a.failedAgain)+len(a.again)+len(a.transient)+len(wants)+len(comps) == 0 && if len(dirty) == 0 && len(a.retry)+len(a.reconcile)+len(a.failedAgain)+len(a.again)+len(a.transient)+len(a.placements)+len(wants)+len(comps) == 0 &&
a.armed.Load() == 0 { a.armed.Load() == 0 {
close(done) close(done)
} else { } else {
@@ -414,6 +420,17 @@ func (a *Agent) Run(ctx context.Context) error {
dirty[ask.module] = true dirty[ask.module] = true
} }
a.mu.Unlock() a.mu.Unlock()
case <-a.placements:
// The placements changed: each module held because the recovery module could not be asked is judged again, its point
// asked once more where a holder is placed, or moved without one where none is (design 47 H, H-A-180, H-A-177).
a.mu.Lock()
for m, att := range a.attempt {
if att.Held && att.Unasked {
delete(a.attempt, m)
dirty[m] = true
}
}
a.mu.Unlock()
case <-a.transient: case <-a.transient:
a.mu.Lock() a.mu.Lock()
for m, att := range a.attempt { for m, att := range a.attempt {
@@ -505,6 +522,15 @@ func (a *Agent) Reconcile(modules ...string) {
} }
} }
// PlacementsChanged says the assigner's placements changed, so each module held because the recovery module could not be asked
// is judged again (design 47 H, H-A-180). It never blocks.
func (a *Agent) PlacementsChanged() {
select {
case a.placements <- struct{}{}:
default:
}
}
// RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it // RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it
// when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never // when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never
// blocks. // blocks.
@@ -1587,6 +1613,23 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
a.mu.Unlock() a.mu.Unlock()
return false return false
} }
switch cause := recoveryCause(h); {
case h.Phase == string(nodeengineclient.PhaseHeld) && cause == recoveryUnasked:
// Held because the recovery module could not be asked for the restore point: no failure and nothing for the
// operator to answer. Judged again at this start: moved without a point where no holder of mesh-recovery is
// placed, the point asked once more where one is or the placements are not read yet (design 47 H, H-A-180,
// H-A-177, H-A-178).
a.mu.Lock()
a.sayOnce("unasked:"+module, fmt.Sprintf("%s was held because the recovery module could not be asked (%s): judged again at this start", module, h.Detail))
a.mu.Unlock()
return false
case h.Phase == string(nodeengineclient.PhaseHeld) && cause == recoveryFailed && h.Wait != nil:
// Held because a holder answered and the restore point failed, written with a wait, which no ask is raised for:
// written again as a failed backup, with no wait, so the operator's ask it was owed is raised (H-A-180, ADR 0568
// §4, design 47 G A-453).
h.Wait = nil
a.writeHave(ctx, h)
}
if h.Phase == string(nodeengineclient.PhaseHeld) && !judgedAgain(h) { if h.Phase == string(nodeengineclient.PhaseHeld) && !judgedAgain(h) {
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is // Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). Said, so // backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). Said, so
+49 -11
View File
@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
@@ -797,25 +798,37 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
phase string phase string
namesNothing bool // the have names no fp and no composition namesNothing bool // the have names no fp and no composition
reason string // the reason of the wait the have carries; empty: it carries none reason string // the reason of the wait the have carries; empty: it carries none
// recovery is the placements of mesh-recovery a recovery module stand-in answers ("none" placed, or "placed"), with a
// kept data whose form rises; empty: no recovery module (design 47 H, H-A-180).
recovery string
// asks is how often snapshot-kept is asked; askOwed says the hold is written again with no wait, so the assigner raises
// the operator's ask (H-A-180, ADR 0568 §4, A-453).
asks int
askOwed bool
}{ }{
{BackupInterrupted, meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, ""}, {BackupInterrupted, meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false},
// novox's postgres: its held have, written before mesh-host #189, names nothing it runs. // novox's postgres: its held have, written before mesh-host #189, names nothing it runs.
{BackupInterrupted, "", 1, 1, nodeengineclient.PhaseRunning, true, ""}, {BackupInterrupted, "", 1, 1, nodeengineclient.PhaseRunning, true, "", "", 0, false},
// A backup that failed, and the want carries the operator's answer, written before this start (nats on novox, // A backup that failed, and the want carries the operator's answer, written before this start (nats on novox,
// 2026-11-04, issue 1585): again backs it up and applies, skip applies without one; no answer keeps the hold. // 2026-11-04, issue 1585): again backs it up and applies, skip applies without one; no answer keeps the hold.
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, ""}, {"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false},
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupSkip, 0, 1, nodeengineclient.PhaseRunning, false, ""}, {"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupSkip, 0, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false},
{"restic: the repository is locked", "", 0, 0, nodeengineclient.PhaseHeld, false, ""}, {"restic: the repository is locked", "", 0, 0, nodeengineclient.PhaseHeld, false, "", "", 0, false},
// A hold the node-engine judges again (novox's indexer, 2026-11-16): its entry was not // A hold the node-engine judges again (novox's indexer, 2026-11-16): its entry was not
// held, which no answer of the operator's releases; at a start it is judged again, here backed up and applied. // held, which no answer of the operator's releases; at a start it is judged again, here backed up and applied.
{EntryMissing + " the catalog half holds no entry of the build", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing"}, {EntryMissing + " the catalog half holds no entry of the build", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing", "", 0, false},
// g14's feh, 2026-11-18: the kept step's hold opens "whether ..." and says its reason in its wait; the words decide nothing. // g14's feh, 2026-11-18: the kept step's hold opens "whether ..." and says its reason in its wait; the words decide nothing.
{"whether feh's kept data changes form is not known", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing"}, {"whether feh's kept data changes form is not known", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing", "", 0, false},
{EntryAsked + " the entry is asked", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-asked"}, {EntryAsked + " the entry is asked", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-asked", "", 0, false},
// Opening words alone are no reason: a failed backup's detail that opens with them waits for the answer. // Opening words alone are no reason: a failed backup's detail that opens with them waits for the answer.
{"parts-differ: as the words of a failed backup", "", 0, 0, nodeengineclient.PhaseHeld, false, ""}, {"parts-differ: as the words of a failed backup", "", 0, 0, nodeengineclient.PhaseHeld, false, "", "", 0, false},
// A restore point not taken waits for the answer, its reason being its words. // A restore point a holder answered and failed waits for the answer, and is written again with no wait so the operator's ask it
{"the restore point of db's data was not taken", "", 0, 0, nodeengineclient.PhaseHeld, false, "the restore point of db's data was not taken"}, // was owed is raised (H-A-180).
{"the restore point of db's data was not taken: disk full", "", 0, 0, nodeengineclient.PhaseHeld, false, "the restore point of db's data was not taken: disk full", "", 0, true},
// novox's indexer, 2026-11-20 19:20: held because the recovery module could not be asked. Judged again at the start: with no
// holder of mesh-recovery placed it moves without a point (H-A-177); with one placed the point is asked once (H-A-180).
{unaskedWords, "", 1, 1, nodeengineclient.PhaseRunning, false, unaskedWords, "none", 0, false},
{unaskedWords, "", 1, 1, nodeengineclient.PhaseRunning, false, unaskedWords, "placed", 1, false},
} { } {
r := newBed(t) r := newBed(t)
r.comps.before(ck1, craw1) r.comps.before(ck1, craw1)
@@ -830,6 +843,14 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
held.FP, held.Composed = "", "" held.FP, held.Composed = "", ""
} }
r.haves.stored = map[string]Have{"db": held} r.haves.stored = map[string]Have{"db": held}
var f *recoveryFake
if row.recovery != "" {
f = &recoveryFake{changes: []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1},
Applied: &catalogue.KeptLayout{Form: 2}}}, noHolder: row.recovery == "none",
answers: []fakeAnswer{{accepted: "a1", then: []Ending{{Act: "a1", Outcome: meshrecovery.EndedEventOutcomeDone}}}}}
f.events, f.expired = make(chan Ending, 16), make(chan struct{})
r.env.Recovery = f.env()
}
stopR := r.run() stopR := r.run()
if row.applied > 0 { if row.applied > 0 {
eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied }) eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied })
@@ -842,6 +863,20 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) { if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) {
t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h) t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h)
} }
if row.askOwed && h.Wait != nil {
t.Errorf("%s: the hold still carries a wait, so no ask is raised: %+v", row.detail, h)
}
if f != nil {
if got := strings.Count(strings.Join(f.seen(), " "), "snapshot-kept"); got != row.asks {
t.Errorf("%s with mesh-recovery %s: snapshot-kept asked %d times, want %d (%v)", row.detail, row.recovery, got, row.asks, f.seen())
}
f.mu.Lock()
raised := len(f.missing) > 0 && strings.HasPrefix(f.missing[0], "raise db: ")
f.mu.Unlock()
if raised != (row.recovery == "none") {
t.Errorf("%s with mesh-recovery %s: restore-point-missing %v", row.detail, row.recovery, f.missing)
}
}
stopR() stopR()
} }
@@ -918,6 +953,9 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
} }
} }
// unaskedWords is the hold a recovery module that could not be asked left (novox's indexer, 2026-11-20).
const unaskedWords = "the restore point of db's cache (form 1) was not taken: the recovery module could not be asked: not asked: nothing on this machine answers mesh.seat.mesh-recovery.call.snapshot-kept.node~n1 (nats: no responders available for request)"
// A data module with a good backup is applied after it; one that never ran has nothing to back up. // A data module with a good backup is applied after it; one that never ran has nothing to back up.
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) { func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange) t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange)
+51 -10
View File
@@ -114,6 +114,39 @@ func keptMoves(changes []catalogue.FormChange, runningFp, appliedFp string) (tak
type recoveryHeld struct { type recoveryHeld struct {
Words string Words string
Interrupted bool Interrupted bool
// Answered says the recovery module answered and the act failed (an error answer, a failed or cancelled ending): a failed
// restore point, which waits for the operator's answer (design 47 H, H-A-180, ADR 0568 §4).
Answered bool
}
// The words a hold of a restore point opens with, and the words of a recovery module that could not be asked, by which the
// cause of a held have is judged again (H-A-180).
const (
pointNotTaken = "the restore point of "
recoveryNotAsked = "the recovery module could not be asked: "
)
// The causes of a held have, as recoveryCause judges them.
const (
recoveryNone = iota
recoveryUnasked
recoveryFailed
)
// recoveryCause says why a held have holds, where a restore point was not taken: the recovery module could not be asked
// (recoveryUnasked), or a holder answered and the point failed (recoveryFailed); recoveryNone for any other hold (design 47 H,
// H-A-180).
func recoveryCause(h Have) int {
if !strings.HasPrefix(h.Detail, pointNotTaken) {
return recoveryNone
}
if strings.Contains(h.Detail, recoveryNotAsked) {
return recoveryUnasked
}
if h.Detail == BackupInterrupted || strings.HasSuffix(h.Detail, BackupInterrupted) {
return recoveryNone
}
return recoveryFailed
} }
// askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the // askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the
@@ -168,16 +201,16 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove,
if ctx.Err() != nil { if ctx.Err() != nil {
return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true} return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
} }
return Ending{}, &recoveryHeld{Words: "the recovery module could not be asked: " + err.Error()} return Ending{}, &recoveryHeld{Words: recoveryNotAsked + err.Error()}
case words != "": case words != "":
return Ending{}, &recoveryHeld{Words: words} return Ending{}, &recoveryHeld{Words: words, Answered: true}
case accepted != "": case accepted != "":
act, wait.Act = accepted, accepted act, wait.Act = accepted, accepted
write() write()
case busy != "": case busy != "":
busyOn = busy busyOn = busy
default: default:
return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy"} return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy", Answered: true}
} }
} }
select { select {
@@ -205,13 +238,13 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove,
if ev.Why != "" { if ev.Why != "" {
words += ": " + ev.Why words += ": " + ev.Why
} }
return ev, &recoveryHeld{Words: words} return ev, &recoveryHeld{Words: words, Answered: true}
default: default:
words := ev.Error words := ev.Error
if words == "" { if words == "" {
words = "the recovery module's act failed" words = "the recovery module's act failed"
} }
return ev, &recoveryHeld{Words: words} return ev, &recoveryHeld{Words: words, Answered: true}
} }
} }
} }
@@ -262,7 +295,10 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert)) a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert))
return false, nil return false, nil
} }
hold := func(words string, interrupted bool) { // holdAs holds the move. A restore point a holder answered and failed (answered) is held as a failed backup, with no wait, so
// the operator's ask (back up again, or go on without) is raised for it; every other hold carries its wait, and one whose
// recovery module could not be asked is judged again when the placements change (design 47 H, H-A-180, ADR 0568 §4, G A-453).
holdAs := func(words string, interrupted, answered bool) {
if interrupted || ctx.Err() != nil { if interrupted || ctx.Err() != nil {
words = BackupInterrupted words = BackupInterrupted
a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words)) a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words))
@@ -270,12 +306,17 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words)) a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words))
} }
a.mu.Lock() a.mu.Lock()
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev} unasked := !answered && strings.HasPrefix(words, pointNotTaken) && strings.Contains(words, recoveryNotAsked)
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev, Unasked: unasked}
a.mu.Unlock() a.mu.Unlock()
since := r.now() since := r.now()
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words, h := Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words}
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}}) if !answered || words == BackupInterrupted {
h.Wait = &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}
}
a.writeHave(ctx, h)
} }
hold := func(words string, interrupted bool) { holdAs(words, interrupted, false) }
// Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken: // Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken:
// the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a // the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a
// holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456). // holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456).
@@ -303,7 +344,7 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
return got.Accepted, got.Busy, got.Error, err return got.Accepted, got.Busy, got.Error, err
}) })
if held != nil { if held != nil {
hold(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted) holdAs(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted, held.Answered)
return true, nil return true, nil
} }
} }
+33 -3
View File
@@ -34,6 +34,7 @@ type recoveryFake struct {
// noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177). // noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177).
noHolder, unread bool noHolder, unread bool
missing []string // the finding restore-point-missing raised ("raise: <words>") or cleared ("clear") missing []string // the finding restore-point-missing raised ("raise: <words>") or cleared ("clear")
askErr error // snapshot-kept is not answered: the recovery module could not be asked
} }
type fakeAnswer struct { type fakeAnswer struct {
@@ -63,6 +64,15 @@ func (f *recoveryFake) env() *Recovery {
return f.changes, f.unknown return f.changes, f.unknown
}, },
SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) { SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) {
f.mu.Lock()
askErr := f.askErr
f.mu.Unlock()
if askErr != nil {
f.mu.Lock()
f.calls = append(f.calls, "snapshot-kept")
f.mu.Unlock()
return meshrecovery.SnapshotKeptAnswer{}, askErr
}
a, b, e := f.next("snapshot-kept", in) a, b, e := f.next("snapshot-kept", in)
return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
}, },
@@ -89,7 +99,7 @@ func (f *recoveryFake) env() *Recovery {
f.catchUp = append(f.catchUp, c) f.catchUp = append(f.catchUp, c)
return nil return nil
}, },
Placed: func() (bool, bool) { return !f.noHolder, !f.unread }, Placed: func() (bool, bool) { f.mu.Lock(); defer f.mu.Unlock(); return !f.noHolder, !f.unread },
PointMissing: func(_ context.Context, module, words string, missing bool) { PointMissing: func(_ context.Context, module, words string, missing bool) {
f.mu.Lock() f.mu.Lock()
defer f.mu.Unlock() defer f.mu.Unlock()
@@ -222,6 +232,24 @@ func testAMoveAcrossAFormChange(t *testing.T) {
}) })
} }
// H-A-180: a hold because the recovery module could not be asked keeps its wait and no operator's answer releases it; when the
// placements change it is judged again, here with no holder placed any more, and moves without its point (H-A-177).
t.Run("a hold whose recovery module could not be asked is judged again when the placements change", func(t *testing.T) {
f := &recoveryFake{changes: up, askErr: fmt.Errorf("nats: no responders available for request")}
b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); !contains(h.Detail, "could not be asked") || h.Wait == nil {
t.Fatalf("the have %+v", h)
}
f.mu.Lock()
f.noHolder, f.askErr = true, nil
f.mu.Unlock()
b.ag.PlacementsChanged()
eventually(t, "the module was not judged again on the placements", func() bool { return applied(b) })
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) {
t.Fatalf("calls %v", got)
}
})
t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) { t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{ f := &recoveryFake{changes: up, answers: []fakeAnswer{
{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}}, {busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
@@ -240,10 +268,12 @@ func testAMoveAcrossAFormChange(t *testing.T) {
{"restore point damaged", "restore point damaged"}, {"restore point damaged", "restore point damaged"},
{"disk full", "disk full"}, {"disk full", "disk full"},
} { } {
t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) { // A point a holder answered and failed holds as a failed backup does, with no wait, so the assigner raises the operator's ask
// (design 47 G-14, H-A-180, ADR 0568 §4, A-453).
t.Run("an ending failed with "+row.error+" holds the move as a failed backup and asks nothing again", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}} f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}}
b := moveDB(t, f, "a", "b", heldHave) b := moveDB(t, f, "a", "b", heldHave)
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" { if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait != nil {
t.Fatalf("the have %+v", h) t.Fatalf("the have %+v", h)
} }
if got := f.seen(); len(got) != 1 { if got := f.seen(); len(got) != 1 {
+6
View File
@@ -172,6 +172,9 @@ type Composer struct {
// OnEngine is told the node-engine's own module each time a composing finds it (design 47 G §527, §634); set before the // OnEngine is told the node-engine's own module each time a composing finds it (design 47 G §527, §634); set before the
// composer runs. // composer runs.
OnEngine func(module string) OnEngine func(module string)
// OnPlacements is told each change of a node's assignment the composer takes in, so the agent judges again a module held
// because the recovery module could not be asked (design 47 H, H-A-180); set before Watch, nil tells nothing.
OnPlacements func()
// AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too. // AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too.
AlsoEntryAgain func(ctx context.Context, fp string) AlsoEntryAgain func(ctx context.Context, fp string)
// AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its // AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its
@@ -904,6 +907,9 @@ func (c *Composer) takeAssignments(u agent.Update) {
if node != "" && err != nil && c.env.Say != nil { if node != "" && err != nil && c.env.Say != nil {
c.env.Say("the seat's assignment of " + node + ": " + err.Error()) c.env.Say("the seat's assignment of " + node + ": " + err.Error())
} }
if node != "" && c.OnPlacements != nil {
c.OnPlacements()
}
} }
func (c *Composer) takeRecord(u agent.Update) { func (c *Composer) takeRecord(u agent.Update) {
+5
View File
@@ -261,6 +261,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
var composed atomic.Int64 // passes made: each reads what runs once var composed atomic.Int64 // passes made: each reads what runs once
env.Running = func() []compositions.Composed { composed.Add(1); return nil } env.Running = func() []compositions.Composed { composed.Add(1); return nil }
c := New(env) c := New(env)
var placements atomic.Int64 // H-A-180: each assignment taken in is told, so the agent judges a recovery hold again
c.OnPlacements = func() { placements.Add(1) }
out, err := c.Watch(ctx, "n1.>") out, err := c.Watch(ctx, "n1.>")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -301,6 +303,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
if placed, read := c.RecoveryPlaced(); !read || placed { if placed, read := c.RecoveryPlaced(); !read || placed {
t.Errorf("once the assignments are caught up RecoveryPlaced answers placed %v read %v, want read with no holder", placed, read) t.Errorf("once the assignments are caught up RecoveryPlaced answers placed %v read %v, want read with no holder", placed, read)
} }
if placements.Load() == 0 {
t.Error("the assignment taken in was not told to OnPlacements (H-A-180)")
}
break break
} }
c, err := compositions.Decode(u.Value) c, err := compositions.Decode(u.Value)