H-A-180: judge a restore point's hold again at the start and on placement changes
A module held because the recovery module could not be asked kept that hold across the engine's start as if it waited for the operator's answer, while its have carried a wait, so the assigner never raised that ask: the hold was for good (the indexer on the control-node). The engine now judges such a hold again at its start and at every change of the placements: with no holder of mesh-recovery placed it moves the module without a point (H-A-177), with one placed it asks the point once more. A point a holder answered and failed is held as a failed backup, with no wait, so the assigner raises the operator's ask (back up again or skip); an older hold of that kind written with a wait is written again without one at the start. Builds: design 47 H-A-180, H-A-177, ADR 0568 §4, G-14 (a snapshot-kept that ends failed holds as a failed backup), A-453
This commit is contained in:
@@ -702,7 +702,13 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
|||||||
composer := selfComposer(session, node, hooks, say)
|
composer := selfComposer(session, node, hooks, say)
|
||||||
composer.AlsoEntryAgain = shadow.EntryAgain
|
composer.AlsoEntryAgain = shadow.EntryAgain
|
||||||
composer.AlsoEntriesAskAgain = shadow.EntriesAskAgain
|
composer.AlsoEntriesAskAgain = shadow.EntriesAskAgain
|
||||||
composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634)
|
composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634)
|
||||||
|
// A change of the placements judges again a module held because the recovery module could not be asked (design 47 H, H-A-180).
|
||||||
|
composer.OnPlacements = func() {
|
||||||
|
if now := own.get(); now != nil {
|
||||||
|
now.PlacementsChanged()
|
||||||
|
}
|
||||||
|
}
|
||||||
machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774)
|
machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774)
|
||||||
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
|
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
|
||||||
session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs
|
session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs
|
||||||
|
|||||||
+45
-2
@@ -290,6 +290,9 @@ type Agent struct {
|
|||||||
gen uint64
|
gen uint64
|
||||||
again chan againAsk
|
again chan againAsk
|
||||||
transient chan struct{}
|
transient chan struct{}
|
||||||
|
// placements says the assigner's placements changed: a module held because the recovery module could not be asked is
|
||||||
|
// judged again (design 47 H, H-A-180).
|
||||||
|
placements chan struct{}
|
||||||
stopped chan struct{}
|
stopped chan struct{}
|
||||||
// quiet is asked by waitIdle: the loop closes the channel it is given once nothing waits to be taken in or applied.
|
// quiet is asked by waitIdle: the loop closes the channel it is given once nothing waits to be taken in or applied.
|
||||||
quiet chan chan struct{}
|
quiet chan chan struct{}
|
||||||
@@ -325,6 +328,9 @@ type attempt struct {
|
|||||||
WantRev, CompRev uint64
|
WantRev, CompRev uint64
|
||||||
Held bool
|
Held bool
|
||||||
AnsweredRev uint64
|
AnsweredRev uint64
|
||||||
|
// Unasked marks a hold whose restore point could not be asked of the recovery module: judged again when the placements
|
||||||
|
// change, never waiting for the operator's answer (design 47 H, H-A-180).
|
||||||
|
Unasked bool
|
||||||
// Transient marks a failure of the bus, retried by the clock; Gen names the retry that is due for it.
|
// Transient marks a failure of the bus, retried by the clock; Gen names the retry that is due for it.
|
||||||
Transient bool
|
Transient bool
|
||||||
Gen uint64
|
Gen uint64
|
||||||
@@ -339,7 +345,7 @@ func New(env Env) *Agent {
|
|||||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, waited: map[string]string{}, notComposed: map[string]notComposedMark{},
|
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, waited: map[string]string{}, notComposed: map[string]notComposedMark{},
|
||||||
pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{},
|
pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{},
|
||||||
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
|
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||||
transient: make(chan struct{}, 1), stopped: make(chan struct{}), quiet: make(chan chan struct{})}
|
transient: make(chan struct{}, 1), placements: make(chan struct{}, 1), stopped: make(chan struct{}), quiet: make(chan chan struct{})}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition
|
// Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition
|
||||||
@@ -395,7 +401,7 @@ func (a *Agent) Run(ctx context.Context) error {
|
|||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
case done := <-a.quiet:
|
case done := <-a.quiet:
|
||||||
// Idle only when every change already sent is taken in and applied; asked again otherwise, after the rest.
|
// Idle only when every change already sent is taken in and applied; asked again otherwise, after the rest.
|
||||||
if len(dirty) == 0 && len(a.retry)+len(a.reconcile)+len(a.failedAgain)+len(a.again)+len(a.transient)+len(wants)+len(comps) == 0 &&
|
if len(dirty) == 0 && len(a.retry)+len(a.reconcile)+len(a.failedAgain)+len(a.again)+len(a.transient)+len(a.placements)+len(wants)+len(comps) == 0 &&
|
||||||
a.armed.Load() == 0 {
|
a.armed.Load() == 0 {
|
||||||
close(done)
|
close(done)
|
||||||
} else {
|
} else {
|
||||||
@@ -414,6 +420,17 @@ func (a *Agent) Run(ctx context.Context) error {
|
|||||||
dirty[ask.module] = true
|
dirty[ask.module] = true
|
||||||
}
|
}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
case <-a.placements:
|
||||||
|
// The placements changed: each module held because the recovery module could not be asked is judged again, its point
|
||||||
|
// asked once more where a holder is placed, or moved without one where none is (design 47 H, H-A-180, H-A-177).
|
||||||
|
a.mu.Lock()
|
||||||
|
for m, att := range a.attempt {
|
||||||
|
if att.Held && att.Unasked {
|
||||||
|
delete(a.attempt, m)
|
||||||
|
dirty[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
case <-a.transient:
|
case <-a.transient:
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
for m, att := range a.attempt {
|
for m, att := range a.attempt {
|
||||||
@@ -505,6 +522,15 @@ func (a *Agent) Reconcile(modules ...string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PlacementsChanged says the assigner's placements changed, so each module held because the recovery module could not be asked
|
||||||
|
// is judged again (design 47 H, H-A-180). It never blocks.
|
||||||
|
func (a *Agent) PlacementsChanged() {
|
||||||
|
select {
|
||||||
|
case a.placements <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it
|
// RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it
|
||||||
// when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never
|
// when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never
|
||||||
// blocks.
|
// blocks.
|
||||||
@@ -1587,6 +1613,23 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
|||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
switch cause := recoveryCause(h); {
|
||||||
|
case h.Phase == string(nodeengineclient.PhaseHeld) && cause == recoveryUnasked:
|
||||||
|
// Held because the recovery module could not be asked for the restore point: no failure and nothing for the
|
||||||
|
// operator to answer. Judged again at this start: moved without a point where no holder of mesh-recovery is
|
||||||
|
// placed, the point asked once more where one is or the placements are not read yet (design 47 H, H-A-180,
|
||||||
|
// H-A-177, H-A-178).
|
||||||
|
a.mu.Lock()
|
||||||
|
a.sayOnce("unasked:"+module, fmt.Sprintf("%s was held because the recovery module could not be asked (%s): judged again at this start", module, h.Detail))
|
||||||
|
a.mu.Unlock()
|
||||||
|
return false
|
||||||
|
case h.Phase == string(nodeengineclient.PhaseHeld) && cause == recoveryFailed && h.Wait != nil:
|
||||||
|
// Held because a holder answered and the restore point failed, written with a wait, which no ask is raised for:
|
||||||
|
// written again as a failed backup, with no wait, so the operator's ask it was owed is raised (H-A-180, ADR 0568
|
||||||
|
// §4, design 47 G A-453).
|
||||||
|
h.Wait = nil
|
||||||
|
a.writeHave(ctx, h)
|
||||||
|
}
|
||||||
if h.Phase == string(nodeengineclient.PhaseHeld) && !judgedAgain(h) {
|
if h.Phase == string(nodeengineclient.PhaseHeld) && !judgedAgain(h) {
|
||||||
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
|
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
|
||||||
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). Said, so
|
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). Said, so
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||||
|
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||||
@@ -797,25 +798,37 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
|||||||
phase string
|
phase string
|
||||||
namesNothing bool // the have names no fp and no composition
|
namesNothing bool // the have names no fp and no composition
|
||||||
reason string // the reason of the wait the have carries; empty: it carries none
|
reason string // the reason of the wait the have carries; empty: it carries none
|
||||||
|
// recovery is the placements of mesh-recovery a recovery module stand-in answers ("none" placed, or "placed"), with a
|
||||||
|
// kept data whose form rises; empty: no recovery module (design 47 H, H-A-180).
|
||||||
|
recovery string
|
||||||
|
// asks is how often snapshot-kept is asked; askOwed says the hold is written again with no wait, so the assigner raises
|
||||||
|
// the operator's ask (H-A-180, ADR 0568 §4, A-453).
|
||||||
|
asks int
|
||||||
|
askOwed bool
|
||||||
}{
|
}{
|
||||||
{BackupInterrupted, meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, ""},
|
{BackupInterrupted, meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false},
|
||||||
// novox's postgres: its held have, written before mesh-host #189, names nothing it runs.
|
// novox's postgres: its held have, written before mesh-host #189, names nothing it runs.
|
||||||
{BackupInterrupted, "", 1, 1, nodeengineclient.PhaseRunning, true, ""},
|
{BackupInterrupted, "", 1, 1, nodeengineclient.PhaseRunning, true, "", "", 0, false},
|
||||||
// A backup that failed, and the want carries the operator's answer, written before this start (nats on novox,
|
// A backup that failed, and the want carries the operator's answer, written before this start (nats on novox,
|
||||||
// 2026-11-04, issue 1585): again backs it up and applies, skip applies without one; no answer keeps the hold.
|
// 2026-11-04, issue 1585): again backs it up and applies, skip applies without one; no answer keeps the hold.
|
||||||
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, ""},
|
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false},
|
||||||
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupSkip, 0, 1, nodeengineclient.PhaseRunning, false, ""},
|
{"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupSkip, 0, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false},
|
||||||
{"restic: the repository is locked", "", 0, 0, nodeengineclient.PhaseHeld, false, ""},
|
{"restic: the repository is locked", "", 0, 0, nodeengineclient.PhaseHeld, false, "", "", 0, false},
|
||||||
// A hold the node-engine judges again (novox's indexer, 2026-11-16): its entry was not
|
// A hold the node-engine judges again (novox's indexer, 2026-11-16): its entry was not
|
||||||
// held, which no answer of the operator's releases; at a start it is judged again, here backed up and applied.
|
// held, which no answer of the operator's releases; at a start it is judged again, here backed up and applied.
|
||||||
{EntryMissing + " the catalog half holds no entry of the build", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing"},
|
{EntryMissing + " the catalog half holds no entry of the build", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing", "", 0, false},
|
||||||
// g14's feh, 2026-11-18: the kept step's hold opens "whether ..." and says its reason in its wait; the words decide nothing.
|
// g14's feh, 2026-11-18: the kept step's hold opens "whether ..." and says its reason in its wait; the words decide nothing.
|
||||||
{"whether feh's kept data changes form is not known", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing"},
|
{"whether feh's kept data changes form is not known", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing", "", 0, false},
|
||||||
{EntryAsked + " the entry is asked", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-asked"},
|
{EntryAsked + " the entry is asked", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-asked", "", 0, false},
|
||||||
// Opening words alone are no reason: a failed backup's detail that opens with them waits for the answer.
|
// Opening words alone are no reason: a failed backup's detail that opens with them waits for the answer.
|
||||||
{"parts-differ: as the words of a failed backup", "", 0, 0, nodeengineclient.PhaseHeld, false, ""},
|
{"parts-differ: as the words of a failed backup", "", 0, 0, nodeengineclient.PhaseHeld, false, "", "", 0, false},
|
||||||
// A restore point not taken waits for the answer, its reason being its words.
|
// A restore point a holder answered and failed waits for the answer, and is written again with no wait so the operator's ask it
|
||||||
{"the restore point of db's data was not taken", "", 0, 0, nodeengineclient.PhaseHeld, false, "the restore point of db's data was not taken"},
|
// was owed is raised (H-A-180).
|
||||||
|
{"the restore point of db's data was not taken: disk full", "", 0, 0, nodeengineclient.PhaseHeld, false, "the restore point of db's data was not taken: disk full", "", 0, true},
|
||||||
|
// novox's indexer, 2026-11-20 19:20: held because the recovery module could not be asked. Judged again at the start: with no
|
||||||
|
// holder of mesh-recovery placed it moves without a point (H-A-177); with one placed the point is asked once (H-A-180).
|
||||||
|
{unaskedWords, "", 1, 1, nodeengineclient.PhaseRunning, false, unaskedWords, "none", 0, false},
|
||||||
|
{unaskedWords, "", 1, 1, nodeengineclient.PhaseRunning, false, unaskedWords, "placed", 1, false},
|
||||||
} {
|
} {
|
||||||
r := newBed(t)
|
r := newBed(t)
|
||||||
r.comps.before(ck1, craw1)
|
r.comps.before(ck1, craw1)
|
||||||
@@ -830,6 +843,14 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
|||||||
held.FP, held.Composed = "", ""
|
held.FP, held.Composed = "", ""
|
||||||
}
|
}
|
||||||
r.haves.stored = map[string]Have{"db": held}
|
r.haves.stored = map[string]Have{"db": held}
|
||||||
|
var f *recoveryFake
|
||||||
|
if row.recovery != "" {
|
||||||
|
f = &recoveryFake{changes: []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1},
|
||||||
|
Applied: &catalogue.KeptLayout{Form: 2}}}, noHolder: row.recovery == "none",
|
||||||
|
answers: []fakeAnswer{{accepted: "a1", then: []Ending{{Act: "a1", Outcome: meshrecovery.EndedEventOutcomeDone}}}}}
|
||||||
|
f.events, f.expired = make(chan Ending, 16), make(chan struct{})
|
||||||
|
r.env.Recovery = f.env()
|
||||||
|
}
|
||||||
stopR := r.run()
|
stopR := r.run()
|
||||||
if row.applied > 0 {
|
if row.applied > 0 {
|
||||||
eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied })
|
eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied })
|
||||||
@@ -842,6 +863,20 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
|||||||
if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) {
|
if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) {
|
||||||
t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h)
|
t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h)
|
||||||
}
|
}
|
||||||
|
if row.askOwed && h.Wait != nil {
|
||||||
|
t.Errorf("%s: the hold still carries a wait, so no ask is raised: %+v", row.detail, h)
|
||||||
|
}
|
||||||
|
if f != nil {
|
||||||
|
if got := strings.Count(strings.Join(f.seen(), " "), "snapshot-kept"); got != row.asks {
|
||||||
|
t.Errorf("%s with mesh-recovery %s: snapshot-kept asked %d times, want %d (%v)", row.detail, row.recovery, got, row.asks, f.seen())
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
raised := len(f.missing) > 0 && strings.HasPrefix(f.missing[0], "raise db: ")
|
||||||
|
f.mu.Unlock()
|
||||||
|
if raised != (row.recovery == "none") {
|
||||||
|
t.Errorf("%s with mesh-recovery %s: restore-point-missing %v", row.detail, row.recovery, f.missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
stopR()
|
stopR()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -918,6 +953,9 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unaskedWords is the hold a recovery module that could not be asked left (novox's indexer, 2026-11-20).
|
||||||
|
const unaskedWords = "the restore point of db's cache (form 1) was not taken: the recovery module could not be asked: not asked: nothing on this machine answers mesh.seat.mesh-recovery.call.snapshot-kept.node~n1 (nats: no responders available for request)"
|
||||||
|
|
||||||
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
|
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
|
||||||
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
|
func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) {
|
||||||
t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange)
|
t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange)
|
||||||
|
|||||||
+51
-10
@@ -114,6 +114,39 @@ func keptMoves(changes []catalogue.FormChange, runningFp, appliedFp string) (tak
|
|||||||
type recoveryHeld struct {
|
type recoveryHeld struct {
|
||||||
Words string
|
Words string
|
||||||
Interrupted bool
|
Interrupted bool
|
||||||
|
// Answered says the recovery module answered and the act failed (an error answer, a failed or cancelled ending): a failed
|
||||||
|
// restore point, which waits for the operator's answer (design 47 H, H-A-180, ADR 0568 §4).
|
||||||
|
Answered bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// The words a hold of a restore point opens with, and the words of a recovery module that could not be asked, by which the
|
||||||
|
// cause of a held have is judged again (H-A-180).
|
||||||
|
const (
|
||||||
|
pointNotTaken = "the restore point of "
|
||||||
|
recoveryNotAsked = "the recovery module could not be asked: "
|
||||||
|
)
|
||||||
|
|
||||||
|
// The causes of a held have, as recoveryCause judges them.
|
||||||
|
const (
|
||||||
|
recoveryNone = iota
|
||||||
|
recoveryUnasked
|
||||||
|
recoveryFailed
|
||||||
|
)
|
||||||
|
|
||||||
|
// recoveryCause says why a held have holds, where a restore point was not taken: the recovery module could not be asked
|
||||||
|
// (recoveryUnasked), or a holder answered and the point failed (recoveryFailed); recoveryNone for any other hold (design 47 H,
|
||||||
|
// H-A-180).
|
||||||
|
func recoveryCause(h Have) int {
|
||||||
|
if !strings.HasPrefix(h.Detail, pointNotTaken) {
|
||||||
|
return recoveryNone
|
||||||
|
}
|
||||||
|
if strings.Contains(h.Detail, recoveryNotAsked) {
|
||||||
|
return recoveryUnasked
|
||||||
|
}
|
||||||
|
if h.Detail == BackupInterrupted || strings.HasSuffix(h.Detail, BackupInterrupted) {
|
||||||
|
return recoveryNone
|
||||||
|
}
|
||||||
|
return recoveryFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the
|
// askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the
|
||||||
@@ -168,16 +201,16 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove,
|
|||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
|
return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true}
|
||||||
}
|
}
|
||||||
return Ending{}, &recoveryHeld{Words: "the recovery module could not be asked: " + err.Error()}
|
return Ending{}, &recoveryHeld{Words: recoveryNotAsked + err.Error()}
|
||||||
case words != "":
|
case words != "":
|
||||||
return Ending{}, &recoveryHeld{Words: words}
|
return Ending{}, &recoveryHeld{Words: words, Answered: true}
|
||||||
case accepted != "":
|
case accepted != "":
|
||||||
act, wait.Act = accepted, accepted
|
act, wait.Act = accepted, accepted
|
||||||
write()
|
write()
|
||||||
case busy != "":
|
case busy != "":
|
||||||
busyOn = busy
|
busyOn = busy
|
||||||
default:
|
default:
|
||||||
return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy"}
|
return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy", Answered: true}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
@@ -205,13 +238,13 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove,
|
|||||||
if ev.Why != "" {
|
if ev.Why != "" {
|
||||||
words += ": " + ev.Why
|
words += ": " + ev.Why
|
||||||
}
|
}
|
||||||
return ev, &recoveryHeld{Words: words}
|
return ev, &recoveryHeld{Words: words, Answered: true}
|
||||||
default:
|
default:
|
||||||
words := ev.Error
|
words := ev.Error
|
||||||
if words == "" {
|
if words == "" {
|
||||||
words = "the recovery module's act failed"
|
words = "the recovery module's act failed"
|
||||||
}
|
}
|
||||||
return ev, &recoveryHeld{Words: words}
|
return ev, &recoveryHeld{Words: words, Answered: true}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -262,7 +295,10 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
|
|||||||
a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert))
|
a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert))
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
hold := func(words string, interrupted bool) {
|
// holdAs holds the move. A restore point a holder answered and failed (answered) is held as a failed backup, with no wait, so
|
||||||
|
// the operator's ask (back up again, or go on without) is raised for it; every other hold carries its wait, and one whose
|
||||||
|
// recovery module could not be asked is judged again when the placements change (design 47 H, H-A-180, ADR 0568 §4, G A-453).
|
||||||
|
holdAs := func(words string, interrupted, answered bool) {
|
||||||
if interrupted || ctx.Err() != nil {
|
if interrupted || ctx.Err() != nil {
|
||||||
words = BackupInterrupted
|
words = BackupInterrupted
|
||||||
a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words))
|
a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words))
|
||||||
@@ -270,12 +306,17 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
|
|||||||
a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words))
|
a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words))
|
||||||
}
|
}
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
|
unasked := !answered && strings.HasPrefix(words, pointNotTaken) && strings.Contains(words, recoveryNotAsked)
|
||||||
|
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev, Unasked: unasked}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
since := r.now()
|
since := r.now()
|
||||||
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
|
h := Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words}
|
||||||
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}})
|
if !answered || words == BackupInterrupted {
|
||||||
|
h.Wait = &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}
|
||||||
|
}
|
||||||
|
a.writeHave(ctx, h)
|
||||||
}
|
}
|
||||||
|
hold := func(words string, interrupted bool) { holdAs(words, interrupted, false) }
|
||||||
// Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken:
|
// Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken:
|
||||||
// the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a
|
// the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a
|
||||||
// holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456).
|
// holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456).
|
||||||
@@ -303,7 +344,7 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
|
|||||||
return got.Accepted, got.Busy, got.Error, err
|
return got.Accepted, got.Busy, got.Error, err
|
||||||
})
|
})
|
||||||
if held != nil {
|
if held != nil {
|
||||||
hold(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted)
|
holdAs(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted, held.Answered)
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ type recoveryFake struct {
|
|||||||
// noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177).
|
// noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177).
|
||||||
noHolder, unread bool
|
noHolder, unread bool
|
||||||
missing []string // the finding restore-point-missing raised ("raise: <words>") or cleared ("clear")
|
missing []string // the finding restore-point-missing raised ("raise: <words>") or cleared ("clear")
|
||||||
|
askErr error // snapshot-kept is not answered: the recovery module could not be asked
|
||||||
}
|
}
|
||||||
|
|
||||||
type fakeAnswer struct {
|
type fakeAnswer struct {
|
||||||
@@ -63,6 +64,15 @@ func (f *recoveryFake) env() *Recovery {
|
|||||||
return f.changes, f.unknown
|
return f.changes, f.unknown
|
||||||
},
|
},
|
||||||
SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) {
|
SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
askErr := f.askErr
|
||||||
|
f.mu.Unlock()
|
||||||
|
if askErr != nil {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.calls = append(f.calls, "snapshot-kept")
|
||||||
|
f.mu.Unlock()
|
||||||
|
return meshrecovery.SnapshotKeptAnswer{}, askErr
|
||||||
|
}
|
||||||
a, b, e := f.next("snapshot-kept", in)
|
a, b, e := f.next("snapshot-kept", in)
|
||||||
return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
|
return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil
|
||||||
},
|
},
|
||||||
@@ -89,7 +99,7 @@ func (f *recoveryFake) env() *Recovery {
|
|||||||
f.catchUp = append(f.catchUp, c)
|
f.catchUp = append(f.catchUp, c)
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
Placed: func() (bool, bool) { return !f.noHolder, !f.unread },
|
Placed: func() (bool, bool) { f.mu.Lock(); defer f.mu.Unlock(); return !f.noHolder, !f.unread },
|
||||||
PointMissing: func(_ context.Context, module, words string, missing bool) {
|
PointMissing: func(_ context.Context, module, words string, missing bool) {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
defer f.mu.Unlock()
|
defer f.mu.Unlock()
|
||||||
@@ -222,6 +232,24 @@ func testAMoveAcrossAFormChange(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// H-A-180: a hold because the recovery module could not be asked keeps its wait and no operator's answer releases it; when the
|
||||||
|
// placements change it is judged again, here with no holder placed any more, and moves without its point (H-A-177).
|
||||||
|
t.Run("a hold whose recovery module could not be asked is judged again when the placements change", func(t *testing.T) {
|
||||||
|
f := &recoveryFake{changes: up, askErr: fmt.Errorf("nats: no responders available for request")}
|
||||||
|
b := moveDB(t, f, "a", "b", heldHave)
|
||||||
|
if h := b.haves.last("db"); !contains(h.Detail, "could not be asked") || h.Wait == nil {
|
||||||
|
t.Fatalf("the have %+v", h)
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.noHolder, f.askErr = true, nil
|
||||||
|
f.mu.Unlock()
|
||||||
|
b.ag.PlacementsChanged()
|
||||||
|
eventually(t, "the module was not judged again on the placements", func() bool { return applied(b) })
|
||||||
|
if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) {
|
||||||
|
t.Fatalf("calls %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
|
t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
|
||||||
f := &recoveryFake{changes: up, answers: []fakeAnswer{
|
f := &recoveryFake{changes: up, answers: []fakeAnswer{
|
||||||
{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
|
{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
|
||||||
@@ -240,10 +268,12 @@ func testAMoveAcrossAFormChange(t *testing.T) {
|
|||||||
{"restore point damaged", "restore point damaged"},
|
{"restore point damaged", "restore point damaged"},
|
||||||
{"disk full", "disk full"},
|
{"disk full", "disk full"},
|
||||||
} {
|
} {
|
||||||
t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) {
|
// A point a holder answered and failed holds as a failed backup does, with no wait, so the assigner raises the operator's ask
|
||||||
|
// (design 47 G-14, H-A-180, ADR 0568 §4, A-453).
|
||||||
|
t.Run("an ending failed with "+row.error+" holds the move as a failed backup and asks nothing again", func(t *testing.T) {
|
||||||
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}}
|
f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}}
|
||||||
b := moveDB(t, f, "a", "b", heldHave)
|
b := moveDB(t, f, "a", "b", heldHave)
|
||||||
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" {
|
if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait != nil {
|
||||||
t.Fatalf("the have %+v", h)
|
t.Fatalf("the have %+v", h)
|
||||||
}
|
}
|
||||||
if got := f.seen(); len(got) != 1 {
|
if got := f.seen(); len(got) != 1 {
|
||||||
|
|||||||
@@ -172,6 +172,9 @@ type Composer struct {
|
|||||||
// OnEngine is told the node-engine's own module each time a composing finds it (design 47 G §527, §634); set before the
|
// OnEngine is told the node-engine's own module each time a composing finds it (design 47 G §527, §634); set before the
|
||||||
// composer runs.
|
// composer runs.
|
||||||
OnEngine func(module string)
|
OnEngine func(module string)
|
||||||
|
// OnPlacements is told each change of a node's assignment the composer takes in, so the agent judges again a module held
|
||||||
|
// because the recovery module could not be asked (design 47 H, H-A-180); set before Watch, nil tells nothing.
|
||||||
|
OnPlacements func()
|
||||||
// AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too.
|
// AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too.
|
||||||
AlsoEntryAgain func(ctx context.Context, fp string)
|
AlsoEntryAgain func(ctx context.Context, fp string)
|
||||||
// AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its
|
// AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its
|
||||||
@@ -904,6 +907,9 @@ func (c *Composer) takeAssignments(u agent.Update) {
|
|||||||
if node != "" && err != nil && c.env.Say != nil {
|
if node != "" && err != nil && c.env.Say != nil {
|
||||||
c.env.Say("the seat's assignment of " + node + ": " + err.Error())
|
c.env.Say("the seat's assignment of " + node + ": " + err.Error())
|
||||||
}
|
}
|
||||||
|
if node != "" && c.OnPlacements != nil {
|
||||||
|
c.OnPlacements()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Composer) takeRecord(u agent.Update) {
|
func (c *Composer) takeRecord(u agent.Update) {
|
||||||
|
|||||||
@@ -261,6 +261,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
var composed atomic.Int64 // passes made: each reads what runs once
|
var composed atomic.Int64 // passes made: each reads what runs once
|
||||||
env.Running = func() []compositions.Composed { composed.Add(1); return nil }
|
env.Running = func() []compositions.Composed { composed.Add(1); return nil }
|
||||||
c := New(env)
|
c := New(env)
|
||||||
|
var placements atomic.Int64 // H-A-180: each assignment taken in is told, so the agent judges a recovery hold again
|
||||||
|
c.OnPlacements = func() { placements.Add(1) }
|
||||||
out, err := c.Watch(ctx, "n1.>")
|
out, err := c.Watch(ctx, "n1.>")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -301,6 +303,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
if placed, read := c.RecoveryPlaced(); !read || placed {
|
if placed, read := c.RecoveryPlaced(); !read || placed {
|
||||||
t.Errorf("once the assignments are caught up RecoveryPlaced answers placed %v read %v, want read with no holder", placed, read)
|
t.Errorf("once the assignments are caught up RecoveryPlaced answers placed %v read %v, want read with no holder", placed, read)
|
||||||
}
|
}
|
||||||
|
if placements.Load() == 0 {
|
||||||
|
t.Error("the assignment taken in was not told to OnPlacements (H-A-180)")
|
||||||
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
c, err := compositions.Decode(u.Value)
|
c, err := compositions.Decode(u.Value)
|
||||||
|
|||||||
Reference in New Issue
Block a user