From 891df501a722117ee55c4534bf7faa48673bda1e Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 19:31:31 +0100 Subject: [PATCH] H-A-180: judge a restore point's hold again at the start and on placement changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A module held because the recovery module could not be asked kept that hold across the engine's start as if it waited for the operator's answer, while its have carried a wait, so the assigner never raised that ask: the hold was for good (the indexer on the control-node). The engine now judges such a hold again at its start and at every change of the placements: with no holder of mesh-recovery placed it moves the module without a point (H-A-177), with one placed it asks the point once more. A point a holder answered and failed is held as a failed backup, with no wait, so the assigner raises the operator's ask (back up again or skip); an older hold of that kind written with a wait is written again without one at the start. Builds: design 47 H-A-180, H-A-177, ADR 0568 §4, G-14 (a snapshot-kept that ends failed holds as a failed backup), A-453 --- cmd/mesh-host/nodeagent.go | 8 +++- internal/agent/agent.go | 47 +++++++++++++++++- internal/agent/agent_test.go | 60 ++++++++++++++++++----- internal/agent/kept.go | 61 ++++++++++++++++++++---- internal/agent/kept_test.go | 36 ++++++++++++-- internal/selfcompose/selfcompose.go | 6 +++ internal/selfcompose/selfcompose_test.go | 5 ++ 7 files changed, 196 insertions(+), 27 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index a32bda60..7dafac33 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -702,7 +702,13 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus composer := selfComposer(session, node, hooks, say) composer.AlsoEntryAgain = shadow.EntryAgain composer.AlsoEntriesAskAgain = shadow.EntriesAskAgain - composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634) + composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634) + // A change of the placements judges again a module held because the recovery module could not be asked (design 47 H, H-A-180). + composer.OnPlacements = func() { + if now := own.get(); now != nil { + now.PlacementsChanged() + } + } machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs diff --git a/internal/agent/agent.go b/internal/agent/agent.go index e2520acb..c78eedf2 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -290,6 +290,9 @@ type Agent struct { gen uint64 again chan againAsk transient chan struct{} + // placements says the assigner's placements changed: a module held because the recovery module could not be asked is + // judged again (design 47 H, H-A-180). + placements chan struct{} stopped chan struct{} // quiet is asked by waitIdle: the loop closes the channel it is given once nothing waits to be taken in or applied. quiet chan chan struct{} @@ -325,6 +328,9 @@ type attempt struct { WantRev, CompRev uint64 Held bool AnsweredRev uint64 + // Unasked marks a hold whose restore point could not be asked of the recovery module: judged again when the placements + // change, never waiting for the operator's answer (design 47 H, H-A-180). + Unasked bool // Transient marks a failure of the bus, retried by the clock; Gen names the retry that is due for it. Transient bool Gen uint64 @@ -339,7 +345,7 @@ func New(env Env) *Agent { attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, waited: map[string]string{}, notComposed: map[string]notComposedMark{}, pulls: map[string]*pullWait{}, settledAt: map[string]time.Time{}, settledSeq: map[string]int{}, retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64), - transient: make(chan struct{}, 1), stopped: make(chan struct{}), quiet: make(chan chan struct{})} + transient: make(chan struct{}, 1), placements: make(chan struct{}, 1), stopped: make(chan struct{}), quiet: make(chan chan struct{})} } // Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition @@ -395,7 +401,7 @@ func (a *Agent) Run(ctx context.Context) error { a.mu.Unlock() case done := <-a.quiet: // Idle only when every change already sent is taken in and applied; asked again otherwise, after the rest. - if len(dirty) == 0 && len(a.retry)+len(a.reconcile)+len(a.failedAgain)+len(a.again)+len(a.transient)+len(wants)+len(comps) == 0 && + if len(dirty) == 0 && len(a.retry)+len(a.reconcile)+len(a.failedAgain)+len(a.again)+len(a.transient)+len(a.placements)+len(wants)+len(comps) == 0 && a.armed.Load() == 0 { close(done) } else { @@ -414,6 +420,17 @@ func (a *Agent) Run(ctx context.Context) error { dirty[ask.module] = true } a.mu.Unlock() + case <-a.placements: + // The placements changed: each module held because the recovery module could not be asked is judged again, its point + // asked once more where a holder is placed, or moved without one where none is (design 47 H, H-A-180, H-A-177). + a.mu.Lock() + for m, att := range a.attempt { + if att.Held && att.Unasked { + delete(a.attempt, m) + dirty[m] = true + } + } + a.mu.Unlock() case <-a.transient: a.mu.Lock() for m, att := range a.attempt { @@ -505,6 +522,15 @@ func (a *Agent) Reconcile(modules ...string) { } } +// PlacementsChanged says the assigner's placements changed, so each module held because the recovery module could not be asked +// is judged again (design 47 H, H-A-180). It never blocks. +func (a *Agent) PlacementsChanged() { + select { + case a.placements <- struct{}{}: + default: + } +} + // RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it // when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never // blocks. @@ -1587,6 +1613,23 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Unlock() return false } + switch cause := recoveryCause(h); { + case h.Phase == string(nodeengineclient.PhaseHeld) && cause == recoveryUnasked: + // Held because the recovery module could not be asked for the restore point: no failure and nothing for the + // operator to answer. Judged again at this start: moved without a point where no holder of mesh-recovery is + // placed, the point asked once more where one is or the placements are not read yet (design 47 H, H-A-180, + // H-A-177, H-A-178). + a.mu.Lock() + a.sayOnce("unasked:"+module, fmt.Sprintf("%s was held because the recovery module could not be asked (%s): judged again at this start", module, h.Detail)) + a.mu.Unlock() + return false + case h.Phase == string(nodeengineclient.PhaseHeld) && cause == recoveryFailed && h.Wait != nil: + // Held because a holder answered and the restore point failed, written with a wait, which no ask is raised for: + // written again as a failed backup, with no wait, so the operator's ask it was owed is raised (H-A-180, ADR 0568 + // §4, design 47 G A-453). + h.Wait = nil + a.writeHave(ctx, h) + } if h.Phase == string(nodeengineclient.PhaseHeld) && !judgedAgain(h) { // Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is // backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4). Said, so diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index f5eec6b9..1493bdde 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" @@ -797,25 +798,37 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { phase string namesNothing bool // the have names no fp and no composition reason string // the reason of the wait the have carries; empty: it carries none + // recovery is the placements of mesh-recovery a recovery module stand-in answers ("none" placed, or "placed"), with a + // kept data whose form rises; empty: no recovery module (design 47 H, H-A-180). + recovery string + // asks is how often snapshot-kept is asked; askOwed says the hold is written again with no wait, so the assigner raises + // the operator's ask (H-A-180, ADR 0568 §4, A-453). + asks int + askOwed bool }{ - {BackupInterrupted, meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, ""}, + {BackupInterrupted, meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false}, // novox's postgres: its held have, written before mesh-host #189, names nothing it runs. - {BackupInterrupted, "", 1, 1, nodeengineclient.PhaseRunning, true, ""}, + {BackupInterrupted, "", 1, 1, nodeengineclient.PhaseRunning, true, "", "", 0, false}, // A backup that failed, and the want carries the operator's answer, written before this start (nats on novox, // 2026-11-04, issue 1585): again backs it up and applies, skip applies without one; no answer keeps the hold. - {"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, ""}, - {"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupSkip, 0, 1, nodeengineclient.PhaseRunning, false, ""}, - {"restic: the repository is locked", "", 0, 0, nodeengineclient.PhaseHeld, false, ""}, + {"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupAgain, 1, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false}, + {"restic: the repository is locked", meshwants.WantsByWantNodeModuleValueBackupSkip, 0, 1, nodeengineclient.PhaseRunning, false, "", "", 0, false}, + {"restic: the repository is locked", "", 0, 0, nodeengineclient.PhaseHeld, false, "", "", 0, false}, // A hold the node-engine judges again (novox's indexer, 2026-11-16): its entry was not // held, which no answer of the operator's releases; at a start it is judged again, here backed up and applied. - {EntryMissing + " the catalog half holds no entry of the build", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing"}, + {EntryMissing + " the catalog half holds no entry of the build", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing", "", 0, false}, // g14's feh, 2026-11-18: the kept step's hold opens "whether ..." and says its reason in its wait; the words decide nothing. - {"whether feh's kept data changes form is not known", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing"}, - {EntryAsked + " the entry is asked", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-asked"}, + {"whether feh's kept data changes form is not known", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-missing", "", 0, false}, + {EntryAsked + " the entry is asked", "", 1, 1, nodeengineclient.PhaseRunning, false, "entry-asked", "", 0, false}, // Opening words alone are no reason: a failed backup's detail that opens with them waits for the answer. - {"parts-differ: as the words of a failed backup", "", 0, 0, nodeengineclient.PhaseHeld, false, ""}, - // A restore point not taken waits for the answer, its reason being its words. - {"the restore point of db's data was not taken", "", 0, 0, nodeengineclient.PhaseHeld, false, "the restore point of db's data was not taken"}, + {"parts-differ: as the words of a failed backup", "", 0, 0, nodeengineclient.PhaseHeld, false, "", "", 0, false}, + // A restore point a holder answered and failed waits for the answer, and is written again with no wait so the operator's ask it + // was owed is raised (H-A-180). + {"the restore point of db's data was not taken: disk full", "", 0, 0, nodeengineclient.PhaseHeld, false, "the restore point of db's data was not taken: disk full", "", 0, true}, + // novox's indexer, 2026-11-20 19:20: held because the recovery module could not be asked. Judged again at the start: with no + // holder of mesh-recovery placed it moves without a point (H-A-177); with one placed the point is asked once (H-A-180). + {unaskedWords, "", 1, 1, nodeengineclient.PhaseRunning, false, unaskedWords, "none", 0, false}, + {unaskedWords, "", 1, 1, nodeengineclient.PhaseRunning, false, unaskedWords, "placed", 1, false}, } { r := newBed(t) r.comps.before(ck1, craw1) @@ -830,6 +843,14 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { held.FP, held.Composed = "", "" } r.haves.stored = map[string]Have{"db": held} + var f *recoveryFake + if row.recovery != "" { + f = &recoveryFake{changes: []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1}, + Applied: &catalogue.KeptLayout{Form: 2}}}, noHolder: row.recovery == "none", + answers: []fakeAnswer{{accepted: "a1", then: []Ending{{Act: "a1", Outcome: meshrecovery.EndedEventOutcomeDone}}}}} + f.events, f.expired = make(chan Ending, 16), make(chan struct{}) + r.env.Recovery = f.env() + } stopR := r.run() if row.applied > 0 { eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied }) @@ -842,6 +863,20 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) { t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h) } + if row.askOwed && h.Wait != nil { + t.Errorf("%s: the hold still carries a wait, so no ask is raised: %+v", row.detail, h) + } + if f != nil { + if got := strings.Count(strings.Join(f.seen(), " "), "snapshot-kept"); got != row.asks { + t.Errorf("%s with mesh-recovery %s: snapshot-kept asked %d times, want %d (%v)", row.detail, row.recovery, got, row.asks, f.seen()) + } + f.mu.Lock() + raised := len(f.missing) > 0 && strings.HasPrefix(f.missing[0], "raise db: ") + f.mu.Unlock() + if raised != (row.recovery == "none") { + t.Errorf("%s with mesh-recovery %s: restore-point-missing %v", row.detail, row.recovery, f.missing) + } + } stopR() } @@ -918,6 +953,9 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { } } +// unaskedWords is the hold a recovery module that could not be asked left (novox's indexer, 2026-11-20). +const unaskedWords = "the restore point of db's cache (form 1) was not taken: the recovery module could not be asked: not asked: nothing on this machine answers mesh.seat.mesh-recovery.call.snapshot-kept.node~n1 (nats: no responders available for request)" + // A data module with a good backup is applied after it; one that never ran has nothing to back up. func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) { t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange) diff --git a/internal/agent/kept.go b/internal/agent/kept.go index f45f315c..58ff297a 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -114,6 +114,39 @@ func keptMoves(changes []catalogue.FormChange, runningFp, appliedFp string) (tak type recoveryHeld struct { Words string Interrupted bool + // Answered says the recovery module answered and the act failed (an error answer, a failed or cancelled ending): a failed + // restore point, which waits for the operator's answer (design 47 H, H-A-180, ADR 0568 §4). + Answered bool +} + +// The words a hold of a restore point opens with, and the words of a recovery module that could not be asked, by which the +// cause of a held have is judged again (H-A-180). +const ( + pointNotTaken = "the restore point of " + recoveryNotAsked = "the recovery module could not be asked: " +) + +// The causes of a held have, as recoveryCause judges them. +const ( + recoveryNone = iota + recoveryUnasked + recoveryFailed +) + +// recoveryCause says why a held have holds, where a restore point was not taken: the recovery module could not be asked +// (recoveryUnasked), or a holder answered and the point failed (recoveryFailed); recoveryNone for any other hold (design 47 H, +// H-A-180). +func recoveryCause(h Have) int { + if !strings.HasPrefix(h.Detail, pointNotTaken) { + return recoveryNone + } + if strings.Contains(h.Detail, recoveryNotAsked) { + return recoveryUnasked + } + if h.Detail == BackupInterrupted || strings.HasSuffix(h.Detail, BackupInterrupted) { + return recoveryNone + } + return recoveryFailed } // askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the @@ -168,16 +201,16 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, if ctx.Err() != nil { return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true} } - return Ending{}, &recoveryHeld{Words: "the recovery module could not be asked: " + err.Error()} + return Ending{}, &recoveryHeld{Words: recoveryNotAsked + err.Error()} case words != "": - return Ending{}, &recoveryHeld{Words: words} + return Ending{}, &recoveryHeld{Words: words, Answered: true} case accepted != "": act, wait.Act = accepted, accepted write() case busy != "": busyOn = busy default: - return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy"} + return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy", Answered: true} } } select { @@ -205,13 +238,13 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, if ev.Why != "" { words += ": " + ev.Why } - return ev, &recoveryHeld{Words: words} + return ev, &recoveryHeld{Words: words, Answered: true} default: words := ev.Error if words == "" { words = "the recovery module's act failed" } - return ev, &recoveryHeld{Words: words} + return ev, &recoveryHeld{Words: words, Answered: true} } } } @@ -262,7 +295,10 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert)) return false, nil } - hold := func(words string, interrupted bool) { + // holdAs holds the move. A restore point a holder answered and failed (answered) is held as a failed backup, with no wait, so + // the operator's ask (back up again, or go on without) is raised for it; every other hold carries its wait, and one whose + // recovery module could not be asked is judged again when the placements change (design 47 H, H-A-180, ADR 0568 §4, G A-453). + holdAs := func(words string, interrupted, answered bool) { if interrupted || ctx.Err() != nil { words = BackupInterrupted a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words)) @@ -270,12 +306,17 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words)) } a.mu.Lock() - a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev} + unasked := !answered && strings.HasPrefix(words, pointNotTaken) && strings.Contains(words, recoveryNotAsked) + a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev, Unasked: unasked} a.mu.Unlock() since := r.now() - a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words, - Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}}) + h := Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words} + if !answered || words == BackupInterrupted { + h.Wait = &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words} + } + a.writeHave(ctx, h) } + hold := func(words string, interrupted bool) { holdAs(words, interrupted, false) } // Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken: // the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a // holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456). @@ -303,7 +344,7 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return got.Accepted, got.Busy, got.Error, err }) if held != nil { - hold(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted) + holdAs(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted, held.Answered) return true, nil } } diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 7747b252..ff59e2a7 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -34,6 +34,7 @@ type recoveryFake struct { // noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177). noHolder, unread bool missing []string // the finding restore-point-missing raised ("raise: ") or cleared ("clear") + askErr error // snapshot-kept is not answered: the recovery module could not be asked } type fakeAnswer struct { @@ -63,6 +64,15 @@ func (f *recoveryFake) env() *Recovery { return f.changes, f.unknown }, SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) { + f.mu.Lock() + askErr := f.askErr + f.mu.Unlock() + if askErr != nil { + f.mu.Lock() + f.calls = append(f.calls, "snapshot-kept") + f.mu.Unlock() + return meshrecovery.SnapshotKeptAnswer{}, askErr + } a, b, e := f.next("snapshot-kept", in) return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil }, @@ -89,7 +99,7 @@ func (f *recoveryFake) env() *Recovery { f.catchUp = append(f.catchUp, c) return nil }, - Placed: func() (bool, bool) { return !f.noHolder, !f.unread }, + Placed: func() (bool, bool) { f.mu.Lock(); defer f.mu.Unlock(); return !f.noHolder, !f.unread }, PointMissing: func(_ context.Context, module, words string, missing bool) { f.mu.Lock() defer f.mu.Unlock() @@ -222,6 +232,24 @@ func testAMoveAcrossAFormChange(t *testing.T) { }) } + // H-A-180: a hold because the recovery module could not be asked keeps its wait and no operator's answer releases it; when the + // placements change it is judged again, here with no holder placed any more, and moves without its point (H-A-177). + t.Run("a hold whose recovery module could not be asked is judged again when the placements change", func(t *testing.T) { + f := &recoveryFake{changes: up, askErr: fmt.Errorf("nats: no responders available for request")} + b := moveDB(t, f, "a", "b", heldHave) + if h := b.haves.last("db"); !contains(h.Detail, "could not be asked") || h.Wait == nil { + t.Fatalf("the have %+v", h) + } + f.mu.Lock() + f.noHolder, f.askErr = true, nil + f.mu.Unlock() + b.ag.PlacementsChanged() + eventually(t, "the module was not judged again on the placements", func() bool { return applied(b) }) + if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) { + t.Fatalf("calls %v", got) + } + }) + t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) { f := &recoveryFake{changes: up, answers: []fakeAnswer{ {busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}}, @@ -240,10 +268,12 @@ func testAMoveAcrossAFormChange(t *testing.T) { {"restore point damaged", "restore point damaged"}, {"disk full", "disk full"}, } { - t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) { + // A point a holder answered and failed holds as a failed backup does, with no wait, so the assigner raises the operator's ask + // (design 47 G-14, H-A-180, ADR 0568 §4, A-453). + t.Run("an ending failed with "+row.error+" holds the move as a failed backup and asks nothing again", func(t *testing.T) { f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}} b := moveDB(t, f, "a", "b", heldHave) - if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" { + if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait != nil { t.Fatalf("the have %+v", h) } if got := f.seen(); len(got) != 1 { diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index e67f1293..66926017 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -172,6 +172,9 @@ type Composer struct { // OnEngine is told the node-engine's own module each time a composing finds it (design 47 G §527, §634); set before the // composer runs. OnEngine func(module string) + // OnPlacements is told each change of a node's assignment the composer takes in, so the agent judges again a module held + // because the recovery module could not be asked (design 47 H, H-A-180); set before Watch, nil tells nothing. + OnPlacements func() // AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too. AlsoEntryAgain func(ctx context.Context, fp string) // AlsoEntriesAskAgain is told each catalog-revision the composer hears, so the shadow, which hears nothing itself, asks its @@ -904,6 +907,9 @@ func (c *Composer) takeAssignments(u agent.Update) { if node != "" && err != nil && c.env.Say != nil { c.env.Say("the seat's assignment of " + node + ": " + err.Error()) } + if node != "" && c.OnPlacements != nil { + c.OnPlacements() + } } func (c *Composer) takeRecord(u agent.Update) { diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 9264aa2c..32736a8f 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -261,6 +261,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { var composed atomic.Int64 // passes made: each reads what runs once env.Running = func() []compositions.Composed { composed.Add(1); return nil } c := New(env) + var placements atomic.Int64 // H-A-180: each assignment taken in is told, so the agent judges a recovery hold again + c.OnPlacements = func() { placements.Add(1) } out, err := c.Watch(ctx, "n1.>") if err != nil { t.Fatal(err) @@ -301,6 +303,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if placed, read := c.RecoveryPlaced(); !read || placed { t.Errorf("once the assignments are caught up RecoveryPlaced answers placed %v read %v, want read with no holder", placed, read) } + if placements.Load() == 0 { + t.Error("the assignment taken in was not told to OnPlacements (H-A-180)") + } break } c, err := compositions.Decode(u.Value)