From 8ab75bb7275eac74fbe9dfd08bb9ca869a08f536 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 00:41:00 +0100 Subject: [PATCH 1/3] assigner: the backup question is asked only for a held have with no wait (A-453) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A held have that carries a wait is a hold of the composition, not a failed backup; a backup ask whose have now carries a wait or is no longer held is withdrawn with its condition. Builds: design 47 G A-453, ADR 0539 §16, §541 --- .../cmd/mesh-delivery-assigner/pass.go | 21 +++++++++++++++ .../cmd/mesh-delivery-assigner/pass_test.go | 26 ++++++++++++++----- .../cmd/mesh-delivery-assigner/revert_test.go | 18 +++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 952f0af1..a4d4c72b 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -839,6 +839,22 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu if w.Fp != k.FP { return "its module's want on " + k.Node + " names " + w.Fp + " now" } + // A backup ask is about a failed backup: a have that is no longer held, or that now carries a wait (a hold of the + // composition, not a failed backup), takes it away (design 47 G A-453). A have not read yet is not stale. + if k.Kind == asks.AskKindBackup { + if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil { + if raw := p.w.value(buckets.Have, hkey); raw != nil { + if h, err := nodeengineclient.DecodeHave(raw); err == nil { + if h.Phase != nodeengineclient.PhaseHeld { + return "its module's have on " + k.Node + " is no longer held" + } + if h.Wait != nil { + return "its module's have on " + k.Node + " carries a wait (" + h.Wait.Reason + "), not a failed backup" + } + } + } + } + } case asks.AskKindHold, asks.AskKindRestore: m, on := trunk[k.Module] if !on { @@ -1065,6 +1081,8 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R // after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen // in a new have value after the answer was written. // +// Only a held have with no wait is a failed backup (design 47 G A-453): the node-engine's other holds carry a wait. +// // A have that holds a revert (reverts, which answers its `.` in reverting) is left to the revert ask. func (p *passer) backups(ctx context.Context, reverting map[string]bool) { wanted := p.w.wantsOf() @@ -1077,6 +1095,9 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) { if err != nil || have.Phase != nodeengineclient.PhaseHeld || reverting[node+"."+module] { continue } + if have.Wait != nil { + continue // a hold of the composition with its wait, not a failed backup (design 47 G A-453) + } want, ok := wanted[module][node] if !ok || want.Removed { continue diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 86dfc7f4..c34e6716 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -160,16 +160,22 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) { for _, c := range []struct { name string want *mwc.Want + have *nodeengine.HaveByNodeModuleValue // A-453: the module's have on the node stale bool }{ - {"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, true}, - {"its want is removed", &mwc.Want{Removed: true}, true}, - {"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, false}, - {"no want yet", nil, false}, + {"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, nil, true}, + {"its want is removed", &mwc.Want{Removed: true}, nil, true}, + {"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, nil, false}, + {"no want yet", nil, nil, false}, + // A-453: only a held have with no wait is a failed backup; a hold that carries its wait, or a have no longer held, takes the ask away. + {"its have is held with no wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false}, + {"its have is held with a wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, + Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true}, + {"its have is no longer held", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true}, } { t.Run("ended backup ask: "+c.name, func(t *testing.T) { bkv := &memKV{values: map[string][]byte{}} - pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}}, now) + pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Have: &memKV{values: map[string][]byte{}}}, now) ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now) bv, err := assign.EncodeAsk(backKey, ba) if err != nil { @@ -181,6 +187,14 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) { raw, _ := mwc.EncodeWant(*c.want) pb.w.put(buckets.Wants(), wkey, raw) } + if c.have != nil { + hkey, _ := nodeengineclient.HaveKey("ace", "web") + raw, err := nodeengineclient.EncodeHave(*c.have) + if err != nil { + t.Fatal(err) + } + pb.w.put(buckets.Have, hkey, raw) + } _, bask, _ := assign.DecodeAsk(backKey, bv) body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired}) if err != nil { @@ -1027,7 +1041,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) { {"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil}, {"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}}, {"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}}, - {"the backup's want names the fingerprint in another composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}}, + {"the backup's want names the fingerprint in another composition (its have waits for a yes, not held: A-453)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}}, {"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil}, {"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil}, {"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil}, diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revert_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revert_test.go index ea662350..b29b01b9 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revert_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/revert_test.go @@ -133,4 +133,22 @@ func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) { if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) { t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised) } + + // A-453: only a held have with no wait is a failed backup and asked; the node-engine's other holds carry a wait. + ukey, _ := asks.BackupAskKey("ace", "web", fpTest) + for _, withWait := range []bool{true, false} { + h := nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"} + if withWait { + h.Wait = &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: time.Now()} + } + raw, err := nodeengineclient.EncodeHave(h) + if err != nil { + t.Fatal(err) + } + p.w.put(buckets.Have, hkey, raw) + pass() + if got := p.w.has(buckets.Asks(), ukey); got == withWait { + t.Fatalf("a held have with a wait = %v: backup ask asked = %v", withWait, got) + } + } } -- 2.54.0 From 1a187445a518bbfa8dd225517d1c07e922f663a5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 00:46:47 +0100 Subject: [PATCH 2/3] assigner: a backup ask stands while its have is mid-move; it goes only past the hold (A-453a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-453, A-453a, ADR 0539 §16 --- .../cmd/mesh-delivery-assigner/integration_test.go | 14 +++++++++++++- .../cmd/mesh-delivery-assigner/pass.go | 6 ++++-- .../cmd/mesh-delivery-assigner/pass_test.go | 6 ++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1c6e113b..1a1372a8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1203,7 +1203,19 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) { if w1.Fp != fNew || w1.Composed != "" || !w1.Self { t.Errorf("the want = %+v", w1) } - // the node retries and the backup fails again: a new have value, asked again under the next number + // the node retries: its have is backing up, not held. The ask stands and its condition is not cleared (A-453a). + backing, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits}) + b.putJSON(t, buckets.Have, "ace.web", backing) + time.Sleep(time.Second) + if !b.hasKey(buckets.Asks(), akey) { + t.Fatal("the backup ask was taken away while the node backs up again") + } + for k, c := range b.conditionsSaid(t) { + if strings.Contains(k, "backup") && c.State == occ.StateCleared { + t.Fatalf("the condition %s was cleared while the node backs up again", k) + } + } + // the backup fails again: a new have value, asked again under the next number again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"}) b.putJSON(t, buckets.Have, "ace.web", again) a2 := b.askNow(t, akey, 2) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index a4d4c72b..9b9d88af 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -845,8 +845,10 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil { if raw := p.w.value(buckets.Have, hkey); raw != nil { if h, err := nodeengineclient.DecodeHave(raw); err == nil { - if h.Phase != nodeengineclient.PhaseHeld { - return "its module's have on " + k.Node + " is no longer held" + // Past the hold is running, removed or another fingerprint; a have mid-move (backing up again after an answer, + // applying, checking) is not (design 47 G A-453a): the ask stays, and asks again under the next number. + if h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseRemoved || h.Fp != k.FP { + return "its module's have on " + k.Node + " has moved past the hold" } if h.Wait != nil { return "its module's have on " + k.Node + " carries a wait (" + h.Wait.Reason + "), not a failed backup" diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index c34e6716..4062499e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -171,7 +171,9 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) { {"its have is held with no wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false}, {"its have is held with a wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true}, - {"its have is no longer held", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true}, + {"its have runs", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true}, + {"its have backs up again (A-453a)", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits}, false}, + {"its have is at another fingerprint", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, true}, } { t.Run("ended backup ask: "+c.name, func(t *testing.T) { bkv := &memKV{values: map[string][]byte{}} @@ -1041,7 +1043,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) { {"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil}, {"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}}, {"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}}, - {"the backup's want names the fingerprint in another composition (its have waits for a yes, not held: A-453)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}}, + {"the backup's want names the fingerprint in another composition (its have waits for a yes: a mid-move have is not past the hold, A-453a)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}}, {"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil}, {"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil}, {"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil}, -- 2.54.0 From 482a618d1e378cb73a080d8724601930fc26b8e5 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 17 Nov 2026 00:49:26 +0100 Subject: [PATCH 3/3] assigner: a held have names the build it keeps running; only a have not held at another fingerprint is past the hold (A-453a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-453, A-453a, ADR 0539 §16 --- .../cmd/mesh-delivery-assigner/integration_test.go | 5 +++-- .../cmd/mesh-delivery-assigner/pass.go | 4 ++-- .../cmd/mesh-delivery-assigner/pass_test.go | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 1a1372a8..9168fddb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -1182,9 +1182,10 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) { b := startBed(t, "") fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`) + fRun := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) // a held have names the build it keeps running b.compose(t, "ace", 'b') b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Self: true}) // a placed node's want names the build alone - held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}) + held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fRun, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}) b.putJSON(t, buckets.Have, "ace.web", held) b.start(t) akey, _ := asks.BackupAskKey("ace", "web", fNew) @@ -1216,7 +1217,7 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) { } } // the backup fails again: a new have value, asked again under the next number - again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"}) + again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fRun, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"}) b.putJSON(t, buckets.Have, "ace.web", again) a2 := b.askNow(t, akey, 2) if a2.Ask.ID == a.Ask.ID { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 9b9d88af..5911577a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -845,9 +845,9 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil { if raw := p.w.value(buckets.Have, hkey); raw != nil { if h, err := nodeengineclient.DecodeHave(raw); err == nil { - // Past the hold is running, removed or another fingerprint; a have mid-move (backing up again after an answer, + // Past the hold is running, removed, or (a have not held) at another fingerprint, a held have naming the build it keeps running; a have mid-move (backing up again after an answer, // applying, checking) is not (design 47 G A-453a): the ask stays, and asks again under the next number. - if h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseRemoved || h.Fp != k.FP { + if h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseRemoved || (h.Phase != nodeengineclient.PhaseHeld && h.Fp != k.FP) { return "its module's have on " + k.Node + " has moved past the hold" } if h.Wait != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 4062499e..2e2631c1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -173,7 +173,8 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) { Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true}, {"its have runs", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true}, {"its have backs up again (A-453a)", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits}, false}, - {"its have is at another fingerprint", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, true}, + {"its have is held, naming the build it keeps running", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false}, + {"its have applies at another fingerprint", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseApplying, Health: nodeengineclient.HealthWaits}, true}, } { t.Run("ended backup ask: "+c.name, func(t *testing.T) { bkv := &memKV{values: map[string][]byte{}} -- 2.54.0