Merge pull request 'assigner: a backup ask only for a failed backup (A-453, A-453a)' (#237) from fix/47-G-backup-ask-only-for-failed-backup into main

This commit was merged in pull request #237.
This commit is contained in:
2026-11-16 23:51:25 +00:00
4 changed files with 80 additions and 9 deletions
@@ -1182,9 +1182,10 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
b := startBed(t, "") b := startBed(t, "")
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`) b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
fRun := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) // a held have names the build it keeps running
b.compose(t, "ace", 'b') b.compose(t, "ace", 'b')
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Self: true}) // a placed node's want names the build alone b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Self: true}) // a placed node's want names the build alone
held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}) held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fRun, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"})
b.putJSON(t, buckets.Have, "ace.web", held) b.putJSON(t, buckets.Have, "ace.web", held)
b.start(t) b.start(t)
akey, _ := asks.BackupAskKey("ace", "web", fNew) akey, _ := asks.BackupAskKey("ace", "web", fNew)
@@ -1203,8 +1204,20 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
if w1.Fp != fNew || w1.Composed != "" || !w1.Self { if w1.Fp != fNew || w1.Composed != "" || !w1.Self {
t.Errorf("the want = %+v", w1) t.Errorf("the want = %+v", w1)
} }
// the node retries and the backup fails again: a new have value, asked again under the next number // the node retries: its have is backing up, not held. The ask stands and its condition is not cleared (A-453a).
again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"}) backing, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits})
b.putJSON(t, buckets.Have, "ace.web", backing)
time.Sleep(time.Second)
if !b.hasKey(buckets.Asks(), akey) {
t.Fatal("the backup ask was taken away while the node backs up again")
}
for k, c := range b.conditionsSaid(t) {
if strings.Contains(k, "backup") && c.State == occ.StateCleared {
t.Fatalf("the condition %s was cleared while the node backs up again", k)
}
}
// the backup fails again: a new have value, asked again under the next number
again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fRun, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, buckets.Have, "ace.web", again) b.putJSON(t, buckets.Have, "ace.web", again)
a2 := b.askNow(t, akey, 2) a2 := b.askNow(t, akey, 2)
if a2.Ask.ID == a.Ask.ID { if a2.Ask.ID == a.Ask.ID {
@@ -839,6 +839,24 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu
if w.Fp != k.FP { if w.Fp != k.FP {
return "its module's want on " + k.Node + " names " + w.Fp + " now" return "its module's want on " + k.Node + " names " + w.Fp + " now"
} }
// A backup ask is about a failed backup: a have that is no longer held, or that now carries a wait (a hold of the
// composition, not a failed backup), takes it away (design 47 G A-453). A have not read yet is not stale.
if k.Kind == asks.AskKindBackup {
if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
// Past the hold is running, removed, or (a have not held) at another fingerprint, a held have naming the build it keeps running; a have mid-move (backing up again after an answer,
// applying, checking) is not (design 47 G A-453a): the ask stays, and asks again under the next number.
if h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseRemoved || (h.Phase != nodeengineclient.PhaseHeld && h.Fp != k.FP) {
return "its module's have on " + k.Node + " has moved past the hold"
}
if h.Wait != nil {
return "its module's have on " + k.Node + " carries a wait (" + h.Wait.Reason + "), not a failed backup"
}
}
}
}
}
case asks.AskKindHold, asks.AskKindRestore: case asks.AskKindHold, asks.AskKindRestore:
m, on := trunk[k.Module] m, on := trunk[k.Module]
if !on { if !on {
@@ -1065,6 +1083,8 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen // after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
// in a new have value after the answer was written. // in a new have value after the answer was written.
// //
// Only a held have with no wait is a failed backup (design 47 G A-453): the node-engine's other holds carry a wait.
//
// A have that holds a revert (reverts, which answers its `<node>.<module>` in reverting) is left to the revert ask. // A have that holds a revert (reverts, which answers its `<node>.<module>` in reverting) is left to the revert ask.
func (p *passer) backups(ctx context.Context, reverting map[string]bool) { func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
wanted := p.w.wantsOf() wanted := p.w.wantsOf()
@@ -1077,6 +1097,9 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
if err != nil || have.Phase != nodeengineclient.PhaseHeld || reverting[node+"."+module] { if err != nil || have.Phase != nodeengineclient.PhaseHeld || reverting[node+"."+module] {
continue continue
} }
if have.Wait != nil {
continue // a hold of the composition with its wait, not a failed backup (design 47 G A-453)
}
want, ok := wanted[module][node] want, ok := wanted[module][node]
if !ok || want.Removed { if !ok || want.Removed {
continue continue
@@ -160,16 +160,25 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
for _, c := range []struct { for _, c := range []struct {
name string name string
want *mwc.Want want *mwc.Want
have *nodeengine.HaveByNodeModuleValue // A-453: the module's have on the node
stale bool stale bool
}{ }{
{"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, true}, {"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, nil, true},
{"its want is removed", &mwc.Want{Removed: true}, true}, {"its want is removed", &mwc.Want{Removed: true}, nil, true},
{"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, false}, {"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, nil, false},
{"no want yet", nil, false}, {"no want yet", nil, nil, false},
// A-453: only a held have with no wait is a failed backup; a hold that carries its wait, or a have no longer held, takes the ask away.
{"its have is held with no wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false},
{"its have is held with a wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits,
Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true},
{"its have runs", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true},
{"its have backs up again (A-453a)", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits}, false},
{"its have is held, naming the build it keeps running", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false},
{"its have applies at another fingerprint", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseApplying, Health: nodeengineclient.HealthWaits}, true},
} { } {
t.Run("ended backup ask: "+c.name, func(t *testing.T) { t.Run("ended backup ask: "+c.name, func(t *testing.T) {
bkv := &memKV{values: map[string][]byte{}} bkv := &memKV{values: map[string][]byte{}}
pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}}, now) pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Have: &memKV{values: map[string][]byte{}}}, now)
ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now) ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now)
bv, err := assign.EncodeAsk(backKey, ba) bv, err := assign.EncodeAsk(backKey, ba)
if err != nil { if err != nil {
@@ -181,6 +190,14 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
raw, _ := mwc.EncodeWant(*c.want) raw, _ := mwc.EncodeWant(*c.want)
pb.w.put(buckets.Wants(), wkey, raw) pb.w.put(buckets.Wants(), wkey, raw)
} }
if c.have != nil {
hkey, _ := nodeengineclient.HaveKey("ace", "web")
raw, err := nodeengineclient.EncodeHave(*c.have)
if err != nil {
t.Fatal(err)
}
pb.w.put(buckets.Have, hkey, raw)
}
_, bask, _ := assign.DecodeAsk(backKey, bv) _, bask, _ := assign.DecodeAsk(backKey, bv)
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired}) body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired})
if err != nil { if err != nil {
@@ -1027,7 +1044,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil}, {"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil},
{"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}}, {"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}},
{"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}}, {"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}},
{"the backup's want names the fingerprint in another composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}}, {"the backup's want names the fingerprint in another composition (its have waits for a yes: a mid-move have is not past the hold, A-453a)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil}, {"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil}, {"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil}, {"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil},
@@ -133,4 +133,22 @@ func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) { if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised) t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
} }
// A-453: only a held have with no wait is a failed backup and asked; the node-engine's other holds carry a wait.
ukey, _ := asks.BackupAskKey("ace", "web", fpTest)
for _, withWait := range []bool{true, false} {
h := nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}
if withWait {
h.Wait = &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: time.Now()}
}
raw, err := nodeengineclient.EncodeHave(h)
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Have, hkey, raw)
pass()
if got := p.w.has(buckets.Asks(), ukey); got == withWait {
t.Fatalf("a held have with a wait = %v: backup ask asked = %v", withWait, got)
}
}
} }