assigner: the backup question is asked only for a held have with no wait (A-453)
A held have that carries a wait is a hold of the composition, not a failed backup; a backup ask whose have now carries a wait or is no longer held is withdrawn with its condition. Builds: design 47 G A-453, ADR 0539 §16, §541
This commit is contained in:
@@ -839,6 +839,22 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu
|
||||
if w.Fp != k.FP {
|
||||
return "its module's want on " + k.Node + " names " + w.Fp + " now"
|
||||
}
|
||||
// A backup ask is about a failed backup: a have that is no longer held, or that now carries a wait (a hold of the
|
||||
// composition, not a failed backup), takes it away (design 47 G A-453). A have not read yet is not stale.
|
||||
if k.Kind == asks.AskKindBackup {
|
||||
if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil {
|
||||
if raw := p.w.value(buckets.Have, hkey); raw != nil {
|
||||
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
|
||||
if h.Phase != nodeengineclient.PhaseHeld {
|
||||
return "its module's have on " + k.Node + " is no longer held"
|
||||
}
|
||||
if h.Wait != nil {
|
||||
return "its module's have on " + k.Node + " carries a wait (" + h.Wait.Reason + "), not a failed backup"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
case asks.AskKindHold, asks.AskKindRestore:
|
||||
m, on := trunk[k.Module]
|
||||
if !on {
|
||||
@@ -1065,6 +1081,8 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
|
||||
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
|
||||
// in a new have value after the answer was written.
|
||||
//
|
||||
// Only a held have with no wait is a failed backup (design 47 G A-453): the node-engine's other holds carry a wait.
|
||||
//
|
||||
// A have that holds a revert (reverts, which answers its `<node>.<module>` in reverting) is left to the revert ask.
|
||||
func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
|
||||
wanted := p.w.wantsOf()
|
||||
@@ -1077,6 +1095,9 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
|
||||
if err != nil || have.Phase != nodeengineclient.PhaseHeld || reverting[node+"."+module] {
|
||||
continue
|
||||
}
|
||||
if have.Wait != nil {
|
||||
continue // a hold of the composition with its wait, not a failed backup (design 47 G A-453)
|
||||
}
|
||||
want, ok := wanted[module][node]
|
||||
if !ok || want.Removed {
|
||||
continue
|
||||
|
||||
@@ -160,16 +160,22 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
want *mwc.Want
|
||||
have *nodeengine.HaveByNodeModuleValue // A-453: the module's have on the node
|
||||
stale bool
|
||||
}{
|
||||
{"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, true},
|
||||
{"its want is removed", &mwc.Want{Removed: true}, true},
|
||||
{"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, false},
|
||||
{"no want yet", nil, false},
|
||||
{"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, nil, true},
|
||||
{"its want is removed", &mwc.Want{Removed: true}, nil, true},
|
||||
{"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, nil, false},
|
||||
{"no want yet", nil, nil, false},
|
||||
// A-453: only a held have with no wait is a failed backup; a hold that carries its wait, or a have no longer held, takes the ask away.
|
||||
{"its have is held with no wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false},
|
||||
{"its have is held with a wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits,
|
||||
Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true},
|
||||
{"its have is no longer held", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true},
|
||||
} {
|
||||
t.Run("ended backup ask: "+c.name, func(t *testing.T) {
|
||||
bkv := &memKV{values: map[string][]byte{}}
|
||||
pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}}, now)
|
||||
pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Have: &memKV{values: map[string][]byte{}}}, now)
|
||||
ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now)
|
||||
bv, err := assign.EncodeAsk(backKey, ba)
|
||||
if err != nil {
|
||||
@@ -181,6 +187,14 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
raw, _ := mwc.EncodeWant(*c.want)
|
||||
pb.w.put(buckets.Wants(), wkey, raw)
|
||||
}
|
||||
if c.have != nil {
|
||||
hkey, _ := nodeengineclient.HaveKey("ace", "web")
|
||||
raw, err := nodeengineclient.EncodeHave(*c.have)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pb.w.put(buckets.Have, hkey, raw)
|
||||
}
|
||||
_, bask, _ := assign.DecodeAsk(backKey, bv)
|
||||
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired})
|
||||
if err != nil {
|
||||
@@ -1027,7 +1041,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil},
|
||||
{"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}},
|
||||
{"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}},
|
||||
{"the backup's want names the fingerprint in another composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}},
|
||||
{"the backup's want names the fingerprint in another composition (its have waits for a yes, not held: A-453)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil},
|
||||
|
||||
@@ -133,4 +133,22 @@ func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
|
||||
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
|
||||
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
|
||||
}
|
||||
|
||||
// A-453: only a held have with no wait is a failed backup and asked; the node-engine's other holds carry a wait.
|
||||
ukey, _ := asks.BackupAskKey("ace", "web", fpTest)
|
||||
for _, withWait := range []bool{true, false} {
|
||||
h := nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}
|
||||
if withWait {
|
||||
h.Wait = &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: time.Now()}
|
||||
}
|
||||
raw, err := nodeengineclient.EncodeHave(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(buckets.Have, hkey, raw)
|
||||
pass()
|
||||
if got := p.w.has(buckets.Asks(), ukey); got == withWait {
|
||||
t.Fatalf("a held have with a wait = %v: backup ask asked = %v", withWait, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user