diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 9956065..2903a82 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -2,6 +2,7 @@ package assign import ( "fmt" + "maps" "slices" "sort" "strings" @@ -315,6 +316,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st ok, target = true, have.FP } } + if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) { + // While a node fails F alone, F reaches no further node: a node whose want does not name F yet gets no new want + // naming it, and wants already written stay (ADR 0557 §2). + ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed))) + } switch { case ok: d, _ := m.composition(node, target) @@ -460,6 +466,12 @@ func failedElsewhere(m Module, n Nodes, node string) bool { return false } +// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone. +func FailedAloneWords(module, fp string, failing []string) string { + return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp), + strings.Join(failing, ", ")) +} + // NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's // health, phase and words (HeldWords without a time), and the healthy witnesses by name. type NodeFailure struct { diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index da9cdbe..3e6985b 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -164,10 +164,29 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { t.Errorf("node failed = %+v", out.NodeFailed) } - // ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is - // not one node's failure (D3). Either way F is held and ace is set back. + // ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants + // at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it. three := m three.Nodes = []string{"ace", "g14", "novox"} + failedHave := n.Haves["ace"]["mail"] + n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave + n.Online["novox"] = true + n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}} + before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)} + out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + if _, alone := out.NodeFailed["g14"]; !alone { + t.Errorf("g14 does not fail alone: %+v", out.NodeFailed) + } + if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld { + t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"]) + } + if _, written := out.Wants["g14"]; written { + t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"]) + } + n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"] + + // ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is + // not one node's failure (D3). Either way F is held and ace is set back. for _, c := range []struct { name string online map[string]bool