jschoubben
bb63d83f84
mesh/delivery merged: source.main moved to novox/mesh-delivery@c38e81c7f5e9
mesh/check being carried to bb63d83f849e on c38e81c7f5e9 | from 2915d74822cd
After the placement seed's revert the rules that hold a want (a hold, rules 3 and 5) kept g14's and shanks's Self wants: the held branch copied them whole, and the rule branch wrote a want naming no composition, which EncodeWant refuses. Module.Self, set once from the placement, is now the one source, and every want the rules write or keep is derived from it: an unplaced node's kept want names the controller's composition of the same fingerprint, asked as any record move; with none, it stays and rule 7 says why.
1249 lines
64 KiB
Go
1249 lines
64 KiB
Go
package assign
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
|
|
|
var (
|
|
fNew = fpOf('b')
|
|
fOld = fpOf('a')
|
|
)
|
|
|
|
// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider
|
|
// store healthy on ace and the canary ace running fNew healthy.
|
|
func ready() (Module, Nodes) {
|
|
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"},
|
|
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
|
|
n := Nodes{
|
|
Online: map[string]bool{"ace": true, "g14": true},
|
|
Haves: map[string]map[string]delivery.Have{
|
|
"ace": {"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, "store": {FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
|
"g14": {"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
|
},
|
|
Providers: map[string]map[string][]string{"database": {"store": {"ace"}}},
|
|
}
|
|
return m, n
|
|
}
|
|
|
|
// dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated.
|
|
func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) }
|
|
|
|
// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither.
|
|
func composedEverywhere(_, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, true
|
|
}
|
|
|
|
func wantedOld() map[string]delivery.Want {
|
|
return map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}, "g14": {FP: fOld, Composed: dig(fOld)}}
|
|
}
|
|
|
|
// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so
|
|
// g14 follows at once: step 0, ADR 0528 clause 3).
|
|
func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
|
|
m, n := ready()
|
|
out := Wants(m, n, wantedOld(), nil)
|
|
for _, node := range []string{"ace", "g14"} {
|
|
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
|
|
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1297/B4, one test per rule: given every rule but N holds, no want moves to F, and the rule N is named.
|
|
func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
|
for rule, breakIt := range map[int]func(*Module, *Nodes){
|
|
delivery.RuleBuildDone: func(m *Module, _ *Nodes) { m.Built = false },
|
|
delivery.RuleNotHeld: func(m *Module, _ *Nodes) { m.Held = true },
|
|
delivery.RuleCanary: func(_ *Module, n *Nodes) {
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
},
|
|
delivery.RuleProviders: func(_ *Module, n *Nodes) {
|
|
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
|
},
|
|
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
|
|
delivery.RuleComposed: func(m *Module, _ *Nodes) {
|
|
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
|
|
},
|
|
} {
|
|
m, n := ready()
|
|
breakIt(&m, &n)
|
|
out := Wants(m, n, wantedOld(), nil)
|
|
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
|
t.Errorf("rule %d broken: g14's want moved to F", rule)
|
|
}
|
|
if got := out.Rules["g14"].Rule; got != rule {
|
|
t.Errorf("rule %d broken: g14 is held back by rule %d (%s)", rule, got, out.Rules["g14"].Words)
|
|
}
|
|
if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) {
|
|
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
|
}
|
|
}
|
|
|
|
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
|
|
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
|
|
// rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it
|
|
// stays and is said; a placed node's want is Self with no composition.
|
|
selfAt := func(fp string) map[string]delivery.Want {
|
|
return map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fp, Self: true, Yes: dig(fp)}}
|
|
}
|
|
atF := func(_ *Module, n *Nodes) {
|
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseFailed, Health: delivery.HealthOK}
|
|
}
|
|
for _, c := range []struct {
|
|
name string
|
|
set func(*Module, *Nodes)
|
|
before string
|
|
want delivery.Want // the want of g14 written; zero: none written
|
|
rule int // the rule g14's holding names
|
|
asked bool
|
|
}{
|
|
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
|
atF(m, n)
|
|
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
|
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleProviders},
|
|
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
|
|
atF(m, n)
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleCanary},
|
|
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
}, want: delivery.Want{FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleCanary,
|
|
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: delivery.RuleCanary},
|
|
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
|
|
want: delivery.Want{FP: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"},
|
|
rule: delivery.RuleNotHeld},
|
|
{name: "held data module", before: fNew, set: func(m *Module, n *Nodes) {
|
|
atF(m, n)
|
|
m.Held = true
|
|
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
|
}, want: delivery.Want{FP: fNew, Composed: dig(fNew),
|
|
Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: delivery.RuleNotHeld},
|
|
{name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) {
|
|
atF(m, n)
|
|
recording(m)
|
|
m.Held = true
|
|
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld)}, true
|
|
}
|
|
}, rule: delivery.RuleNotHeld, asked: true},
|
|
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
|
|
atF(m, n)
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
if node == "g14" {
|
|
return delivery.ComposedDigest{}, false
|
|
}
|
|
return composedEverywhere(node, fp)
|
|
}
|
|
}, rule: delivery.RuleComposed},
|
|
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
|
|
atF(m, n)
|
|
m.Self = func(node string) bool { return node == "g14" }
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
if node == "g14" {
|
|
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither}, true
|
|
}
|
|
return composedEverywhere(node, fp)
|
|
}
|
|
}, want: delivery.Want{FP: fNew, Self: true}},
|
|
} {
|
|
m, n := ready()
|
|
c.set(&m, &n)
|
|
before := selfAt(c.before)
|
|
if c.name == "placed" {
|
|
before["g14"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
|
}
|
|
out := Wants(m, n, before, nil)
|
|
if got := out.Wants["g14"]; got != c.want {
|
|
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
|
|
}
|
|
if got := out.Rules["g14"].Rule; got != c.rule {
|
|
t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule)
|
|
}
|
|
if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked {
|
|
t.Errorf("%s: record asks %+v", c.name, out.Record)
|
|
}
|
|
if w, ok := out.Wants["g14"]; ok {
|
|
if _, err := delivery.EncodeWant(w); err != nil {
|
|
t.Errorf("%s: g14's want does not encode: %v", c.name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
|
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
|
m, n := ready()
|
|
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
|
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
|
out := Wants(m, n, wantedOld(), nil)
|
|
if w := out.Wants["ace"]; w.FP != fNew {
|
|
t.Errorf("ace, whose account the grant names, wants %+v", w)
|
|
}
|
|
if got := out.Rules["g14"]; got.Rule != delivery.RuleGrant || !strings.Contains(got.Words, "on g14") {
|
|
t.Errorf("g14, whose account the grant does not name, is held by %+v", got)
|
|
}
|
|
for _, c := range []struct {
|
|
accounts []string
|
|
node, module string
|
|
named bool
|
|
}{
|
|
{nil, "g14", "mail", true},
|
|
{[]string{"g14.mail.send"}, "g14", "mail", true},
|
|
{[]string{"g14.mailer"}, "g14", "mail", false},
|
|
{[]string{"ace.mail"}, "g14", "mail", false},
|
|
} {
|
|
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
|
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
|
|
func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
|
|
m, n := ready()
|
|
n.Online["ace"] = false
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
n.Providers["database"]["store"] = []string{"ace", "g14"}
|
|
n.Haves["g14"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
if c := Canary(m.Nodes, n.Online); c != "g14" {
|
|
t.Fatalf("canary = %q, want g14", c)
|
|
}
|
|
out := Wants(m, n, wantedOld(), nil)
|
|
if out.Wants["g14"].FP != fNew {
|
|
t.Errorf("the canary g14's want = %+v", out.Wants["g14"])
|
|
}
|
|
if out.Rules["ace"].Rule != delivery.RuleCanary {
|
|
t.Errorf("ace, behind the canary g14, is held back by %+v", out.Rules["ace"])
|
|
}
|
|
}
|
|
|
|
// 1297/B4: a failed health check of F holds F and sets the nodes that want it back to their last healthy fingerprint.
|
|
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
|
m, n := ready()
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
|
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
|
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
|
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
|
|
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
|
|
t.Fatalf("held = %+v", out.Held)
|
|
}
|
|
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
|
t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w)
|
|
}
|
|
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
|
t.Errorf("g14 moved to a held fingerprint: %+v", w)
|
|
}
|
|
|
|
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
|
|
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
|
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
|
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
|
|
if out.Held != nil || len(out.Wants) != 0 {
|
|
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
|
|
}
|
|
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
|
|
len(out.NodeFailed) != 1 {
|
|
t.Errorf("node failed = %+v", out.NodeFailed)
|
|
}
|
|
|
|
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
|
|
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
|
|
three := m
|
|
three.Nodes = []string{"ace", "g14", "novox"}
|
|
failedHave := n.Haves["ace"]["mail"]
|
|
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
|
|
n.Online["novox"] = true
|
|
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
|
|
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
|
|
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
|
if _, alone := out.NodeFailed["g14"]; !alone {
|
|
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
|
|
}
|
|
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld {
|
|
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
|
|
}
|
|
if _, written := out.Wants["g14"]; written {
|
|
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
|
|
}
|
|
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
|
|
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
delete(before, "novox")
|
|
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
|
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
|
|
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
|
|
}
|
|
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
|
|
|
|
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
|
|
// not one node's failure (D3). Either way F is held and ace is set back.
|
|
for _, c := range []struct {
|
|
name string
|
|
online map[string]bool
|
|
novox delivery.Health
|
|
}{
|
|
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
|
|
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
|
|
} {
|
|
n.Online = c.online
|
|
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
|
|
if c.novox == delivery.HealthOK {
|
|
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
}
|
|
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
|
|
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
|
|
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
|
|
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
|
|
}
|
|
if w := out.Wants["ace"]; w.FP != fOld {
|
|
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1297/B4: a module holding data is not set back; its want stays and says why.
|
|
func TestADataModuleIsNotSetBack(t *testing.T) {
|
|
m, n := ready()
|
|
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
|
|
if out.Held == nil {
|
|
t.Fatal("F was not held")
|
|
}
|
|
if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") {
|
|
t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w)
|
|
}
|
|
// 1906/D3: and the node is to be asked whether to restore it
|
|
if !slices.Equal(out.Restore, []string{"ace"}) {
|
|
t.Errorf("restore asks = %v, want ace", out.Restore)
|
|
}
|
|
}
|
|
|
|
// 1906/D3: a module without data is set back and asked about nothing.
|
|
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
|
|
m, n := ready()
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
|
|
if len(out.Restore) != 0 {
|
|
t.Errorf("restore asks = %v", out.Restore)
|
|
}
|
|
}
|
|
|
|
// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that
|
|
// is already as the rules give it is not written again.
|
|
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
|
|
m, n := ready()
|
|
m.Nodes = []string{"ace"}
|
|
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
|
if w := out.Wants["g14"]; !w.Removed {
|
|
t.Errorf("g14's want = %+v, want removed", w)
|
|
}
|
|
if _, ok := out.Wants["ace"]; ok {
|
|
t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"])
|
|
}
|
|
}
|
|
|
|
// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its
|
|
// digest once it exists; a node with no want before still gets none until then.
|
|
func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
|
|
m, n := ready()
|
|
have := map[string]bool{}
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
if !have[node+"|"+fp] {
|
|
return delivery.ComposedDigest{}, false
|
|
}
|
|
return composedEverywhere(node, fp)
|
|
}
|
|
out := Wants(m, n, wantedOld(), nil)
|
|
for node, w := range out.Wants {
|
|
if w.FP == fNew {
|
|
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
|
|
}
|
|
}
|
|
if r := out.Rules["ace"]; r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "ace") {
|
|
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
|
|
}
|
|
// and a node that never had a want gets none
|
|
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
|
|
t.Errorf("wants without a composition: %+v", out.Wants)
|
|
}
|
|
have["ace|"+fNew] = true
|
|
out = Wants(m, n, wantedOld(), nil)
|
|
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
|
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
|
|
}
|
|
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
|
t.Errorf("g14, whose composition does not exist, moved: %+v", w)
|
|
}
|
|
}
|
|
|
|
// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want,
|
|
// by the same rules: written when they hold, and not while one does not.
|
|
func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
|
|
m, n := ready()
|
|
revised := "sha256:" + strings.Repeat("9", 64)
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedNeither}, true
|
|
}
|
|
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
|
out := Wants(m, n, running, nil)
|
|
for _, node := range []string{"ace", "g14"} {
|
|
if w := out.Wants[node]; w.FP != fNew || w.Composed != revised {
|
|
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
|
|
}
|
|
}
|
|
m.Granted = false
|
|
out = Wants(m, n, running, nil)
|
|
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
|
|
t.Errorf("a revision was written while a rule holds: %+v", w)
|
|
}
|
|
|
|
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
|
|
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
|
|
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
|
|
out = Wants(m, n, wantedOld(), nil)
|
|
if w := out.Wants["g14"]; w.FP != fOld || w.Composed != revised || w.Newest != fNew {
|
|
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
|
|
}
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord,
|
|
Against: dig(fOld), Words: "recreates"}, true
|
|
}
|
|
out = Wants(m, n, wantedOld(), nil)
|
|
if w := out.Wants["g14"]; w.Composed == revised {
|
|
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
|
|
}
|
|
if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) {
|
|
t.Errorf("the revision was not asked: %+v", out.Record)
|
|
}
|
|
}
|
|
|
|
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
|
|
func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
|
|
m, n := ready()
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
|
|
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
if fp == fOld {
|
|
return delivery.ComposedDigest{}, false
|
|
}
|
|
return composedEverywhere(node, fp)
|
|
}
|
|
out := Wants(m, n, running, map[string]string{"ace": fOld})
|
|
if w := out.Wants["ace"]; w.FP == fOld {
|
|
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
|
|
}
|
|
m.Composed = composedEverywhere
|
|
out = Wants(m, n, running, map[string]string{"ace": fOld})
|
|
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
|
|
t.Errorf("set-back want = %+v", w)
|
|
}
|
|
}
|
|
|
|
// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F).
|
|
func movingTo(j delivery.Judgement) (Module, Nodes, map[string]delivery.Want) {
|
|
m, n := ready()
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: j}
|
|
if j != delivery.JudgedNeither {
|
|
d.Against, d.Words = dig(fOld), "judged"
|
|
}
|
|
return d, true
|
|
}
|
|
return m, n, wantedOld()
|
|
}
|
|
|
|
func recording(m *Module) {
|
|
m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"}
|
|
}
|
|
|
|
// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be
|
|
// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint.
|
|
func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) {
|
|
for _, j := range []delivery.Judgement{delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot} {
|
|
m, n, wanted := movingTo(j)
|
|
recording(&m)
|
|
out := Wants(m, n, wanted, nil)
|
|
if w, ok := out.Wants["ace"]; ok {
|
|
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
|
|
}
|
|
var ace *RecordNeed
|
|
for i := range out.Record {
|
|
if out.Record[i].Node == "ace" {
|
|
ace = &out.Record[i]
|
|
}
|
|
}
|
|
if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j {
|
|
t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record)
|
|
}
|
|
// a new revision at the same fingerprint is asked too
|
|
same := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}}
|
|
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
|
|
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
|
|
}
|
|
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
|
|
out = Wants(m, n, wanted, nil)
|
|
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
|
t.Errorf("%s: after the yes ace's want = %+v", j, w)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module
|
|
// that rolls (not record) is never asked.
|
|
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
|
|
m, n, wanted := movingTo(delivery.JudgedNeither)
|
|
recording(&m)
|
|
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
|
t.Errorf("neither: %+v", out)
|
|
}
|
|
m, n, _ = movingTo(delivery.JudgedRecreates)
|
|
recording(&m)
|
|
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
|
|
delete(n.Haves["ace"], "mail")
|
|
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
|
t.Errorf("a node that never ran it: %+v", out)
|
|
}
|
|
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
|
|
m, n, _ = movingTo(delivery.JudgedCannot)
|
|
recording(&m)
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew || len(out.Record) == 0 {
|
|
t.Errorf("a node that runs it with no want: %+v", out)
|
|
}
|
|
m, n, wanted = movingTo(delivery.JudgedRecreates)
|
|
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
|
t.Errorf("a rolled module: %+v", out)
|
|
}
|
|
}
|
|
|
|
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
|
|
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
|
|
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
|
m, n, _ := movingTo(delivery.JudgedNeither)
|
|
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
|
|
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
|
|
}
|
|
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
|
|
out := Wants(m, n, running, nil)
|
|
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
|
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
|
|
}
|
|
if len(out.Record) != 0 {
|
|
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
|
}
|
|
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
|
|
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
|
|
put := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
|
for _, c := range []struct {
|
|
name string
|
|
step string
|
|
wanted map[string]delivery.Want
|
|
want *delivery.Want // nil: nothing written on ace
|
|
}{
|
|
{"waiting for the operator", delivery.PutPerson, wantedOld(), nil},
|
|
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
|
|
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
|
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
|
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
|
|
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
|
} {
|
|
p := put
|
|
p.Step = c.step
|
|
m.Puts = map[string]delivery.Put{"ace": p}
|
|
if c.step == "" {
|
|
m.Puts = nil
|
|
}
|
|
out := Wants(m, n, c.wanted, nil)
|
|
got, wrote := out.Wants["ace"]
|
|
switch {
|
|
case c.want == nil && wrote:
|
|
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
|
|
case c.want != nil && (!wrote || !delivery.SameWant(normal(got), *c.want)):
|
|
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
|
|
}
|
|
}
|
|
userList := "sha256:" + strings.Repeat("7", 64)
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
|
|
}
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
held := put
|
|
held.Step = delivery.PutHolding
|
|
m.Puts = map[string]delivery.Put{"ace": held}
|
|
if w := Wants(m, n, map[string]delivery.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
|
|
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
|
}
|
|
m.Puts = nil
|
|
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
|
|
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
|
|
// condition stays raised. A node with a live want keeps it and is not on neither list.
|
|
m.Composed = composedEverywhere
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
|
if w, wrote := out.Wants["ace"]; wrote {
|
|
t.Errorf("the bus bar wrote ace's want %+v", w)
|
|
}
|
|
h, on := out.Neither["ace"]
|
|
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
|
|
!strings.Contains(h.Words, "assigner_put") {
|
|
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
|
|
}
|
|
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
|
|
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
|
|
}
|
|
stepsOfAPut(t)
|
|
}
|
|
|
|
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
|
|
// operator's yes, the bus's snapshot, the want, and the have.
|
|
func stepsOfAPut(t *testing.T) {
|
|
t.Helper()
|
|
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
|
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
|
|
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
|
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
bus := func() PutWorld {
|
|
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
|
|
Composed: &d, Granted: true, Have: &running, Now: now}
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
world func() PutWorld
|
|
step string
|
|
says string
|
|
ask, snap bool
|
|
warranted bool
|
|
}{
|
|
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, delivery.PutRefused, "not assigned", false, false, false},
|
|
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, delivery.PutBuild, "builds by itself", false, false, false},
|
|
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, delivery.PutRefused, "holds no build", false, false, false},
|
|
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = delivery.BuildFailed, "go vet"; return w }, delivery.PutRefused, "go vet", false, false, false},
|
|
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, delivery.PutCompose, "no composition of nats for novox", false, false, false},
|
|
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, delivery.PutCompose, "does not name novox", false, false, false},
|
|
{"none within the bound: refused", func() PutWorld {
|
|
w := bus()
|
|
w.Composed = nil
|
|
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), delivery.PutCompose, "waits")
|
|
return w
|
|
}, delivery.PutRefused, "mesh-hand-compose", false, false, false},
|
|
{"the bus asks the operator", bus, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
|
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, delivery.PutRefused, "answered no", false, false, false},
|
|
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, delivery.PutRefused, "expired unanswered", false, false, false},
|
|
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, delivery.PutSnapshot, "asked novox's backup holder", false, true, false},
|
|
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, delivery.PutSnapshot, "waits for novox's backup holder", false, false, false},
|
|
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, delivery.PutRefused, "no restic", false, false, false},
|
|
{"snapshotted: the want, warranted", func() PutWorld {
|
|
w := bus()
|
|
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
|
|
w.Put.Ask, _ = delivery.RecordAskKey("novox", "nats", d.Digest)
|
|
return w
|
|
}, delivery.PutWanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
|
|
{"a user-list revision of the bus needs nobody", func() PutWorld {
|
|
w := bus()
|
|
ul := d
|
|
ul.UserListOnly = true
|
|
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
return w
|
|
}, delivery.PutWanted, "held until released", false, false, false},
|
|
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
|
|
{"a recorded module judged to recreate asks", func() PutWorld {
|
|
w := bus()
|
|
w.Bus, w.Recorded = false, true
|
|
r := d
|
|
r.Judgement = delivery.JudgedRecreates
|
|
w.Composed = &r
|
|
return w
|
|
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
|
{"wanted, the node runs it: holding", func() PutWorld {
|
|
w := bus()
|
|
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
|
w.Put.Composed = d.Digest
|
|
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
return w
|
|
}, delivery.PutHolding, "held there until released", false, false, false},
|
|
{"wanted without hold: done", func() PutWorld {
|
|
w := bus()
|
|
w.Put.Hold = false
|
|
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
|
w.Put.Composed = d.Digest
|
|
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
return w
|
|
}, delivery.PutDone, "the rules write its want again", false, false, false},
|
|
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
|
|
}
|
|
for _, c := range cases {
|
|
next := StepPut(c.world())
|
|
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
|
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
|
}
|
|
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
|
|
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
|
|
c.ask, c.snap, c.warranted)
|
|
}
|
|
if next.Ask && next.Put.Ask == "" {
|
|
t.Errorf("put, %s: an ask without its key", c.name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
|
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
|
m, n := ready()
|
|
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew), Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
|
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
|
t.Errorf("the answer was erased: %+v", w)
|
|
}
|
|
}
|
|
|
|
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
|
// wants move by the rules again.
|
|
func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
|
|
m, n := ready()
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
|
|
m.Released = true
|
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleNotHeld}}, map[string]string{"ace": fOld})
|
|
if out.Held != nil {
|
|
t.Errorf("a released fingerprint was held again: %+v", out.Held)
|
|
}
|
|
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) || w.Rule != 0 {
|
|
t.Errorf("ace's want = %+v, want the held fingerprint written again", w)
|
|
}
|
|
}
|
|
|
|
// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes;
|
|
// answered "leave stopped" (or not answered) it stays.
|
|
func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
|
|
m, n := ready()
|
|
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
|
|
m.Restored = func(node string) bool { return false }
|
|
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.FP == fOld {
|
|
t.Errorf("set back without the operator's restore: %+v", w)
|
|
}
|
|
m.Restored = func(node string) bool { return node == "ace" }
|
|
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
|
|
t.Errorf("after restore ace's want = %+v", w)
|
|
}
|
|
}
|
|
|
|
// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included;
|
|
// a composition with none (made by hand) falls back to what the manifest derives.
|
|
func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
|
|
m, n, wanted := movingTo(delivery.JudgedMovesData)
|
|
with := func(policy, from string) {
|
|
inner := m.Composed
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
d, ok := inner(node, fp)
|
|
d.Policy, d.PolicyFrom = policy, from
|
|
return d, ok
|
|
}
|
|
}
|
|
with(delivery.PolicyRecord, "person") // a roll module the operator set to record
|
|
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].FP == fNew {
|
|
t.Errorf("the operator's record policy was not honoured: %+v", out)
|
|
}
|
|
recording(&m) // the manifest says record, the operator chose roll
|
|
with("roll", "person")
|
|
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
|
|
t.Errorf("the operator's roll policy was not honoured: %+v", out)
|
|
}
|
|
with("", "") // made by hand: the manifest decides
|
|
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
|
|
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
|
|
}
|
|
}
|
|
|
|
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
|
|
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
|
|
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
|
m, n, _ := movingTo(delivery.JudgedNeither)
|
|
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
|
|
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
|
|
}
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
|
|
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
|
|
}
|
|
}
|
|
|
|
// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written
|
|
// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it
|
|
// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against
|
|
// the composition the node runs, policy record, UserListOnly set.
|
|
func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) {
|
|
m, n, _ := movingTo(delivery.JudgedRecreates)
|
|
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
userList := "sha256:" + strings.Repeat("7", 64)
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
|
|
Policy: delivery.PolicyRecord, PolicyFrom: "the bus"}
|
|
if fp == fOld {
|
|
d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list
|
|
}
|
|
return d, true
|
|
}
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
for name, wanted := range map[string]map[string]delivery.Want{
|
|
"a want at the fingerprint it runs": {"ace": {FP: fOld, Composed: dig(fOld)}},
|
|
"no readable want": nil,
|
|
} {
|
|
out := Wants(m, n, wanted, nil)
|
|
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != userList {
|
|
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
|
|
}
|
|
if len(out.Record) != 0 {
|
|
t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record)
|
|
}
|
|
}
|
|
// the same flag on a module that is not the bus changes nothing
|
|
m.Manifest.Provides = nil
|
|
if out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
|
|
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
|
|
}
|
|
}
|
|
|
|
// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine
|
|
// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F
|
|
// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the
|
|
// declaration it kept); a new fingerprint still waits for its ask.
|
|
func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
|
|
for name, bus := range map[string]bool{"a record module": false, "the bus": true} {
|
|
// only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold
|
|
unchanged := func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedUnchanged}, true
|
|
}
|
|
m, n, _ := movingTo(delivery.JudgedCannot)
|
|
recording(&m)
|
|
m.Nodes = []string{"ace"}
|
|
if bus {
|
|
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
}
|
|
m.Composed = unchanged
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} // no composition named
|
|
out := Wants(m, n, nil, nil)
|
|
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
|
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
|
|
}
|
|
if len(out.Record) != 0 {
|
|
t.Errorf("%s: asked at the switch: %+v", name, out.Record)
|
|
}
|
|
// a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge,
|
|
// still wait for the ask
|
|
for what, j := range map[string]delivery.Judgement{"differing": delivery.JudgedRecreates, "unjudged": delivery.JudgedCannot, "judged neither": delivery.JudgedNeither} {
|
|
m2, n2, _ := movingTo(j)
|
|
recording(&m2)
|
|
m2.Nodes = []string{"ace"}
|
|
if bus {
|
|
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
|
}
|
|
n2.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
out := Wants(m2, n2, nil, nil)
|
|
if j == delivery.JudgedNeither {
|
|
if out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
|
t.Errorf("%s, %s: %+v", name, what, out)
|
|
}
|
|
continue
|
|
}
|
|
if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 {
|
|
t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record)
|
|
}
|
|
}
|
|
// a have that names a composition is judged as before
|
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
|
|
}
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew && len(out.Record) == 0 {
|
|
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
|
|
}
|
|
// a new fingerprint still asks (the bus is barred instead)
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
out = Wants(m, n, nil, nil)
|
|
if w, ok := out.Wants["ace"]; ok {
|
|
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
|
|
}
|
|
if !bus && len(out.Record) == 0 {
|
|
t.Errorf("%s: a new fingerprint was not asked", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
|
|
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
|
|
m, n := ready()
|
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
|
|
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
|
|
t.Fatalf("held = %+v", out.Held)
|
|
}
|
|
}
|
|
|
|
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
|
|
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
|
|
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
|
|
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
|
everywhere := func() (Module, Nodes) {
|
|
m, n := ready()
|
|
m.Held = true
|
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
return m, n
|
|
}
|
|
m, n := everywhere()
|
|
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
|
|
if !out.Lifted || out.Held != nil {
|
|
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
|
|
}
|
|
for _, node := range []string{"ace", "g14"} {
|
|
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
|
|
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
|
|
}
|
|
}
|
|
|
|
m, n = everywhere()
|
|
delete(n.Haves["g14"], "mail")
|
|
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
|
|
t.Error("a node that does not run the module kept the hold")
|
|
}
|
|
|
|
for name, change := range map[string]func(*Module, *Nodes){
|
|
"set back": func(_ *Module, n *Nodes) {
|
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
},
|
|
"failing": func(_ *Module, n *Nodes) {
|
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
},
|
|
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
|
|
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
|
|
"not held": func(m *Module, _ *Nodes) { m.Held = false },
|
|
} {
|
|
m, n := everywhere()
|
|
change(&m, &n)
|
|
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
|
|
t.Errorf("%s: the hold was lifted", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld
|
|
// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld
|
|
// (health unknown) and holds no want for it. The canary g14 does not run F.
|
|
func x11() (Module, Nodes) {
|
|
m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"},
|
|
GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld },
|
|
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
|
|
n := Nodes{
|
|
Online: map[string]bool{"g14": true, "shanks": true},
|
|
Haves: map[string]map[string]delivery.Have{
|
|
"g14": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
|
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
|
"shanks": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
|
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
|
},
|
|
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
|
|
}
|
|
return m, n
|
|
}
|
|
|
|
// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by
|
|
// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that
|
|
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
|
|
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
|
|
m, n := x11()
|
|
out := Wants(m, n, nil, nil)
|
|
for _, node := range []string{"g14", "shanks"} {
|
|
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
|
|
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
|
|
}
|
|
}
|
|
if len(out.Neither) != 0 {
|
|
t.Errorf("left on neither list: %+v", out.Neither)
|
|
}
|
|
// with the provider healthy, only the canary holds shanks back: still at what it runs
|
|
m, n = x11()
|
|
n.Haves["g14"]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
|
|
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.FP != fOld || w.Rule != delivery.RuleCanary {
|
|
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
|
|
}
|
|
// a node already running F gets its want at F; a removed want is no want either
|
|
m, n = x11()
|
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
|
out = Wants(m, n, map[string]delivery.Want{"g14": {Removed: true}}, nil)
|
|
if w := out.Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) || w.Removed {
|
|
t.Errorf("g14 running F with a removed want = %+v, want F", w)
|
|
}
|
|
}
|
|
|
|
// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no
|
|
// want past the rules, and is named on neither list; the ordering rules still order every move from a want.
|
|
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
|
for what, change := range map[string]func(*Module, *Nodes){
|
|
"an old-form build": func(_ *Module, n *Nodes) {
|
|
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
|
},
|
|
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
|
|
"no composition": func(m *Module, _ *Nodes) {
|
|
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, fp == fNew
|
|
}
|
|
},
|
|
} {
|
|
m, n := x11()
|
|
change(&m, &n)
|
|
out := Wants(m, n, nil, nil)
|
|
if w, ok := out.Wants["shanks"]; ok {
|
|
t.Errorf("%s: shanks got a want: %+v", what, w)
|
|
}
|
|
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
|
|
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
|
|
}
|
|
}
|
|
m, n := x11()
|
|
before := map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}, "shanks": {FP: fOld, Composed: dig(fOld)}}
|
|
out := Wants(m, n, before, nil)
|
|
if len(out.Neither) != 0 {
|
|
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
|
|
}
|
|
for node, w := range out.Wants {
|
|
if w.FP != fOld {
|
|
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
|
|
}
|
|
}
|
|
}
|
|
|
|
// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is
|
|
// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new
|
|
// form goes with the node's own composition and is never named.
|
|
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
|
m, n := x11()
|
|
delete(n.Haves["shanks"], "screen-lock")
|
|
out := Wants(m, n, nil, nil)
|
|
if w, ok := out.Wants["shanks"]; ok {
|
|
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
|
|
}
|
|
if _, ok := out.Neither["shanks"]; ok {
|
|
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
|
|
}
|
|
m.Built = false
|
|
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
|
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
|
}
|
|
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
|
|
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
|
|
// runs the module is on neither list, its removed want left as it is.
|
|
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
|
|
for _, c := range []struct {
|
|
name string
|
|
before map[string]delivery.Want
|
|
want map[string]delivery.Want
|
|
}{
|
|
{"no key: nothing written", nil, map[string]delivery.Want{}},
|
|
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
|
|
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
|
|
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
|
|
map[string]delivery.Want{}},
|
|
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
|
|
} {
|
|
out := Wants(m, n, c.before, nil)
|
|
if len(out.Wants) != len(c.want) {
|
|
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
|
|
continue
|
|
}
|
|
for node, w := range c.want {
|
|
if !delivery.SameWant(normal(out.Wants[node]), w) {
|
|
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
|
|
}
|
|
}
|
|
}
|
|
m.FP = ""
|
|
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
|
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
|
}
|
|
}
|
|
|
|
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
|
|
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
|
|
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
|
|
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
|
|
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
|
oldForm := func(n Nodes, node string) {
|
|
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
|
}
|
|
providersHealthy := func(n Nodes) {
|
|
for _, node := range []string{"g14", "shanks"} {
|
|
n.Haves[node]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
}
|
|
}
|
|
// the canary g14 runs F healthy: shanks's first want names F
|
|
m, n := x11()
|
|
providersHealthy(n)
|
|
oldForm(n, "shanks")
|
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
out := Wants(m, n, nil, nil)
|
|
if w := out.Wants["shanks"]; w.FP != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
|
|
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
|
|
}
|
|
if _, ok := out.Neither["shanks"]; ok {
|
|
t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither)
|
|
}
|
|
// the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3
|
|
m, n = x11()
|
|
providersHealthy(n)
|
|
oldForm(n, "shanks")
|
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
|
out = Wants(m, n, nil, nil)
|
|
if w, ok := out.Wants["shanks"]; ok {
|
|
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
|
|
}
|
|
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
|
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
|
|
}
|
|
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
|
|
m, n = x11()
|
|
oldForm(n, "shanks")
|
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
out = Wants(m, n, nil, nil)
|
|
if w, ok := out.Wants["shanks"]; ok {
|
|
t.Errorf("providers not healthy: shanks got a want: %+v", w)
|
|
}
|
|
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
|
|
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
|
|
}
|
|
// F not granted: nothing, even with the canary healthy
|
|
m, n = x11()
|
|
providersHealthy(n)
|
|
oldForm(n, "shanks")
|
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
|
m.Granted = false
|
|
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
|
|
t.Errorf("F not granted: shanks got a want: %+v", w)
|
|
}
|
|
// the old-form node is the canary itself: F once its providers run healthy
|
|
m, n = x11()
|
|
providersHealthy(n)
|
|
oldForm(n, "g14")
|
|
if w := Wants(m, n, nil, nil).Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) {
|
|
t.Errorf("old-form canary: g14's want = %+v, want F", w)
|
|
}
|
|
}
|
|
|
|
// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking.
|
|
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
|
|
// canary two seconds before the new build was healthy.
|
|
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
|
|
for _, phase := range []delivery.Phase{delivery.PhaseBackingUp, delivery.PhaseApplying, delivery.PhaseChecking} {
|
|
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
|
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
|
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
|
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
|
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
|
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
|
|
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
|
|
}
|
|
}
|
|
for _, phase := range []delivery.Phase{delivery.PhaseRunning, delivery.PhaseFailed} {
|
|
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
|
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
|
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
|
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
|
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
|
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
|
|
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module
|
|
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
|
|
// fingerprint, or failed, still does.
|
|
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
|
|
setup := func(phase delivery.Phase, fp string, health delivery.Health) (Module, Nodes) {
|
|
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
|
|
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]delivery.Have{
|
|
"ace": {"web": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
|
"g14": {"web": {FP: fp, Phase: phase, Health: health}}}}
|
|
return m, n
|
|
}
|
|
for _, phase := range []delivery.Phase{delivery.PhaseRemoved, delivery.PhaseHeld, delivery.PhaseApplying, delivery.PhaseBackingUp, delivery.PhaseChecking} {
|
|
m, n := setup(phase, fOld, delivery.HealthUnknown)
|
|
if out := Wants(m, n, nil, nil); !out.Lifted {
|
|
t.Errorf("g14's have in phase %s kept the hold up", phase)
|
|
}
|
|
}
|
|
for what, h := range map[string]delivery.Have{
|
|
"running another fingerprint": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
|
"failed": {FP: fNew, Phase: delivery.PhaseFailed, Health: delivery.HealthFailed},
|
|
} {
|
|
m, n := setup(h.Phase, h.FP, h.Health)
|
|
if out := Wants(m, n, nil, nil); out.Lifted {
|
|
t.Errorf("g14 %s: the hold was lifted", what)
|
|
}
|
|
}
|
|
// a failed health on a removed have holds nothing
|
|
m, n := setup(delivery.PhaseRemoved, fNew, delivery.HealthFailed)
|
|
m.Held = false
|
|
if out := Wants(m, n, nil, nil); out.Held != nil {
|
|
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
|
|
}
|
|
}
|
|
|
|
// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a
|
|
// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes.
|
|
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
|
|
m, n := x11()
|
|
recording(&m)
|
|
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord}, true
|
|
}
|
|
out := Wants(m, n, map[string]delivery.Want{"shanks": {Removed: true}}, nil)
|
|
if w, ok := out.Wants["shanks"]; ok {
|
|
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
|
|
}
|
|
asked := false
|
|
for _, r := range out.Record {
|
|
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == delivery.JudgedRecreates)
|
|
}
|
|
if !asked {
|
|
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
|
|
}
|
|
}
|
|
|
|
// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the
|
|
// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on
|
|
// neither list: a held module is on the held list, whose hold is said and answered on its own.
|
|
func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
|
|
for what, change := range map[string]func(*Module, *Nodes){
|
|
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
|
|
"held at F, running an old-form build": func(m *Module, n *Nodes) {
|
|
m.Held, m.Kept = true, true
|
|
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
|
},
|
|
"held by this pass": func(_ *Module, n *Nodes) {
|
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
|
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
|
},
|
|
} {
|
|
m, n := x11()
|
|
change(&m, &n)
|
|
out := Wants(m, n, nil, nil)
|
|
if len(out.Neither) != 0 {
|
|
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
|
|
}
|
|
if out.Rules["shanks"].Rule != delivery.RuleNotHeld {
|
|
t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"])
|
|
}
|
|
}
|
|
}
|