novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

AS22: the assigner reads a build's health from the node-engine's running alone

Browse Source 
Rules 3, 4 and 5 (canary, healthy elsewhere, a hold's lift) and the outside-cause
judge read `running.healthy` (a running with none reads healthy, nothing heard
reads not healthy); no fallback to the have. Phase and failed-health reads of the
have stay where `running` says nothing of them.

Builds: design 47 G §520, §556 (survey AS22)
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
jschoubben
2026-11-12 05:42:25 +01:00
parent b4155cc5a2
commit cbfea2f961
6 changed files with 172 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+20 -1
View File
@@ -176,7 +176,14 @@ func startBed(t *testing.T, leaveOut string) *bed {
Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.mesh-catalog.event.>", "mesh.seat.mesh-conditions.event.>", "mesh.seat.mesh-module-assigner.event.>", enginereads.SubjectPrefix + "*"}}); err != nil {
t.Fatal(err)
}
// The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, nodeengine.StreamRunning, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
// The assigner's own stream of its findings' events (design 47 G §383), which its finding writer says into.
@@ -412,6 +419,18 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil {
t.Fatalf("%s %s: %v", bucket, key, err)
}
if bucket == buckets.Have { // the node-engine says `running` with each have it writes (§520): the assigner reads health from it alone
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
h, derr := nodeengineclient.DecodeHave(value)
if err != nil || isNode || derr != nil || h.Phase != nodeengineclient.PhaseRunning {
return
}
ok := h.Health == nodeengineclient.HealthOK
raw, _ := json.Marshal(nodeengine.Running{At: time.Now(), Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, module), raw); err != nil {
t.Fatal(err)
}
}
}
// place adds module to the seat's assignments of node, written as the seat writes it: one node key and a key per module
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go
+1 -1
View File
@@ -269,7 +269,7 @@ func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []s
dirty := o.dirty[node]
cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause]
o.mu.Unlock()
failing, healthy := ok && h.Fp == fp && assign.Failed(h), ok && assign.Healthy(h, fp)
failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp)
if !failing && !healthy && prev == nil {
continue
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go
+2
View File
@@ -66,6 +66,8 @@ func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) as
r.t.Helper()
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}}
healthy := r.health == nodeengineclient.HealthOK
n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}}
if moved == nil {
moved = func(string, string) bool { return false }
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+1 -1
View File
@@ -404,7 +404,7 @@ func (p *passer) wants(ctx context.Context) {
func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool {
return func(owner, node string) bool {
w, ok := wanted[owner][node]
return ok && !w.Removed && !assign.Healthy(n.Haves[node][owner], w.Fp)
return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp)
}
}
modules/mesh-delivery-assigner/internal/assign/wants.go
+15 -14
View File
@@ -159,9 +159,6 @@ func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
}
// Healthy says whether a have runs fp healthy.
func Healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return healthy(h, fp) }
// Failed says whether a have's health check failed once the node is done moving.
func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) }
@@ -202,7 +199,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case canary == "":
return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
case node != canary && !n.RunsHealthy(canary, m.Name, m.FP):
return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
}
@@ -322,15 +319,19 @@ func (m Module) MissingProvisions(node string) []string {
return out
}
// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and
// does not say it stopped; with the seat's events not heard, the have's phase and health.
// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it
// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy.
func (n Nodes) runsHealthy(node, module string) bool {
if n.Running != nil {
r, ok := n.Running[node][module]
return ok && !r.Stopped && r.Fingerprint != ""
}
h, ok := n.Haves[node][module]
return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
r, ok := n.Running[node][module]
return ok && !r.Stopped && r.Fingerprint != "" && (r.Healthy == nil || *r.Healthy)
}
// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone
// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a
// module of which nothing was heard counts as not healthy. The have is never read for it.
func (n Nodes) RunsHealthy(node, module, fp string) bool {
r, ok := n.Running[node][module]
return ok && n.runsHealthy(node, module) && r.Fingerprint == fp
}
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
@@ -702,7 +703,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string {
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
// healthy now.
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) {
out = append(out, other)
}
}
@@ -750,7 +751,7 @@ func allRunHealthy(m Module, n Nodes) bool {
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
continue
}
if !healthy(h, m.FP) {
if !n.RunsHealthy(node, m.Name, m.FP) {
return false
}
ran++
modules/mesh-delivery-assigner/internal/assign/wants_test.go
+133 -83
View File
@@ -60,7 +60,7 @@ func wantedOld() map[string]mwc.Want {
// g14 follows at once: step 0, ADR 0528 clause 3).
func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
m, n := ready()
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
for _, node := range []string{"ace", "g14"} {
if out.Wants[node].Fp != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
@@ -89,7 +89,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
} {
m, n := ready()
breakIt(&m, &n)
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if w, ok := out.Wants["g14"]; ok && w.Fp == fNew {
t.Errorf("rule %d broken: g14's want moved to F", rule)
}
@@ -110,7 +110,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
}, nil} {
m, n := ready()
m.Provisions = all
if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].Fp != fNew {
if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["g14"].Fp != fNew {
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
}
}
@@ -161,7 +161,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m, n := ready()
m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms
n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"]
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if moved := out.Wants["g14"].Fp == fNew; moved == c.held {
t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"])
}
@@ -208,7 +208,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
if c.holds == graph.BusState {
words = graph.BusStateWaitWords("mail", "g14")
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if c.holds == "" {
if out.Wants["g14"].Fp != fNew {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
@@ -235,7 +235,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
return map[string]meshprovisionerclient.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state},
node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}}
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if got := out.Wants["g14"].Fp == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) {
t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words)
}
@@ -280,7 +280,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m.Provisions = func(string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{}
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if c.missing == "" {
if out.Wants["g14"].Fp != fNew || len(m.MissingProvisions("g14")) != 0 {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
@@ -307,7 +307,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m.Provisions = func(string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{}
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if got := out.Wants["g14"].Fp == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != mwc.RuleProvisions ||
!strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) {
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
@@ -342,7 +342,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
if c.have != nil {
n.Haves["novox"][c.module] = *c.have
}
out := Wants(m, n, c.before, nil)
out := Wants(m, heardOfHaves(n), c.before, nil)
w, written := out.Wants["novox"]
if !written {
w = c.before["novox"]
@@ -431,7 +431,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
if c.name == "placed" {
before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)}
}
out := Wants(m, n, before, nil)
out := Wants(m, heardOfHaves(n), before, nil)
if got := out.Wants["g14"]; !reflect.DeepEqual(got, c.want) {
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
}
@@ -453,7 +453,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m, n := ready()
grant := grantsstate.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["ace"]; w.Fp != fNew {
t.Errorf("ace, whose account the grant names, wants %+v", w)
}
@@ -486,7 +486,7 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
if c := Canary(m.Nodes, n.Online); c != "g14" {
t.Fatalf("canary = %q, want g14", c)
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if out.Wants["g14"].Fp != fNew {
t.Errorf("the canary g14's want = %+v", out.Wants["g14"])
}
@@ -500,7 +500,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
t.Fatalf("held = %+v", out.Held)
@@ -516,7 +516,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
before := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}}
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
out = Wants(m, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld})
if out.Held != nil || len(out.Wants) != 0 {
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
}
@@ -534,7 +534,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
n.Online["novox"] = true
n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}}
before["novox"] = mwc.Want{Fp: fOld, Composed: dig(fOld)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if _, alone := out.NodeFailed["g14"]; !alone {
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
}
@@ -547,7 +547,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
delete(before, "novox")
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if w := out.Wants["novox"]; w.Fp != fNew || len(out.Neither) != 0 {
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
}
@@ -569,7 +569,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}
before["novox"] = mwc.Want{Fp: fNew, Composed: dig(fNew)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
}
@@ -584,7 +584,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
if out.Held == nil {
t.Fatal("F was not held")
}
@@ -601,7 +601,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
if len(out.Restore) != 0 {
t.Errorf("restore asks = %v", out.Restore)
}
@@ -612,7 +612,7 @@ func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
m, n := ready()
m.Nodes = []string{"ace"}
out := Wants(m, n, map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil)
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil)
if w := out.Wants["g14"]; !w.Removed {
t.Errorf("g14's want = %+v, want removed", w)
}
@@ -632,7 +632,7 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
for node, w := range out.Wants {
if w.Fp == fNew {
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
@@ -642,11 +642,11 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
}
// and a node that never had a want gets none
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Wants) != 0 {
t.Errorf("wants without a composition: %+v", out.Wants)
}
have["ace|"+fNew] = true
out = Wants(m, n, wantedOld(), nil)
out = Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
}
@@ -664,14 +664,14 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true
}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}}
out := Wants(m, n, running, nil)
out := Wants(m, heardOfHaves(n), running, nil)
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.Fp != fNew || w.Composed != revised {
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
}
}
m.Granted = false
out = Wants(m, n, running, nil)
out = Wants(m, heardOfHaves(n), running, nil)
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
t.Errorf("a revision was written while a rule holds: %+v", w)
}
@@ -679,7 +679,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
out = Wants(m, n, wantedOld(), nil)
out = Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["g14"]; w.Fp != fOld || w.Composed != revised || w.Newest != fNew {
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
}
@@ -687,7 +687,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord,
Against: dig(fOld), Words: "recreates"}, true
}
out = Wants(m, n, wantedOld(), nil)
out = Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["g14"]; w.Composed == revised {
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
}
@@ -707,12 +707,12 @@ func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, running, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.Fp == fOld {
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
}
m.Composed = composedEverywhere
out = Wants(m, n, running, map[string]string{"ace": fOld})
out = Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
t.Errorf("set-back want = %+v", w)
}
@@ -741,7 +741,7 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T)
for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} {
m, n, wanted := movingTo(j)
recording(&m)
out := Wants(m, n, wanted, nil)
out := Wants(m, heardOfHaves(n), wanted, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
}
@@ -756,11 +756,11 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T)
}
// a new revision at the same fingerprint is asked too
same := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}}
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
if out := Wants(m, heardOfHaves(n), same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
}
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
out = Wants(m, n, wanted, nil)
out = Wants(m, heardOfHaves(n), wanted, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: after the yes ace's want = %+v", j, w)
}
@@ -772,25 +772,25 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T)
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
m, n, wanted := movingTo(JudgedNeither)
recording(&m)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("neither: %+v", out)
}
m, n, _ = movingTo(JudgedRecreates)
recording(&m)
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
delete(n.Haves["ace"], "mail")
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("a node that never ran it: %+v", out)
}
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
m, n, _ = movingTo(JudgedCannot)
recording(&m)
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 {
t.Errorf("a node that runs it with no want: %+v", out)
}
m, n, wanted = movingTo(JudgedRecreates)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("a rolled module: %+v", out)
}
}
@@ -802,11 +802,11 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
// backs its streams up before it applies.
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
t.Errorf("the bus did not move by the rules: %+v", out)
}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}, "g14": {Fp: fNew, Composed: dig(fOld)}}
out := Wants(m, n, running, nil)
out := Wants(m, heardOfHaves(n), running, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
}
@@ -835,7 +835,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
if c.step == "" {
m.Puts = nil
}
out := Wants(m, n, c.wanted, nil)
out := Wants(m, heardOfHaves(n), c.wanted, nil)
got, wrote := out.Wants["ace"]
switch {
case c.want == nil && wrote:
@@ -852,7 +852,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
held := put
held.Step = puts.Holding
m.Puts = map[string]puts.Put{"ace": held}
if w := Wants(m, n, map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
if w := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
@@ -983,14 +983,14 @@ func stepsOfAPut(t *testing.T) {
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
m, n := ready()
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew), Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
if w, ok := Wants(m, heardOfHaves(n), running, nil).Wants["ace"]; ok {
t.Errorf("the answer was erased: %+v", w)
}
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
m.Self = func(string) bool { return true }
placed := map[string]mwc.Want{"ace": {Fp: fNew, Self: true, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
if w, ok := Wants(m, heardOfHaves(n), placed, nil).Wants["ace"]; ok {
t.Errorf("the answer in a placed node's want was erased: %+v", w)
}
}
@@ -1001,7 +1001,7 @@ func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"}
m.Released = true
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld})
if out.Held != nil {
t.Errorf("a released fingerprint was held again: %+v", out.Held)
}
@@ -1018,11 +1018,11 @@ func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}
m.Restored = func(node string) bool { return false }
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld {
if w, ok := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld {
t.Errorf("set back without the operator's restore: %+v", w)
}
m.Restored = func(node string) bool { return node == "ace" }
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
if w := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
t.Errorf("after restore ace's want = %+v", w)
}
}
@@ -1040,22 +1040,22 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
}
}
with(catalogue.PolicyRecord, "person") // a roll module the operator set to record
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew {
t.Errorf("the operator's record policy was not honoured: %+v", out)
}
recording(&m) // the manifest says record, the operator chose roll
with("roll", "person")
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
t.Errorf("the operator's roll policy was not honoured: %+v", out)
}
with("", "") // made by hand: the manifest decides
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 {
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
}
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
with(catalogue.PolicyRecord, "person")
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
}
}
@@ -1066,11 +1066,11 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
m, n, _ := movingTo(JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
}
}
@@ -1096,7 +1096,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test
"a want at the fingerprint it runs": {"ace": {Fp: fOld, Composed: dig(fOld)}},
"no readable want": nil,
} {
out := Wants(m, n, wanted, nil)
out := Wants(m, heardOfHaves(n), wanted, nil)
if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != userList {
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
}
@@ -1106,7 +1106,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test
}
// the same flag on a module that is not the bus changes nothing
m.Manifest.Provides = nil
if out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
if out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
}
}
@@ -1129,7 +1129,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
}
m.Composed = unchanged
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
}
@@ -1146,7 +1146,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m2, n2, nil, nil)
out := Wants(m2, heardOfHaves(n2), nil, nil)
if j == JudgedNeither {
if out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("%s, %s: %+v", name, what, out)
@@ -1162,12 +1162,12 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 {
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
}
// a new fingerprint still asks (the bus is barred instead)
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
out = Wants(m, heardOfHaves(n), nil, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
}
@@ -1181,7 +1181,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
t.Fatalf("held = %+v", out.Held)
}
@@ -1198,7 +1198,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
return m, n
}
m, n := everywhere()
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
if !out.Lifted || out.Held != nil {
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
}
@@ -1210,7 +1210,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
m, n = everywhere()
delete(n.Haves["g14"], "mail")
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
if out := Wants(m, heardOfHaves(n), wantedOld(), nil); !out.Lifted {
t.Error("a node that does not run the module kept the hold")
}
@@ -1227,7 +1227,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
} {
m, n := everywhere()
change(&m, &n)
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
if out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
t.Errorf("%s: the hold was lifted", name)
}
}
@@ -1258,7 +1258,7 @@ func x11() (Module, Nodes) {
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
if w := out.Wants["g14"]; w.Fp != fNew || out.Unreached["g14"] == "" {
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
@@ -1273,13 +1273,13 @@ func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T)
m, n = x11()
n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary {
if w := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary {
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
}
// a node already running F gets its want at F; a removed want is no want either
m, n = x11()
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, map[string]mwc.Want{"g14": {Removed: true}}, nil)
out = Wants(m, heardOfHaves(n), map[string]mwc.Want{"g14": {Removed: true}}, nil)
if w := out.Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Removed {
t.Errorf("g14 running F with a removed want = %+v, want F", w)
}
@@ -1301,7 +1301,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if what == "an old-form build" {
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
if w, ok := out.Wants["shanks"]; ok {
@@ -1322,7 +1322,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
}
m, n := x11()
before := map[string]mwc.Want{"g14": {Fp: fOld, Composed: dig(fOld)}, "shanks": {Fp: fOld, Composed: dig(fOld)}}
out := Wants(m, n, before, nil)
out := Wants(m, heardOfHaves(n), before, nil)
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
@@ -1341,7 +1341,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
if w := out.Wants["shanks"]; w.Fp != fNew {
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
@@ -1350,7 +1350,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone {
if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone {
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
}
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
@@ -1369,7 +1369,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
map[string]mwc.Want{}},
{"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}},
} {
out := Wants(m, n, c.before, nil)
out := Wants(m, heardOfHaves(n), c.before, nil)
if len(out.Wants) != len(c.want) {
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
continue
@@ -1381,7 +1381,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
}
}
m.FP = ""
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 0 {
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
}
}
@@ -1404,7 +1404,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if w := out.Wants["shanks"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
}
@@ -1416,7 +1416,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, nil, nil)
out = Wants(m, heardOfHaves(n), nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
}
@@ -1427,7 +1427,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
out = Wants(m, heardOfHaves(n), nil, nil)
if w := out.Wants["shanks"]; w.Fp != fNew || out.Unreached["shanks"] == "" {
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
}
@@ -1440,14 +1440,14 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
m.Granted = false
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; ok {
t.Errorf("F not granted: shanks got a want: %+v", w)
}
// the old-form node is the canary itself: F once its providers run healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "g14")
if w := Wants(m, n, nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) {
if w := Wants(m, heardOfHaves(n), nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("old-form canary: g14's want = %+v, want F", w)
}
}
@@ -1462,7 +1462,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held != nil {
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
}
}
@@ -1472,7 +1472,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
}
}
@@ -1491,7 +1491,7 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
}
for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} {
m, n := setup(phase, fOld, nodeengineclient.HealthUnknown)
if out := Wants(m, n, nil, nil); !out.Lifted {
if out := Wants(m, heardOfHaves(n), nil, nil); !out.Lifted {
t.Errorf("g14's have in phase %s kept the hold up", phase)
}
}
@@ -1500,14 +1500,14 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
"failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed},
} {
m, n := setup(h.Phase, h.Fp, h.Health)
if out := Wants(m, n, nil, nil); out.Lifted {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Lifted {
t.Errorf("g14 %s: the hold was lifted", what)
}
}
// a failed health on a removed have holds nothing
m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed)
m.Held = false
if out := Wants(m, n, nil, nil); out.Held != nil {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Held != nil {
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
}
}
@@ -1520,7 +1520,7 @@ func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true
}
out := Wants(m, n, map[string]mwc.Want{"shanks": {Removed: true}}, nil)
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"shanks": {Removed: true}}, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
}
@@ -1550,7 +1550,7 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if len(out.Neither) != 0 {
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
}
@@ -1602,9 +1602,59 @@ func TestAWantNamesAKeptBuildsPartsAndNodeForANodeThatReadsThem(t *testing.T) {
m, n := ready()
n.Reads = map[string][]string{"ace": row.reads, "g14": row.reads}
m.KeptBuilds = row.kept
w := Wants(m, n, wantedOld(), nil).Wants["ace"]
w := Wants(m, heardOfHaves(n), wantedOld(), nil).Wants["ace"]
if w.Fp != fNew || (w.KeptOn == "g14") != row.wanted || (w.Version == "1.2.3") != row.wanted || (len(w.Parts) == 1) != row.wanted {
t.Errorf("%s: ace's want = %+v", name, w)
}
}
}
// heardOfHaves gives each have that runs the `running` the node-engine seat says of it (design 47 G §520: health is read from
// `running` alone), for a fixture written as haves; a fixture that names its own `running` is left as it is.
func heardOfHaves(n Nodes) Nodes {
if n.Running != nil {
return n
}
n.Running = map[string]map[string]nodeengine.Running{}
for node, byModule := range n.Haves {
n.Running[node] = map[string]nodeengine.Running{}
for module, h := range byModule {
if h.Phase == nodeengineclient.PhaseRunning && h.Fp != "" {
ok := h.Health == nodeengineclient.HealthOK
n.Running[node][module] = nodeengine.Running{Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok}
}
}
}
return n
}
// §520: health is read from the node-engine seat's `running` alone, whatever the have says, and a build of which nothing was heard
// is not healthy.
func TestHealthIsReadFromRunningAlone(t *testing.T) {
yes, no := true, false
say := func(node, module, fp string, healthy *bool) nodeengine.Running {
return nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Healthy: healthy}
}
for _, c := range []struct {
name string
running map[string]map[string]nodeengine.Running
rule int
}{
{"all healthy by running", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &yes)}}, 0},
{"a running with no healthy reads healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, nil), "store": say("ace", "store", fpOf('s'), nil)}}, 0},
{"the canary says unhealthy, its have says healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &no), "store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary},
{"nothing heard of the canary", map[string]map[string]nodeengine.Running{"ace": {"store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary},
} {
m, n := ready()
n.Running = c.running
if got := Wants(m, n, wantedOld(), nil).Rules["g14"].Rule; got != c.rule {
t.Errorf("%s: g14 held by rule %d, want %d", c.name, got, c.rule)
}
}
// rule 5: a provider that says unhealthy, or that nothing was heard of, holds no want but leaves the consumer unreached
m, n := ready()
n.Running = map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &no)}}
if out := Wants(m, n, wantedOld(), nil); len(out.Unreached) == 0 {
t.Errorf("a provider saying unhealthy: no module unreached, %+v", out)
}
}