From cbfea2f961fbd577705c353c56d39d2fc1a24057 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:31:42 +0100 Subject: [PATCH] AS22: the assigner reads a build's health from the node-engine's `running` alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rules 3, 4 and 5 (canary, healthy elsewhere, a hold's lift) and the outside-cause judge read `running.healthy` (a running with none reads healthy, nothing heard reads not healthy); no fallback to the have. Phase and failed-health reads of the have stay where `running` says nothing of them. Builds: design 47 G §520, §556 (survey AS22) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../integration_test.go | 21 +- .../cmd/mesh-delivery-assigner/outside.go | 2 +- .../mesh-delivery-assigner/outside_test.go | 2 + .../cmd/mesh-delivery-assigner/pass.go | 2 +- .../internal/assign/wants.go | 29 +-- .../internal/assign/wants_test.go | 216 +++++++++++------- 6 files changed, 172 insertions(+), 100 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 80f04ef5..ba7ce20e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -176,7 +176,14 @@ func startBed(t *testing.T, leaveOut string) *bed { Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.mesh-catalog.event.>", "mesh.seat.mesh-conditions.event.>", "mesh.seat.mesh-module-assigner.event.>", enginereads.SubjectPrefix + "*"}}); err != nil { + t.Fatal(err) + } + // The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520). + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, nodeengine.StreamRunning, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } // The assigner's own stream of its findings' events (design 47 G §383), which its finding writer says into. @@ -412,6 +419,18 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) { if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil { t.Fatalf("%s %s: %v", bucket, key, err) } + if bucket == buckets.Have { // the node-engine says `running` with each have it writes (§520): the assigner reads health from it alone + node, module, isNode, err := nodeengineclient.ParseHaveKey(key) + h, derr := nodeengineclient.DecodeHave(value) + if err != nil || isNode || derr != nil || h.Phase != nodeengineclient.PhaseRunning { + return + } + ok := h.Health == nodeengineclient.HealthOK + raw, _ := json.Marshal(nodeengine.Running{At: time.Now(), Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok}) + if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, module), raw); err != nil { + t.Fatal(err) + } + } } // place adds module to the seat's assignments of node, written as the seat writes it: one node key and a key per module diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go index 1af5f53c..6fc18759 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go @@ -269,7 +269,7 @@ func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []s dirty := o.dirty[node] cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause] o.mu.Unlock() - failing, healthy := ok && h.Fp == fp && assign.Failed(h), ok && assign.Healthy(h, fp) + failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp) if !failing && !healthy && prev == nil { continue } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go index 4a06236e..3fff5401 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go @@ -66,6 +66,8 @@ func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) as r.t.Helper() n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}} + healthy := r.health == nodeengineclient.HealthOK + n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}} if moved == nil { moved = func(string, string) bool { return false } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 71581d0e..2c9f7b45 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -404,7 +404,7 @@ func (p *passer) wants(ctx context.Context) { func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool { return func(owner, node string) bool { w, ok := wanted[owner][node] - return ok && !w.Removed && !assign.Healthy(n.Haves[node][owner], w.Fp) + return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp) } } diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index d3cb0742..af04e1b6 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -159,9 +159,6 @@ func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK } -// Healthy says whether a have runs fp healthy. -func Healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return healthy(h, fp) } - // Failed says whether a have's health check failed once the node is done moving. func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) } @@ -202,7 +199,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { switch { case canary == "": return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name) - case node != canary && !healthy(n.Haves[canary][m.Name], m.FP): + case node != canary && !n.RunsHealthy(canary, m.Name, m.FP): return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP)) } } @@ -322,15 +319,19 @@ func (m Module) MissingProvisions(node string) []string { return out } -// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and -// does not say it stopped; with the seat's events not heard, the have's phase and health. +// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it +// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy. func (n Nodes) runsHealthy(node, module string) bool { - if n.Running != nil { - r, ok := n.Running[node][module] - return ok && !r.Stopped && r.Fingerprint != "" - } - h, ok := n.Haves[node][module] - return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK + r, ok := n.Running[node][module] + return ok && !r.Stopped && r.Fingerprint != "" && (r.Healthy == nil || *r.Healthy) +} + +// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone +// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a +// module of which nothing was heard counts as not healthy. The have is never read for it. +func (n Nodes) RunsHealthy(node, module, fp string) bool { + r, ok := n.Running[node][module] + return ok && n.runsHealthy(node, module) && r.Fingerprint == fp } // providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned @@ -702,7 +703,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string { for _, other := range slices.Sorted(slices.Values(m.Nodes)) { // A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs // healthy now. - if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) { + if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) { out = append(out, other) } } @@ -750,7 +751,7 @@ func allRunHealthy(m Module, n Nodes) bool { if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) { continue } - if !healthy(h, m.FP) { + if !n.RunsHealthy(node, m.Name, m.FP) { return false } ran++ diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index d0f3be9f..5cd5b4af 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -60,7 +60,7 @@ func wantedOld() map[string]mwc.Want { // g14 follows at once: step 0, ADR 0528 clause 3). func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) { m, n := ready() - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) for _, node := range []string{"ace", "g14"} { if out.Wants[node].Fp != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) { t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew) @@ -89,7 +89,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { } { m, n := ready() breakIt(&m, &n) - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if w, ok := out.Wants["g14"]; ok && w.Fp == fNew { t.Errorf("rule %d broken: g14's want moved to F", rule) } @@ -110,7 +110,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { }, nil} { m, n := ready() m.Provisions = all - if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["g14"].Fp != fNew { t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words) } } @@ -161,7 +161,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m, n := ready() m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if moved := out.Wants["g14"].Fp == fNew; moved == c.held { t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"]) } @@ -208,7 +208,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.holds == graph.BusState { words = graph.BusStateWaitWords("mail", "g14") } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if c.holds == "" { if out.Wants["g14"].Fp != fNew { t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) @@ -235,7 +235,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { return map[string]meshprovisionerclient.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state}, node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if got := out.Wants["g14"].Fp == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) { t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words) } @@ -280,7 +280,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m.Provisions = func(string) map[string]meshprovisionerclient.Provision { return map[string]meshprovisionerclient.Provision{} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if c.missing == "" { if out.Wants["g14"].Fp != fNew || len(m.MissingProvisions("g14")) != 0 { t.Errorf("%s: held %+v", c.name, out.Rules["g14"]) @@ -307,7 +307,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m.Provisions = func(string) map[string]meshprovisionerclient.Provision { return map[string]meshprovisionerclient.Provision{} } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if got := out.Wants["g14"].Fp == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != mwc.RuleProvisions || !strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) { t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"]) @@ -342,7 +342,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.have != nil { n.Haves["novox"][c.module] = *c.have } - out := Wants(m, n, c.before, nil) + out := Wants(m, heardOfHaves(n), c.before, nil) w, written := out.Wants["novox"] if !written { w = c.before["novox"] @@ -431,7 +431,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { if c.name == "placed" { before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} } - out := Wants(m, n, before, nil) + out := Wants(m, heardOfHaves(n), before, nil) if got := out.Wants["g14"]; !reflect.DeepEqual(got, c.want) { t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want) } @@ -453,7 +453,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { m, n := ready() grant := grantsstate.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}} m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["ace"]; w.Fp != fNew { t.Errorf("ace, whose account the grant names, wants %+v", w) } @@ -486,7 +486,7 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) { if c := Canary(m.Nodes, n.Online); c != "g14" { t.Fatalf("canary = %q, want g14", c) } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) if out.Wants["g14"].Fp != fNew { t.Errorf("the canary g14's want = %+v", out.Wants["g14"]) } @@ -500,7 +500,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"} m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC) - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) // 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words. if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" { t.Fatalf("held = %+v", out.Held) @@ -516,7 +516,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { // back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes). n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} before := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld}) + out = Wants(m, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld}) if out.Held != nil || len(out.Wants) != 0 { t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants) } @@ -534,7 +534,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { n.Online["novox"] = true n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}} before["novox"] = mwc.Want{Fp: fOld, Composed: dig(fOld)} - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if _, alone := out.NodeFailed["g14"]; !alone { t.Errorf("g14 does not fail alone: %+v", out.NodeFailed) } @@ -547,7 +547,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { // A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945). n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} delete(before, "novox") - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if w := out.Wants["novox"]; w.Fp != fNew || len(out.Neither) != 0 { t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither) } @@ -569,7 +569,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} } before["novox"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} - out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) + out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld}) if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 { t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed) } @@ -584,7 +584,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) { m, n := ready() m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) if out.Held == nil { t.Fatal("F was not held") } @@ -601,7 +601,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) { func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld}) if len(out.Restore) != 0 { t.Errorf("restore asks = %v", out.Restore) } @@ -612,7 +612,7 @@ func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) { m, n := ready() m.Nodes = []string{"ace"} - out := Wants(m, n, map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil) if w := out.Wants["g14"]; !w.Removed { t.Errorf("g14's want = %+v, want removed", w) } @@ -632,7 +632,7 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { } return composedEverywhere(node, fp) } - out := Wants(m, n, wantedOld(), nil) + out := Wants(m, heardOfHaves(n), wantedOld(), nil) for node, w := range out.Wants { if w.Fp == fNew { t.Errorf("%s's want moved to F before its composition exists: %+v", node, w) @@ -642,11 +642,11 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { t.Errorf("ace is held back by %+v, want rule 7 naming the node", r) } // and a node that never had a want gets none - if out := Wants(m, n, nil, nil); len(out.Wants) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Wants) != 0 { t.Errorf("wants without a composition: %+v", out.Wants) } have["ace|"+fNew] = true - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("ace's want = %+v, want F naming the composition's digest", w) } @@ -664,14 +664,14 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true } running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}} - out := Wants(m, n, running, nil) + out := Wants(m, heardOfHaves(n), running, nil) for _, node := range []string{"ace", "g14"} { if w := out.Wants[node]; w.Fp != fNew || w.Composed != revised { t.Errorf("%s's want = %+v, want F naming the revised digest", node, w) } } m.Granted = false - out = Wants(m, n, running, nil) + out = Wants(m, heardOfHaves(n), running, nil) if w, ok := out.Wants["ace"]; ok && w.Composed == revised { t.Errorf("a revision was written while a rule holds: %+v", w) } @@ -679,7 +679,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { // A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the // controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while // the rules still hold F; for a record module whose revision recreates a container, only after the ask. - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["g14"]; w.Fp != fOld || w.Composed != revised || w.Newest != fNew { t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w) } @@ -687,7 +687,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord, Against: dig(fOld), Words: "recreates"}, true } - out = Wants(m, n, wantedOld(), nil) + out = Wants(m, heardOfHaves(n), wantedOld(), nil) if w := out.Wants["g14"]; w.Composed == revised { t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w) } @@ -707,12 +707,12 @@ func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) { } return composedEverywhere(node, fp) } - out := Wants(m, n, running, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) if w := out.Wants["ace"]; w.Fp == fOld { t.Errorf("set back to a fingerprint whose composition is gone: %+v", w) } m.Composed = composedEverywhere - out = Wants(m, n, running, map[string]string{"ace": fOld}) + out = Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}) if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { t.Errorf("set-back want = %+v", w) } @@ -741,7 +741,7 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} { m, n, wanted := movingTo(j) recording(&m) - out := Wants(m, n, wanted, nil) + out := Wants(m, heardOfHaves(n), wanted, nil) if w, ok := out.Wants["ace"]; ok { t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w) } @@ -756,11 +756,11 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) } // a new revision at the same fingerprint is asked too same := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}} - if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { + if out := Wants(m, heardOfHaves(n), same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out) } m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) } - out = Wants(m, n, wanted, nil) + out = Wants(m, heardOfHaves(n), wanted, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("%s: after the yes ace's want = %+v", j, w) } @@ -772,25 +772,25 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) { m, n, wanted := movingTo(JudgedNeither) recording(&m) - if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("neither: %+v", out) } m, n, _ = movingTo(JudgedRecreates) recording(&m) m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone delete(n.Haves["ace"], "mail") - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("a node that never ran it: %+v", out) } // but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below) m, n, _ = movingTo(JudgedCannot) recording(&m) n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 { t.Errorf("a node that runs it with no want: %+v", out) } m, n, wanted = movingTo(JudgedRecreates) - if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("a rolled module: %+v", out) } } @@ -802,11 +802,11 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} // ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node // backs its streams up before it applies. - if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 { t.Errorf("the bus did not move by the rules: %+v", out) } running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}, "g14": {Fp: fNew, Composed: dig(fOld)}} - out := Wants(m, n, running, nil) + out := Wants(m, heardOfHaves(n), running, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("a user-list revision of the bus: ace's want = %+v", w) } @@ -835,7 +835,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T if c.step == "" { m.Puts = nil } - out := Wants(m, n, c.wanted, nil) + out := Wants(m, heardOfHaves(n), c.wanted, nil) got, wrote := out.Wants["ace"] switch { case c.want == nil && wrote: @@ -852,7 +852,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T held := put held.Step = puts.Holding m.Puts = map[string]puts.Put{"ace": held} - if w := Wants(m, n, map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold { + if w := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold { t.Errorf("a held bus did not take its user-list revision as a put: %+v", w) } m.Puts = nil @@ -983,14 +983,14 @@ func stepsOfAPut(t *testing.T) { func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) { m, n := ready() running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew), Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok { + if w, ok := Wants(m, heardOfHaves(n), running, nil).Wants["ace"]; ok { t.Errorf("the answer was erased: %+v", w) } // A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its // have names is not in the want, so a want kept for its digest would erase the answer at every pass). m.Self = func(string) bool { return true } placed := map[string]mwc.Want{"ace": {Fp: fNew, Self: true, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}} - if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok { + if w, ok := Wants(m, heardOfHaves(n), placed, nil).Wants["ace"]; ok { t.Errorf("the answer in a placed node's want was erased: %+v", w) } } @@ -1001,7 +1001,7 @@ func TestAReleasedHoldIsNotHeldAgain(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"} m.Released = true - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld}) if out.Held != nil { t.Errorf("a released fingerprint was held again: %+v", out.Held) } @@ -1018,11 +1018,11 @@ func TestARestoreAnswerSetsADataModuleBack(t *testing.T) { n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}} m.Restored = func(node string) bool { return false } - if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld { + if w, ok := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld { t.Errorf("set back without the operator's restore: %+v", w) } m.Restored = func(node string) bool { return node == "ace" } - if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { + if w := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) { t.Errorf("after restore ace's want = %+v", w) } } @@ -1040,22 +1040,22 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) { } } with(catalogue.PolicyRecord, "person") // a roll module the operator set to record - if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew { t.Errorf("the operator's record policy was not honoured: %+v", out) } recording(&m) // the manifest says record, the operator chose roll with("roll", "person") - if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { t.Errorf("the operator's roll policy was not honoured: %+v", out) } with("", "") // made by hand: the manifest decides - if out := Wants(m, n, wanted, nil); len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 { t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out) } // 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition. with(catalogue.PolicyRecord, "person") m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"} - if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { + if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew { t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out) } } @@ -1066,11 +1066,11 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) { m, n, _ := movingTo(JudgedNeither) m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok) } n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew { t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok) } } @@ -1096,7 +1096,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test "a want at the fingerprint it runs": {"ace": {Fp: fOld, Composed: dig(fOld)}}, "no readable want": nil, } { - out := Wants(m, n, wanted, nil) + out := Wants(m, heardOfHaves(n), wanted, nil) if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != userList { t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w) } @@ -1106,7 +1106,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test } // the same flag on a module that is not the bus changes nothing m.Manifest.Provides = nil - if out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { + if out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"]) } } @@ -1129,7 +1129,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { } m.Composed = unchanged n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w) } @@ -1146,7 +1146,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} } n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m2, n2, nil, nil) + out := Wants(m2, heardOfHaves(n2), nil, nil) if j == JudgedNeither { if out.Wants["ace"].Fp != fNew || len(out.Record) != 0 { t.Errorf("%s, %s: %+v", name, what, out) @@ -1162,12 +1162,12 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true } n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 { t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out) } // a new fingerprint still asks (the bus is barred instead) n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w, ok := out.Wants["ace"]; ok { t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w) } @@ -1181,7 +1181,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed} - out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld}) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld}) if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" { t.Fatalf("held = %+v", out.Held) } @@ -1198,7 +1198,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { return m, n } m, n := everywhere() - out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}) + out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}) if !out.Lifted || out.Held != nil { t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held) } @@ -1210,7 +1210,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { m, n = everywhere() delete(n.Haves["g14"], "mail") - if out := Wants(m, n, wantedOld(), nil); !out.Lifted { + if out := Wants(m, heardOfHaves(n), wantedOld(), nil); !out.Lifted { t.Error("a node that does not run the module kept the hold") } @@ -1227,7 +1227,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) { } { m, n := everywhere() change(&m, &n) - if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted { + if out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted { t.Errorf("%s: the hold was lifted", name) } } @@ -1258,7 +1258,7 @@ func x11() (Module, Nodes) { // composition, never the newest, and says the newest it is held back from; the canary is never passed over. func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) { m, n := x11() - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) // 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said if w := out.Wants["g14"]; w.Fp != fNew || out.Unreached["g14"] == "" { t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"]) @@ -1273,13 +1273,13 @@ func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) m, n = x11() n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"] - if w := Wants(m, n, nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary { + if w := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary { t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w) } // a node already running F gets its want at F; a removed want is no want either m, n = x11() n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, n, map[string]mwc.Want{"g14": {Removed: true}}, nil) + out = Wants(m, heardOfHaves(n), map[string]mwc.Want{"g14": {Removed: true}}, nil) if w := out.Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Removed { t.Errorf("g14 running F with a removed want = %+v, want F", w) } @@ -1301,7 +1301,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { } { m, n := x11() change(&m, &n) - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if what == "an old-form build" { // the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945) if w, ok := out.Wants["shanks"]; ok { @@ -1322,7 +1322,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { } m, n := x11() before := map[string]mwc.Want{"g14": {Fp: fOld, Composed: dig(fOld)}, "shanks": {Fp: fOld, Composed: dig(fOld)}} - out := Wants(m, n, before, nil) + out := Wants(m, heardOfHaves(n), before, nil) if len(out.Neither) != 0 { t.Errorf("nodes holding a want named on neither list: %+v", out.Neither) } @@ -1341,7 +1341,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) { func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { m, n := x11() delete(n.Haves["shanks"], "screen-lock") - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) // 1960: the canary orders moves, never a first want: the new assignment is wanted at F if w := out.Wants["shanks"]; w.Fp != fNew { t.Errorf("a new assignment while the canary does not run F: %+v, want F", w) @@ -1350,7 +1350,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { t.Errorf("a new assignment is named on neither list: %+v", out.Neither) } m.Built = false - if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone { t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither) } // 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run @@ -1369,7 +1369,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { map[string]mwc.Want{}}, {"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}}, } { - out := Wants(m, n, c.before, nil) + out := Wants(m, heardOfHaves(n), c.before, nil) if len(out.Wants) != len(c.want) { t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want) continue @@ -1381,7 +1381,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { } } m.FP = "" - if out := Wants(m, n, nil, nil); len(out.Neither) != 0 { + if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 0 { t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither) } } @@ -1404,7 +1404,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { providersHealthy(n) oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["shanks"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 { t.Errorf("canary healthy on F: shanks's want = %+v, want F", w) } @@ -1416,7 +1416,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { providersHealthy(n) oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w, ok := out.Wants["shanks"]; ok { t.Errorf("canary not healthy on F: shanks got a want: %+v", w) } @@ -1427,7 +1427,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { m, n = x11() oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} - out = Wants(m, n, nil, nil) + out = Wants(m, heardOfHaves(n), nil, nil) if w := out.Wants["shanks"]; w.Fp != fNew || out.Unreached["shanks"] == "" { t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"]) } @@ -1440,14 +1440,14 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) { oldForm(n, "shanks") n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} m.Granted = false - if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok { + if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; ok { t.Errorf("F not granted: shanks got a want: %+v", w) } // the old-form node is the canary itself: F once its providers run healthy m, n = x11() providersHealthy(n) oldForm(n, "g14") - if w := Wants(m, n, nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) { + if w := Wants(m, heardOfHaves(n), nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) { t.Errorf("old-form canary: g14's want = %+v, want F", w) } } @@ -1462,7 +1462,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) { n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil { + if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held != nil { t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held) } } @@ -1472,7 +1472,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) { n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{ "novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}} before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}} - if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" { + if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" { t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held) } } @@ -1491,7 +1491,7 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) { } for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} { m, n := setup(phase, fOld, nodeengineclient.HealthUnknown) - if out := Wants(m, n, nil, nil); !out.Lifted { + if out := Wants(m, heardOfHaves(n), nil, nil); !out.Lifted { t.Errorf("g14's have in phase %s kept the hold up", phase) } } @@ -1500,14 +1500,14 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) { "failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}, } { m, n := setup(h.Phase, h.Fp, h.Health) - if out := Wants(m, n, nil, nil); out.Lifted { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Lifted { t.Errorf("g14 %s: the hold was lifted", what) } } // a failed health on a removed have holds nothing m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed) m.Held = false - if out := Wants(m, n, nil, nil); out.Held != nil { + if out := Wants(m, heardOfHaves(n), nil, nil); out.Held != nil { t.Errorf("a removed have with a failed health held F: %+v", out.Held) } } @@ -1520,7 +1520,7 @@ func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) { m.Composed = func(_, fp string) (ComposedDigest, bool) { return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true } - out := Wants(m, n, map[string]mwc.Want{"shanks": {Removed: true}}, nil) + out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"shanks": {Removed: true}}, nil) if w, ok := out.Wants["shanks"]; ok { t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w) } @@ -1550,7 +1550,7 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) { } { m, n := x11() change(&m, &n) - out := Wants(m, n, nil, nil) + out := Wants(m, heardOfHaves(n), nil, nil) if len(out.Neither) != 0 { t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither) } @@ -1602,9 +1602,59 @@ func TestAWantNamesAKeptBuildsPartsAndNodeForANodeThatReadsThem(t *testing.T) { m, n := ready() n.Reads = map[string][]string{"ace": row.reads, "g14": row.reads} m.KeptBuilds = row.kept - w := Wants(m, n, wantedOld(), nil).Wants["ace"] + w := Wants(m, heardOfHaves(n), wantedOld(), nil).Wants["ace"] if w.Fp != fNew || (w.KeptOn == "g14") != row.wanted || (w.Version == "1.2.3") != row.wanted || (len(w.Parts) == 1) != row.wanted { t.Errorf("%s: ace's want = %+v", name, w) } } } + +// heardOfHaves gives each have that runs the `running` the node-engine seat says of it (design 47 G §520: health is read from +// `running` alone), for a fixture written as haves; a fixture that names its own `running` is left as it is. +func heardOfHaves(n Nodes) Nodes { + if n.Running != nil { + return n + } + n.Running = map[string]map[string]nodeengine.Running{} + for node, byModule := range n.Haves { + n.Running[node] = map[string]nodeengine.Running{} + for module, h := range byModule { + if h.Phase == nodeengineclient.PhaseRunning && h.Fp != "" { + ok := h.Health == nodeengineclient.HealthOK + n.Running[node][module] = nodeengine.Running{Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok} + } + } + } + return n +} + +// §520: health is read from the node-engine seat's `running` alone, whatever the have says, and a build of which nothing was heard +// is not healthy. +func TestHealthIsReadFromRunningAlone(t *testing.T) { + yes, no := true, false + say := func(node, module, fp string, healthy *bool) nodeengine.Running { + return nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Healthy: healthy} + } + for _, c := range []struct { + name string + running map[string]map[string]nodeengine.Running + rule int + }{ + {"all healthy by running", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &yes)}}, 0}, + {"a running with no healthy reads healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, nil), "store": say("ace", "store", fpOf('s'), nil)}}, 0}, + {"the canary says unhealthy, its have says healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &no), "store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, + {"nothing heard of the canary", map[string]map[string]nodeengine.Running{"ace": {"store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary}, + } { + m, n := ready() + n.Running = c.running + if got := Wants(m, n, wantedOld(), nil).Rules["g14"].Rule; got != c.rule { + t.Errorf("%s: g14 held by rule %d, want %d", c.name, got, c.rule) + } + } + // rule 5: a provider that says unhealthy, or that nothing was heard of, holds no want but leaves the consumer unreached + m, n := ready() + n.Running = map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &no)}} + if out := Wants(m, n, wantedOld(), nil); len(out.Unreached) == 0 { + t.Errorf("a provider saying unhealthy: no module unreached, %+v", out) + } +}