Files
mesh-delivery/modules/mesh-delivery-assigner/internal/assign/wants_test.go
T
jschoubben b71358e2cc
mesh/delivery merged: source.main moved to novox/mesh-delivery@05e21fd8a382
mesh/check being carried to b71358e2cc14 on 05e21fd8a382 | from fc21f2d1bd56
A put without a hold ends once the node runs its fingerprint healthy, and assigner_puts says a put older than the newest build
A put with hold false waited for the node's composition digest to equal its own and never ended when the node composes its own,
pinning the module's want (issue 1966, F21).

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
2026-11-09 19:54:16 +01:00

1548 lines
83 KiB
Go

package assign
import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"mesh-delivery-assigner/internal/puts"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
func fpOf(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) }
var (
fNew = fpOf('b')
fOld = fpOf('a')
)
// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider
// store healthy on ace and the canary ace running fNew healthy.
func ready() (Module, Nodes) {
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"},
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
n := Nodes{
Online: map[string]bool{"ace": true, "g14": true},
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"mail": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, "store": {Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}},
"g14": {"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}},
},
Providers: map[string]map[string][]string{"database": {"store": {"ace"}}},
}
return m, n
}
// dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated.
func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) }
// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither.
func composedEverywhere(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedNeither}, true
}
func wantedOld() map[string]mwc.Want {
return map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}, "g14": {Fp: fOld, Composed: dig(fOld)}}
}
// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so
// g14 follows at once: step 0, ADR 0528 clause 3).
func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
m, n := ready()
out := Wants(m, n, wantedOld(), nil)
for _, node := range []string{"ace", "g14"} {
if out.Wants[node].Fp != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
}
}
}
// 1297/B4, one test per rule: given every rule but N holds, no want moves to F, and the rule N is named.
func TestEachRuleHoldsTheWantBack(t *testing.T) {
for rule, breakIt := range map[int]func(*Module, *Nodes){
mwc.RuleBuildDone: func(m *Module, _ *Nodes) { m.Built = false },
mwc.RuleNotHeld: func(m *Module, _ *Nodes) { m.Held = true },
mwc.RuleCanary: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
},
mwc.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
mwc.RuleComposed: func(m *Module, _ *Nodes) {
m.Composed = func(string, string) (ComposedDigest, bool) { return ComposedDigest{}, false }
},
// 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want.
mwc.RuleProvisions: func(m *Module, _ *Nodes) {
m.Provisions = func(node string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{node + ".mail.database": {Provision: "database", State: meshprovisionerclient.Asked}, node + ".mail.oidc": {Provision: "oidc", State: meshprovisionerclient.Provided}}
}
},
} {
m, n := ready()
breakIt(&m, &n)
out := Wants(m, n, wantedOld(), nil)
if w, ok := out.Wants["g14"]; ok && w.Fp == fNew {
t.Errorf("rule %d broken: g14's want moved to F", rule)
}
if got := out.Rules["g14"].Rule; got != rule {
t.Errorf("rule %d broken: g14 is held back by rule %d (%s)", rule, got, out.Rules["g14"].Words)
}
if w := out.Wants["g14"]; w.Fp != "" && (w.Fp != fOld || w.Newest != fNew || w.Rule != int64(rule)) {
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
}
if rule == mwc.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) {
t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words)
}
}
// 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it.
for _, all := range []func(string) map[string]meshprovisionerclient.Provision{
func(node string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{node + ".mail.database": {Provision: "database", State: meshprovisionerclient.Provided}}
}, nil} {
m, n := ready()
m.Provisions = all
if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].Fp != fNew {
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
}
}
// ADR 0614 §4: a want is held on a node whose engine does not state a reader capability the manifest needs, under rule 7
// with the capability as its words, and moves once the engine states it; an engine stating nothing reads the baseline,
// so a manifest of the baseline's words, the node-engine's own new build among them, is not held.
hostNeeds, err := catalogue.ManifestNeeds([]byte(`{"module":"mesh-host","version":"1","slug":"host","build":{"artifacts":[` +
`{"name":"host-arch","kind":"bundle","language":"go","system":"arch","from":"cmd/mesh-host"}]},"consumes":["restic.backed-up"],` +
`"state":[{"name":"health"}],"config":{"x":{"type":"number","minimum":1,"default":50,"runtime":true}},` +
`"resources":[{"id":"k","type":"file","path":"/var/lib/k","content":"${config:x}\n"}]}`))
if err != nil {
t.Fatal(err)
}
for _, c := range []struct {
why string
name string
needs []string
reads map[string][]string
held bool
// forms are the build's reference forms (ADR 0624 §2), and names what the hold's words name.
forms []string
names string
}{
{"a word the baseline lacks, on engines stating nothing", "mail", []string{"word:module", "word:tool-runner"}, nil, true, nil, "word:tool-runner"},
{"the same, once both engines state it", "mail", []string{"word:module", "word:tool-runner"},
map[string][]string{"ace": catalogue.ManifestReads(), "g14": catalogue.ManifestReads()}, false, nil, ""},
{"the baseline's words on engines stating nothing", "mail", []string{"word:claims.serves", "seat:mesh-catalog", "form:config"}, nil, false, nil, ""},
{"the node-engine's new build, whose contract has more words than its manifest uses", "mesh-host", hostNeeds, nil, false, nil, ""},
// Issue 1966: g14's engine states its words and forms but no seat (its statement made before its seat list was in
// use); needs of the bootstrapped seats mesh-catalog and node-backup move all the same (ADR 0636 §2), one of a seat
// not bootstrapped is held.
{"mesh-catalog on an engine stating no seat", "mesh-host", []string{"seat:mesh-catalog", "word:module"},
map[string][]string{"g14": {"word:module"}}, false, nil, ""},
{"node-backup on an engine stating no seat", "restic", []string{"seat:node-backup", "word:module"},
map[string][]string{"g14": {"word:module"}}, false, nil, ""},
{"a seat not bootstrapped on an engine stating no seat: held, naming it", "mail", []string{"seat:mesh-vault", "word:module"},
map[string][]string{"g14": {"word:module"}}, true, nil, "seat:mesh-vault"},
// ADR 0624 §2, with form:new a form no engine reads unless it states it and artifact-store the first form.
{"g14 back stating {old}, a build carrying {old,new}: moves", "mail", nil, map[string][]string{"g14": {"word:module"}}, false,
[]string{"form:artifact-store", "form:new"}, ""},
{"an engine stating nothing, a build carrying the baseline's form: moves", "mail", nil, nil, false, []string{"form:artifact-store"}, ""},
{"a build carrying {new} only: held, naming the form", "mail", nil, map[string][]string{"g14": {"word:module"}}, true,
[]string{"form:new"}, "form:new"},
{"the same, once g14's engine states it", "mail", nil, map[string][]string{"g14": {"form:new"}}, false, []string{"form:new"}, ""},
{"a manifest needing an unstated word, its build in the first form: held, naming the word", "mail", []string{"word:tool-runner"}, nil, true,
[]string{"form:artifact-store"}, "word:tool-runner"},
} {
m, n := ready()
m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms
n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"]
out := Wants(m, n, wantedOld(), nil)
if moved := out.Wants["g14"].Fp == fNew; moved == c.held {
t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"])
}
if r := out.Rules["g14"]; c.held && (r.Rule != mwc.RuleComposed || !strings.Contains(r.Words, c.names) ||
!strings.Contains(r.Words, "node-engine of g14")) {
t.Errorf("%s: held by %+v, want rule 7 naming the capability", c.why, r)
}
if w := out.Wants["g14"]; c.held && (w.Rule != mwc.RuleComposed || w.Newest != fNew) {
t.Errorf("%s: g14's want %+v, want it kept at what it runs and naming rule 7", c.why, w)
}
}
// ADR 0614 §2, §3: a want waits, under the provisions' rule, until each verb it invokes is served by a running build,
// and moves once one is; a verb a running build serves already adds no wait, and an edge the graph leaves out on a cycle
// is not among Invokes, so it holds nothing; nothing read of what runs waits for none.
for _, c := range []struct {
name string
invokes []string
served []string
nilRead bool
holds string
}{
{"a verb no running build serves", []string{"store.store_put", "seat:mesh-catalog.modules"}, []string{"store.store_put"}, false, "seat:mesh-catalog.modules"},
{"once a build serving it runs", []string{"seat:mesh-catalog.modules"}, []string{"seat:mesh-catalog.modules"}, false, ""},
{"an edge left out on a cycle", nil, nil, false, ""},
{"nothing read of what runs", []string{"seat:mesh-catalog.modules"}, nil, true, ""},
// ADR 0614 §5: a module asking bus state is held while no build offering bus-state runs, and moves once one does.
{"bus state no running build offers", nil, nil, false, graph.BusState},
{"bus state a running build offers", nil, []string{graph.BusState}, false, ""},
{"bus state its own module offers", nil, nil, false, ""},
} {
m, n := ready()
m.Invokes = c.invokes
if !c.nilRead {
m.Served = func(v string) bool { return slices.Contains(c.served, v) }
}
if strings.HasPrefix(c.name, "bus state") {
m.Manifest.BusState = []catalogue.BusStateEntry{{Name: "cache", Storage: "key-value"}}
m.Offering = func(p string) bool { return slices.Contains(c.served, p) }
if c.name == "bus state its own module offers" {
m.Manifest.Provides = []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}
}
}
words := graph.VerbWaitWords(c.holds, "mail", "g14")
if c.holds == graph.BusState {
words = graph.BusStateWaitWords("mail", "g14")
}
out := Wants(m, n, wantedOld(), nil)
if c.holds == "" {
if out.Wants["g14"].Fp != fNew {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
}
continue
}
if out.Wants["g14"].Fp == fNew || out.Rules["g14"].Rule != mwc.RuleProvisions || out.Rules["g14"].Words != words {
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
}
}
// ADR 0603 §9: the newest source revision governs. The module now at fNew no longer requires the database an older revision
// asked for and requires oidc instead; the seat's keys follow the newest manifest, so the older edge is no wait: a provided
// oidc moves the want, an asked one holds it and names oidc and never database.
for _, c := range []struct {
name string
state, older string
moves bool
}{{"the replaced edge provided, the older edge retiring", meshprovisionerclient.Provided, meshprovisionerclient.Retiring, true},
{"the replaced edge provided, the older edge asked", meshprovisionerclient.Provided, meshprovisionerclient.Asked, true},
{"the replaced edge asked, the older edge retiring", meshprovisionerclient.Asked, meshprovisionerclient.Retiring, false}} {
m, n := ready()
m.Manifest.Requires = []string{"oidc"}
m.Provisions = func(node string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state},
node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}}
}
out := Wants(m, n, wantedOld(), nil)
if got := out.Wants["g14"].Fp == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) {
t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words)
}
}
// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the
// composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for
// oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine.
{
handed := &catalogue.Handed{Source: "s", Digest: "d", Templates: map[string]string{"c": "c"}}
keycloak := catalogue.Manifest{Module: "keycloak", Provides: []catalogue.Offer{{Name: "oidc-client", Scope: catalogue.ScopeMesh}}}
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route", Scope: catalogue.ScopeMesh,
AnsweredBy: catalogue.AnsweredByHelper}}}
store := catalogue.Manifest{Module: "store", Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}}
goTools := catalogue.Manifest{Module: "go", Provides: []catalogue.Offer{{Name: "go-toolchain", Scope: catalogue.ScopeNode}}}
gitea := catalogue.Manifest{Module: "gitea", Provides: []catalogue.Offer{{Name: "forge-account", Scope: catalogue.ScopeMesh}}}
answering := func(p string) []catalogue.Manifest {
for _, m := range []catalogue.Manifest{keycloak, proxy, store, goTools, gitea} {
if m.Provides[0].Name == p {
return []catalogue.Manifest{m}
}
}
return nil
}
for _, c := range []struct {
name string
manifest catalogue.Manifest
missing string // the provision named, "" for none: the want moves
}{
{"a handed module's database with no key", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
Requires: []string{"database"}}, "database"},
{"a module not handed waits for no login the controller makes (grafana on ace, oidc-client)",
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client"}}, ""},
{"a module not handed waits for its route, which the helper answers (grafana's route)",
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client", "route"}}, "route"},
{"a handed builder waits for no toolchain on its own machine (mesh-build-go, go-toolchain)",
catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, Requires: []string{"go-toolchain"}}, ""},
{"and does wait for its forge account", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
Requires: []string{"go-toolchain", "forge-account"}}, "forge-account"},
} {
m, n := ready()
m.Manifest, m.Answering = c.manifest, answering
m.Provisions = func(string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{}
}
out := Wants(m, n, wantedOld(), nil)
if c.missing == "" {
if out.Wants["g14"].Fp != fNew || len(m.MissingProvisions("g14")) != 0 {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
}
continue
}
if out.Rules["g14"].Rule != mwc.RuleProvisions || !strings.Contains(out.Rules["g14"].Words,
"provision "+c.missing+" has no key yet") || len(m.MissingProvisions("g14")) != 1 {
t.Errorf("%s: %+v", c.name, out.Rules["g14"])
}
}
}
// ADR 0603 §5: a requirement nobody offers is an unknown provider and waits, loudly; one something offers (a module's own
// name, a module the mesh provides itself) is left to catalogue.Awaited as before and does not.
for _, c := range []struct {
name, requires string
offered bool
}{{"a module's own name (node-tools)", "node-tools", true}, {"a module the mesh provides itself (mesh-addressing)", "mesh-addressing", true},
{"a provision nobody offers", "database", false}} {
m, n := ready()
m.Manifest.Requires = []string{c.requires}
m.Answering = func(string) []catalogue.Manifest { return nil }
m.Offered = func(p string) bool { return c.offered && p == c.requires }
m.Provisions = func(string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{}
}
out := Wants(m, n, wantedOld(), nil)
if got := out.Wants["g14"].Fp == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != mwc.RuleProvisions ||
!strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) {
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
}
}
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
// healthy held photos, and novox offline left a module with no canary.
for _, c := range []struct {
name, module, provision, provider string
have *nodeengine.HaveByNodeModuleValue // what novox runs of the module; nil: a new assignment
before map[string]mwc.Want
online bool
want string
}{
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
&nodeengine.HaveByNodeModuleValue{Fp: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}, nil, true, fNew},
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
&nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK},
map[string]mwc.Want{"novox": {Fp: fOld, Composed: dig(fOld)}}, true, fNew},
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
&nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK},
map[string]mwc.Want{"novox": {Fp: fOld, Composed: dig(fOld)}}, false, fOld},
} {
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {
c.provider: {Fp: fpOf('p'), Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}}},
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
if c.have != nil {
n.Haves["novox"][c.module] = *c.have
}
out := Wants(m, n, c.before, nil)
w, written := out.Wants["novox"]
if !written {
w = c.before["novox"]
}
if w.Fp != c.want || w.Removed {
t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want))
}
if _, on := out.Neither["novox"]; on {
t.Errorf("%s: on neither list: %+v", c.name, out.Neither)
}
if c.online && !strings.Contains(out.Unreached["novox"], c.provision) {
t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"])
}
}
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the assignments is gone, names
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
// rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it
// stays and is said; a placed node's want is Self with no composition.
selfAt := func(fp string) map[string]mwc.Want {
return map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fp, Self: true, Yes: dig(fp)}}
}
atF := func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthOK}
}
for _, c := range []struct {
name string
set func(*Module, *Nodes)
before string
want mwc.Want // the want of g14 written; zero: none written
rule int // the rule g14's holding names
asked bool
}{
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["store"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthWaits}
}, want: mwc.Want{Fp: fNew, Composed: dig(fNew)}},
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}, want: mwc.Want{Fp: fNew, Composed: dig(fNew)}, rule: mwc.RuleCanary},
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}, want: mwc.Want{Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleCanary,
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: mwc.RuleCanary},
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
want: mwc.Want{Fp: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"},
rule: mwc.RuleNotHeld},
{name: "held data module", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Held = true
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
}, want: mwc.Want{Fp: fNew, Composed: dig(fNew),
Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: mwc.RuleNotHeld},
{name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
recording(m)
m.Held = true
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld)}, true
}
}, rule: mwc.RuleNotHeld, asked: true},
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if node == "g14" {
return ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
}, rule: mwc.RuleComposed},
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Self = func(node string) bool { return node == "g14" }
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if node == "g14" {
return ComposedDigest{Judgement: JudgedNeither}, true
}
return composedEverywhere(node, fp)
}
}, want: mwc.Want{Fp: fNew, Self: true}},
} {
m, n := ready()
c.set(&m, &n)
before := selfAt(c.before)
if c.name == "placed" {
before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)}
}
out := Wants(m, n, before, nil)
if got := out.Wants["g14"]; got != c.want {
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
}
if got := out.Rules["g14"].Rule; got != c.rule {
t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule)
}
if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked {
t.Errorf("%s: record asks %+v", c.name, out.Record)
}
if w, ok := out.Wants["g14"]; ok {
if _, err := mwc.EncodeWant(w); err != nil {
t.Errorf("%s: g14's want does not encode: %v", c.name, err)
}
}
}
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
m, n := ready()
grant := grantsstate.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
out := Wants(m, n, wantedOld(), nil)
if w := out.Wants["ace"]; w.Fp != fNew {
t.Errorf("ace, whose account the grant names, wants %+v", w)
}
if got := out.Rules["g14"]; got.Rule != mwc.RuleGrant || !strings.Contains(got.Words, "on g14") {
t.Errorf("g14, whose account the grant does not name, is held by %+v", got)
}
for _, c := range []struct {
accounts []string
node, module string
named bool
}{
{nil, "g14", "mail", true},
{[]string{"g14.mail.send"}, "g14", "mail", true},
{[]string{"g14.mailer"}, "g14", "mail", false},
{[]string{"ace.mail"}, "g14", "mail", false},
} {
if got := GrantNamesNode(grantsstate.Grant{Accounts: c.accounts}, c.node, c.module); got != c.named {
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
}
}
}
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
m, n := ready()
n.Online["ace"] = false
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
n.Providers["database"]["store"] = []string{"ace", "g14"}
n.Haves["g14"]["store"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if c := Canary(m.Nodes, n.Online); c != "g14" {
t.Fatalf("canary = %q, want g14", c)
}
out := Wants(m, n, wantedOld(), nil)
if out.Wants["g14"].Fp != fNew {
t.Errorf("the canary g14's want = %+v", out.Wants["g14"])
}
if out.Rules["ace"].Rule != mwc.RuleCanary {
t.Errorf("ace, behind the canary g14, is held back by %+v", out.Rules["ace"])
}
}
// 1297/B4: a failed health check of F holds F and sets the nodes that want it back to their last healthy fingerprint.
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
t.Fatalf("held = %+v", out.Held)
}
if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != mwc.RuleNotHeld {
t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w)
}
if w, ok := out.Wants["g14"]; ok && w.Fp == fNew {
t.Errorf("g14 moved to a held fingerprint: %+v", w)
}
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
before := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}}
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
if out.Held != nil || len(out.Wants) != 0 {
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
}
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
len(out.NodeFailed) != 1 {
t.Errorf("node failed = %+v", out.NodeFailed)
}
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
three := m
three.Nodes = []string{"ace", "g14", "novox"}
failedHave := n.Haves["ace"]["mail"]
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
n.Online["novox"] = true
n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}}
before["novox"] = mwc.Want{Fp: fOld, Composed: dig(fOld)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if _, alone := out.NodeFailed["g14"]; !alone {
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
}
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.Fp == fNew) || out.Rules["novox"].Rule != mwc.RuleNotHeld {
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
}
if _, written := out.Wants["g14"]; written {
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
}
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
delete(before, "novox")
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if w := out.Wants["novox"]; w.Fp != fNew || len(out.Neither) != 0 {
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
}
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
// not one node's failure (D3). Either way F is held and ace is set back.
for _, c := range []struct {
name string
online map[string]bool
novox string
}{
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, nodeengineclient.HealthOK},
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, nodeengineclient.HealthFailed},
} {
n.Online = c.online
n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: c.novox}}
if c.novox == nodeengineclient.HealthOK {
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}
before["novox"] = mwc.Want{Fp: fNew, Composed: dig(fNew)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
}
if w := out.Wants["ace"]; w.Fp != fOld {
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
}
}
}
// 1297/B4: a module holding data is not set back; its want stays and says why.
func TestADataModuleIsNotSetBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
if out.Held == nil {
t.Fatal("F was not held")
}
if w := out.Wants["ace"]; w.Fp != fNew || !strings.Contains(w.Words, "holds data") {
t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w)
}
// 1906/D3: and the node is to be asked whether to restore it
if !slices.Equal(out.Restore, []string{"ace"}) {
t.Errorf("restore asks = %v, want ace", out.Restore)
}
}
// 1906/D3: a module without data is set back and asked about nothing.
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
if len(out.Restore) != 0 {
t.Errorf("restore asks = %v", out.Restore)
}
}
// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that
// is already as the rules give it is not written again.
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
m, n := ready()
m.Nodes = []string{"ace"}
out := Wants(m, n, map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil)
if w := out.Wants["g14"]; !w.Removed {
t.Errorf("g14's want = %+v, want removed", w)
}
if _, ok := out.Wants["ace"]; ok {
t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"])
}
}
// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its
// digest once it exists; a node with no want before still gets none until then.
func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
m, n := ready()
have := map[string]bool{}
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if !have[node+"|"+fp] {
return ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, wantedOld(), nil)
for node, w := range out.Wants {
if w.Fp == fNew {
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
}
}
if r := out.Rules["ace"]; r.Rule != mwc.RuleComposed || !strings.Contains(r.Words, "ace") {
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
}
// and a node that never had a want gets none
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
t.Errorf("wants without a composition: %+v", out.Wants)
}
have["ace|"+fNew] = true
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
}
if w, ok := out.Wants["g14"]; ok && w.Fp == fNew {
t.Errorf("g14, whose composition does not exist, moved: %+v", w)
}
}
// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want,
// by the same rules: written when they hold, and not while one does not.
func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
m, n := ready()
revised := "sha256:" + strings.Repeat("9", 64)
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true
}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}}
out := Wants(m, n, running, nil)
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.Fp != fNew || w.Composed != revised {
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
}
}
m.Granted = false
out = Wants(m, n, running, nil)
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
t.Errorf("a revision was written while a rule holds: %+v", w)
}
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["g14"]; w.Fp != fOld || w.Composed != revised || w.Newest != fNew {
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
}
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord,
Against: dig(fOld), Words: "recreates"}, true
}
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["g14"]; w.Composed == revised {
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
}
if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) {
t.Errorf("the revision was not asked: %+v", out.Record)
}
}
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if fp == fOld {
return ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.Fp == fOld {
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
}
m.Composed = composedEverywhere
out = Wants(m, n, running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
t.Errorf("set-back want = %+v", w)
}
}
// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F).
func movingTo(j Judgement) (Module, Nodes, map[string]mwc.Want) {
m, n := ready()
m.Composed = func(node, fp string) (ComposedDigest, bool) {
d := ComposedDigest{Digest: dig(fp), Judgement: j}
if j != JudgedNeither {
d.Against, d.Words = dig(fOld), "judged"
}
return d, true
}
return m, n, wantedOld()
}
func recording(m *Module) {
m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"}
}
// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be
// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint.
func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) {
for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} {
m, n, wanted := movingTo(j)
recording(&m)
out := Wants(m, n, wanted, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
}
var ace *RecordNeed
for i := range out.Record {
if out.Record[i].Node == "ace" {
ace = &out.Record[i]
}
}
if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j {
t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record)
}
// a new revision at the same fingerprint is asked too
same := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}}
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
}
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
out = Wants(m, n, wanted, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: after the yes ace's want = %+v", j, w)
}
}
}
// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module
// that rolls (not record) is never asked.
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
m, n, wanted := movingTo(JudgedNeither)
recording(&m)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("neither: %+v", out)
}
m, n, _ = movingTo(JudgedRecreates)
recording(&m)
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
delete(n.Haves["ace"], "mail")
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("a node that never ran it: %+v", out)
}
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
m, n, _ = movingTo(JudgedCannot)
recording(&m)
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 {
t.Errorf("a node that runs it with no want: %+v", out)
}
m, n, wanted = movingTo(JudgedRecreates)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("a rolled module: %+v", out)
}
}
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
m, n, _ := movingTo(JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
// backs its streams up before it applies.
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
t.Errorf("the bus did not move by the rules: %+v", out)
}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}, "g14": {Fp: fNew, Composed: dig(fOld)}}
out := Wants(m, n, running, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
}
if len(out.Record) != 0 {
t.Errorf("a user-list revision was asked: %+v", out.Record)
}
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
put := puts.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
for _, c := range []struct {
name string
step string
wanted map[string]mwc.Want
want *mwc.Want // nil: nothing written on ace
}{
{"waiting for the operator", puts.Person, wantedOld(), nil},
{"wanted", puts.Wanted, wantedOld(), &mwc.Want{Fp: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
{"holding, written already", puts.Holding, map[string]mwc.Want{"ace": PutWant(put)}, nil},
{"released: the rules again", puts.Released, map[string]mwc.Want{"ace": PutWant(put)}, &mwc.Want{Fp: fNew, Composed: dig(fNew)}},
{"not heard: a held want at an older fingerprint stands", "", map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
{"not heard: a held want the rules would rewrite stands", "", map[string]mwc.Want{"ace": PutWant(put)}, nil},
} {
p := put
p.Step = c.step
m.Puts = map[string]puts.Put{"ace": p}
if c.step == "" {
m.Puts = nil
}
out := Wants(m, n, c.wanted, nil)
got, wrote := out.Wants["ace"]
switch {
case c.want == nil && wrote:
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
case c.want != nil && (!wrote || !mwc.SameWant(normal(got), *c.want)):
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
}
}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: userList, Judgement: JudgedNeither, UserListOnly: true}, true
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
held := put
held.Step = puts.Holding
m.Puts = map[string]puts.Put{"ace": held}
if w := Wants(m, n, map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
stepsOfAPut(t)
}
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
// operator's yes, the bus's snapshot, the want, and the have.
func stepsOfAPut(t *testing.T) {
t.Helper()
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
base := puts.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, puts.Asked, "asked")
d := ComposedDigest{Digest: dig(fNew), Judgement: JudgedCannot, Words: "the have names no composition"}
running := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
bus := func() PutWorld {
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: builds.BuildDone, Newest: fNew,
Composed: &d, Granted: true, Have: &running, Now: now}
}
cases := []struct {
name string
world func() PutWorld
step string
says string
ask, snap bool
warranted bool
}{
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, puts.Refused, "not assigned", false, false, false},
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, puts.Build, "builds by itself", false, false, false},
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, puts.Refused, "holds no build", false, false, false},
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = builds.BuildFailed, "go vet"; return w }, puts.Refused, "go vet", false, false, false},
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, puts.Compose, "no composition of nats for novox", false, false, false},
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, puts.Compose, "does not name novox", false, false, false},
{"none within the bound: refused", func() PutWorld {
w := bus()
w.Composed = nil
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), puts.Compose, "waits")
return w
}, puts.Refused, "mesh-hand-compose", false, false, false},
{"the bus asks the operator", bus, puts.Person, "waits for the operator's yes", true, false, false},
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, puts.Refused, "answered no", false, false, false},
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, puts.Refused, "expired unanswered", false, false, false},
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, puts.Snapshot, "asked novox's backup holder", false, true, false},
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, puts.Snapshot, "waits for novox's backup holder", false, false, false},
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, puts.Refused, "no restic", false, false, false},
{"snapshotted: the want, warranted", func() PutWorld {
w := bus()
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
w.Put.Ask, _ = asks.RecordAskKey("novox", "nats", d.Digest)
return w
}, puts.Wanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
{"a user-list revision of the bus needs nobody", func() PutWorld {
w := bus()
ul := d
ul.UserListOnly = true
w.Composed, w.Have = &ul, &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, puts.Wanted, "held until released", false, false, false},
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, puts.Wanted, "names", false, false, false},
{"a recorded module judged to recreate asks", func() PutWorld {
w := bus()
w.Bus, w.Recorded = false, true
r := d
r.Judgement = JudgedRecreates
w.Composed = &r
return w
}, puts.Person, "waits for the operator's yes", true, false, false},
{"wanted, the node runs it: holding", func() PutWorld {
w := bus()
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, puts.Holding, "held there until released", false, false, false},
{"wanted without hold: done", func() PutWorld {
w := bus()
w.Put.Hold = false
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, puts.Done, "the rules write its want again", false, false, false},
{"wanted without hold, the node composes its own: done at the fingerprint", func() PutWorld {
w := bus()
w.Put.Hold = false
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, puts.Done, "the rules write its want again", false, false, false},
{"wanted with hold, the node composes its own: stays wanted", func() PutWorld {
w := bus()
w.Put = w.Put.Said(now, puts.Wanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, puts.Wanted, "wanted", false, false, false},
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, puts.Released, "released"); return w }, puts.Released, "released", false, false, false},
}
for _, c := range cases {
next := StepPut(c.world())
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
}
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
c.ask, c.snap, c.warranted)
}
if next.Ask && next.Put.Ask == "" {
t.Errorf("put, %s: an ask without its key", c.name)
}
}
}
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
m, n := ready()
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew), Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
t.Errorf("the answer was erased: %+v", w)
}
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
m.Self = func(string) bool { return true }
placed := map[string]mwc.Want{"ace": {Fp: fNew, Self: true, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
t.Errorf("the answer in a placed node's want was erased: %+v", w)
}
}
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
// wants move by the rules again.
func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"}
m.Released = true
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld})
if out.Held != nil {
t.Errorf("a released fingerprint was held again: %+v", out.Held)
}
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Rule != 0 {
t.Errorf("ace's want = %+v, want the held fingerprint written again", w)
}
}
// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes;
// answered "leave stopped" (or not answered) it stays.
func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}
m.Restored = func(node string) bool { return false }
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld {
t.Errorf("set back without the operator's restore: %+v", w)
}
m.Restored = func(node string) bool { return node == "ace" }
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
t.Errorf("after restore ace's want = %+v", w)
}
}
// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included;
// a composition with none (made by hand) falls back to what the manifest derives.
func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
m, n, wanted := movingTo(JudgedMovesData)
with := func(policy, from string) {
inner := m.Composed
m.Composed = func(node, fp string) (ComposedDigest, bool) {
d, ok := inner(node, fp)
d.Policy, d.PolicyFrom = policy, from
return d, ok
}
}
with(catalogue.PolicyRecord, "person") // a roll module the operator set to record
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew {
t.Errorf("the operator's record policy was not honoured: %+v", out)
}
recording(&m) // the manifest says record, the operator chose roll
with("roll", "person")
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
t.Errorf("the operator's roll policy was not honoured: %+v", out)
}
with("", "") // made by hand: the manifest decides
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
}
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
with(catalogue.PolicyRecord, "person")
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
}
}
// A node that runs the bus with no readable want gets its want by the rules, at the fingerprint it runs or at a new one alike: the
// bus has no bar of its own (ADR 0601 §1).
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
m, n, _ := movingTo(JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
}
}
// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written
// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it
// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against
// the composition the node runs, policy record, UserListOnly set.
func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) {
m, n, _ := movingTo(JudgedRecreates)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (ComposedDigest, bool) {
d := ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
Policy: catalogue.PolicyRecord, PolicyFrom: "the bus"}
if fp == fOld {
d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list
}
return d, true
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
for name, wanted := range map[string]map[string]mwc.Want{
"a want at the fingerprint it runs": {"ace": {Fp: fOld, Composed: dig(fOld)}},
"no readable want": nil,
} {
out := Wants(m, n, wanted, nil)
if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != userList {
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
}
if len(out.Record) != 0 {
t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record)
}
}
// the same flag on a module that is not the bus changes nothing
m.Manifest.Provides = nil
if out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
}
}
// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine
// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F
// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the
// declaration it kept); a new fingerprint still waits for its ask.
func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
for name, bus := range map[string]bool{"a record module": false, "the bus": true} {
// only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold
unchanged := func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedUnchanged}, true
}
m, n, _ := movingTo(JudgedCannot)
recording(&m)
m.Nodes = []string{"ace"}
if bus {
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
m.Composed = unchanged
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named
out := Wants(m, n, nil, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
}
if len(out.Record) != 0 {
t.Errorf("%s: asked at the switch: %+v", name, out.Record)
}
// a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge,
// still wait for the ask
for what, j := range map[string]Judgement{"differing": JudgedRecreates, "unjudged": JudgedCannot, "judged neither": JudgedNeither} {
m2, n2, _ := movingTo(j)
recording(&m2)
m2.Nodes = []string{"ace"}
if bus {
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m2, n2, nil, nil)
if j == JudgedNeither {
if out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("%s, %s: %+v", name, what, out)
}
continue
}
if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 {
t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record)
}
}
// a have that names a composition is judged as before
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 {
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
}
// a new fingerprint still asks (the bus is barred instead)
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
}
if !bus && len(out.Record) == 0 {
t.Errorf("%s: a new fingerprint was not asked", name)
}
}
}
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
t.Fatalf("held = %+v", out.Held)
}
}
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
everywhere := func() (Module, Nodes) {
m, n := ready()
m.Held = true
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return m, n
}
m, n := everywhere()
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
if !out.Lifted || out.Held != nil {
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
}
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.Fp != fNew || w.Rule != 0 {
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
}
}
m, n = everywhere()
delete(n.Haves["g14"], "mail")
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
t.Error("a node that does not run the module kept the hold")
}
for name, change := range map[string]func(*Module, *Nodes){
"set back": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
},
"failing": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
},
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
"not held": func(m *Module, _ *Nodes) { m.Held = false },
} {
m, n := everywhere()
change(&m, &n)
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
t.Errorf("%s: the hold was lifted", name)
}
}
}
// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld
// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld
// (health unknown) and holds no want for it. The canary g14 does not run F.
func x11() (Module, Nodes) {
m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"},
GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld },
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
n := Nodes{
Online: map[string]bool{"g14": true, "shanks": true},
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"g14": {"screen-lock": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown},
"xorg": {Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}},
"shanks": {"screen-lock": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown},
"xorg": {Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}},
},
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
}
return m, n
}
// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by
// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
if w := out.Wants["g14"]; w.Fp != fNew || out.Unreached["g14"] == "" {
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
}
if w := out.Wants["shanks"]; w.Fp != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != mwc.RuleCanary {
t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w)
}
if len(out.Neither) != 0 {
t.Errorf("left on neither list: %+v", out.Neither)
}
// with the provider healthy, only the canary holds shanks back: still at what it runs
m, n = x11()
n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary {
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
}
// a node already running F gets its want at F; a removed want is no want either
m, n = x11()
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, map[string]mwc.Want{"g14": {Removed: true}}, nil)
if w := out.Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Removed {
t.Errorf("g14 running F with a removed want = %+v, want F", w)
}
}
// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no
// want past the rules, and is named on neither list; the ordering rules still order every move from a want.
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
for what, change := range map[string]func(*Module, *Nodes){
"an old-form build": func(_ *Module, n *Nodes) {
n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
},
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
"no composition": func(m *Module, _ *Nodes) {
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedNeither}, fp == fNew
}
},
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if what == "an old-form build" {
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule != mwc.RuleCanary {
t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok)
}
continue
}
// 1960: no want can name what it runs, so its first want names F rather than none: never on neither list
if w := out.Wants["shanks"]; w.Fp != fNew {
t.Errorf("%s: shanks's want = %+v, want F", what, w)
}
if h, ok := out.Neither["shanks"]; ok {
t.Errorf("%s: shanks on neither list = %+v", what, h)
}
}
m, n := x11()
before := map[string]mwc.Want{"g14": {Fp: fOld, Composed: dig(fOld)}, "shanks": {Fp: fOld, Composed: dig(fOld)}}
out := Wants(m, n, before, nil)
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
// the canary g14 moves to F (its provider's health no rule); shanks waits for it
if w := out.Wants["shanks"]; w.Fp != "" && w.Fp != fOld {
t.Errorf("shanks's want moved from its want past the canary: %+v", w)
}
if w := out.Wants["g14"]; w.Fp != fNew {
t.Errorf("the canary g14's want = %+v, want F", w)
}
}
// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is
// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new
// form goes with the node's own composition and is never named.
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
if w := out.Wants["shanks"]; w.Fp != fNew {
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone {
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
}
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
// runs the module is on neither list, its removed want left as it is.
gone := mwc.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
for _, c := range []struct {
name string
before map[string]mwc.Want
want map[string]mwc.Want
}{
{"no key: nothing written", nil, map[string]mwc.Want{}},
{"a removed want: it says why", map[string]mwc.Want{"shanks": gone},
map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
{"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
map[string]mwc.Want{}},
{"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}},
} {
out := Wants(m, n, c.before, nil)
if len(out.Wants) != len(c.want) {
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
continue
}
for node, w := range c.want {
if !mwc.SameWant(normal(out.Wants[node]), w) {
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
}
}
}
m.FP = ""
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
}
}
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm := func(n Nodes, node string) {
n.Haves[node]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-1792260876956849831", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
}
providersHealthy := func(n Nodes) {
for _, node := range []string{"g14", "shanks"} {
n.Haves[node]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}
}
// the canary g14 runs F healthy: shanks's first want names F
m, n := x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither)
}
// the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule != mwc.RuleCanary {
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
}
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.Fp != fNew || out.Unreached["shanks"] == "" {
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
}
if _, on := out.Neither["shanks"]; on {
t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither)
}
// F not granted: nothing, even with the canary healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
m.Granted = false
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
t.Errorf("F not granted: shanks got a want: %+v", w)
}
// the old-form node is the canary itself: F once its providers run healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "g14")
if w := Wants(m, n, nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("old-form canary: g14's want = %+v, want F", w)
}
}
// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking.
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
// canary two seconds before the new build was healthy.
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
for _, phase := range []string{nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseApplying, nodeengineclient.PhaseChecking} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
}
}
for _, phase := range []string{nodeengineclient.PhaseRunning, nodeengineclient.PhaseFailed} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
}
}
}
// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
// fingerprint, or failed, still does.
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
setup := func(phase string, fp string, health string) (Module, Nodes) {
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"web": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}},
"g14": {"web": {Fp: fp, Phase: phase, Health: health}}}}
return m, n
}
for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} {
m, n := setup(phase, fOld, nodeengineclient.HealthUnknown)
if out := Wants(m, n, nil, nil); !out.Lifted {
t.Errorf("g14's have in phase %s kept the hold up", phase)
}
}
for what, h := range map[string]nodeengine.HaveByNodeModuleValue{
"running another fingerprint": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK},
"failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed},
} {
m, n := setup(h.Phase, h.Fp, h.Health)
if out := Wants(m, n, nil, nil); out.Lifted {
t.Errorf("g14 %s: the hold was lifted", what)
}
}
// a failed health on a removed have holds nothing
m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed)
m.Held = false
if out := Wants(m, n, nil, nil); out.Held != nil {
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
}
}
// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a
// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes.
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
m, n := x11()
recording(&m)
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true
}
out := Wants(m, n, map[string]mwc.Want{"shanks": {Removed: true}}, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
}
asked := false
for _, r := range out.Record {
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == JudgedRecreates)
}
if !asked {
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
}
}
// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the
// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on
// neither list: a held module is on the held list, whose hold is said and answered on its own.
func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
for what, change := range map[string]func(*Module, *Nodes){
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
"held at F, running an old-form build": func(m *Module, n *Nodes) {
m.Held, m.Kept = true, true
n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
},
"held by this pass": func(_ *Module, n *Nodes) {
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
},
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if len(out.Neither) != 0 {
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
}
if out.Rules["shanks"].Rule != mwc.RuleNotHeld {
t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"])
}
}
}