novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Assigner: the node-engine's have, phases, healths and candidate answers are the node-engine contract's and client's, the buckets it reads are its own package's, held equal to the manifest (B7f finding 3; ADR 0634 §1; issue 1966)

Browse Source 
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
jschoubben
2026-11-05 15:43:00 +01:00
parent be93d5f71b
commit cd94b19384
29 changed files with 814 additions and 615 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/composed.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
)
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
// `composed-<node>` (delivery.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// `composed-<node>` (buckets.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
// where they part. The assigner composes nothing and never writes these.
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+97 -95
View File
@@ -7,8 +7,10 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"io"
"maps"
"mesh-delivery-assigner/internal/buckets"
"os"
"path/filepath"
"slices"
@@ -232,14 +234,14 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
was := PutsReopenEvery
PutsReopenEvery = 0
defer func() { PutsReopenEvery = was }()
b := startBed(t, delivery.AssignerPutsBucket)
b := startBed(t, buckets.Puts)
done := b.start(t)
select {
case err := <-done:
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
t.Fatalf("without %s the assigner stopped: %v", buckets.Puts, err)
case <-time.After(2 * time.Second):
}
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: buckets.Puts, History: 5})
if err != nil {
t.Fatal(err)
}
@@ -251,7 +253,7 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
if _, err := b.buckets[buckets.Presence].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
t.Fatal(err)
}
if e, err := puts.Get(b.ctx, pkey); err == nil {
@@ -282,7 +284,7 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
t.Fatal(err)
}
if nw, err := delivery.DecodeNodeWant(pb.get(t, delivery.AssignerWantsBucket, "want.n1.node")); err != nil || !nw.Self {
if nw, err := delivery.DecodeNodeWant(pb.get(t, buckets.Wants, "want.n1.node")); err != nil || !nw.Self {
t.Fatalf("the placement state made after the start was not read: n1's own want = %+v, %v", nw, err)
}
}
@@ -322,16 +324,16 @@ func (b *bed) place(t *testing.T, node, module string) {
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
func (b *bed) compose(t *testing.T, node string, d byte) string {
t.Helper()
key, err := delivery.HaveNodeKey(node)
key, err := nodeengineclient.HaveNodeKey(node)
if err != nil {
t.Fatal(err)
}
digest := "sha256:" + strings.Repeat(string(d), 64)
v, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
v, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, key, v)
b.putJSON(t, buckets.Have, key, v)
return digest
}
@@ -344,7 +346,7 @@ func (b *bed) manifest(t *testing.T, module, body string) string {
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ManifestsBucket, key, value)
b.putJSON(t, buckets.Manifests, key, value)
return digest
}
@@ -384,25 +386,25 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.putJSON(t, buckets.Modules, "web", mod)
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
if err != nil {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
b.putJSON(t, buckets.Builds, bkey, built)
for _, fp := range []string{fNew, fOld} {
gkey, _ := delivery.PartGrantKey("web", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
b.putJSON(t, buckets.Grants, gkey, grant)
}
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
for _, node := range []string{"ace", "g14", "shanks"} {
b.compose(t, node, 'c')
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, node, on)
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, node+".web", have)
b.putJSON(t, buckets.Presence, node, on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: runs[node], Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, node+".web", have)
b.place(t, node, "web")
}
b.compose(t, "shanks", 'a')
@@ -434,7 +436,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
wkey, _ := delivery.WantKey("ace", "web")
want, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey))
want, err := delivery.DecodeWant(b.get(t, buckets.Wants, wkey))
if err != nil || want.FP != fNew {
t.Fatalf("the canary ace's want = %+v, %v", want, err)
}
@@ -455,17 +457,17 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if neither.Subject.Machine != "g14" || !strings.HasPrefix(neither.Subject.ID, "g14.web") || neither.Severity != occ.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
}
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
if keys, err := b.buckets[buckets.Differences].Keys(b.ctx); err == nil && len(keys) != 0 {
t.Errorf("differences were written: %v", keys)
}
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
if e, err := b.buckets[buckets.Wants].Get(b.ctx, "want.g14.web"); err == nil {
t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
}
// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
// there by itself: the condition is cleared.
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
healthyF, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthyF)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
t.Errorf("g14's want = %+v, want F, the node composing it", w)
}
@@ -495,7 +497,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
nkey, _ := delivery.WantNodeKey("ace")
var self bool
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
if e, err := b.buckets[buckets.Wants].Get(b.ctx, nkey); err == nil {
nw, err := delivery.DecodeNodeWant(e.Value())
self = err == nil && nw.Self
}
@@ -530,7 +532,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
fp, digest string
}{"db": {fpOf('2'), dbDigest}, "app": {fpOf('3'), appDigest}, "tool": {fpOf('4'), toolDigest}} {
mod, _ := delivery.EncodeModule(delivery.Module{FP2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, module, mod)
b.putJSON(t, buckets.Modules, module, mod)
fps := []string{row.fp}
if module == "tool" {
fps = append(fps, toolOld)
@@ -538,20 +540,20 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
for _, fp := range fps {
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest})
bkey, _ := delivery.BuilderBuildKey(module, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := delivery.PartGrantKey(module, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
b.putJSON(t, buckets.Grants, gkey, grant)
}
b.place(t, "ace", module)
}
runsOld, _ := delivery.EncodeHave(delivery.Have{FP: toolOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.tool", runsOld)
runsOld, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: toolOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.tool", runsOld)
if w := b.wantNow(t, "ace", "tool", "the build module's want at the second build", func(w delivery.Want) bool { return w.FP == fpOf('4') }); w.FP != fpOf('4') {
t.Errorf("tool's want = %+v", w)
}
b.wantNow(t, "ace", "db", "the provider's want at its fingerprint", func(w delivery.Want) bool { return w.FP == fpOf('2') })
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.ace.app"); err == nil {
if e, err := b.buckets[buckets.Wants].Get(b.ctx, "want.ace.app"); err == nil {
t.Errorf("app was wanted while its provision was asked: %s", e.Value())
}
asked(provisions.Provided, "")
@@ -589,7 +591,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
formsDigest := b.manifest(t, "forms", `{"module":"forms","version":"1"}`)
formsAt := func(fp string) {
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: formsDigest, Repository: "novox/forms", Commit: strings.Repeat("3", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "forms", mod)
b.putJSON(t, buckets.Modules, "forms", mod)
}
for _, fp := range []string{formsOld, formsNew} {
build := delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: formsDigest}
@@ -601,13 +603,13 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey("forms", fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := delivery.PartGrantKey("forms", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
b.putJSON(t, buckets.Grants, gkey, grant)
}
runsFormsOld, _ := delivery.EncodeHave(delivery.Have{FP: formsOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.forms", runsFormsOld)
runsFormsOld, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: formsOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.forms", runsFormsOld)
b.place(t, "ace", "forms")
formsAt(formsOld)
b.wantNow(t, "ace", "forms", "the want at what ace runs, a build in the first form", func(w delivery.Want) bool { return w.FP == formsOld })
@@ -680,21 +682,21 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
fNew := delivery.SourcePrefix2 + strings.Repeat("c", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.putJSON(t, buckets.Modules, "web", mod)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putJSON(t, buckets.Presence, "ace", on)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.place(t, "ace", "web")
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
if h, err := delivery.DecodeHeld(b.get(t, buckets.Wants, hkey)); err != nil || h.Node != "ace" ||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
t.Fatalf("held = %+v, %v", h, err)
}
akey, _ := delivery.HoldAskKey("web", fNew)
a, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
a, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
t.Fatalf("the ask = %+v, %v", a, err)
}
@@ -738,23 +740,23 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64)
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "mail", mod)
b.putJSON(t, buckets.Modules, "mail", mod)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.mail", failed)
b.putJSON(t, buckets.Presence, "ace", on)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.mail", failed)
wantKey, _ := delivery.WantKey("ace", "mail")
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
b.putJSON(t, buckets.Wants, wantKey, want)
b.place(t, "ace", "mail")
b.start(t)
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
a, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
a, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
t.Fatalf("the restore ask = %+v, %v", a, err)
}
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew {
if w, err := delivery.DecodeWant(b.get(t, buckets.Wants, wantKey)); err != nil || w.FP != fNew {
t.Errorf("the data module was set back: want = %+v, %v", w, err)
}
key := ""
@@ -780,7 +782,7 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
b := startBed(t, "")
const modules = 40
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.putJSON(t, buckets.Presence, "ace", on)
names := make([]string, modules)
for i := range names {
name := fmt.Sprintf("mod%02d", i)
@@ -788,21 +790,21 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name))
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
b.putJSON(t, buckets.Modules, name, mod)
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
if err != nil {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := delivery.PartGrantKey(name, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
b.putJSON(t, buckets.Grants, gkey, grant)
b.compose(t, "ace", 'c')
b.place(t, "ace", name)
}
count := func() int {
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
keys, err := b.buckets[buckets.Wants].Keys(b.ctx)
if err != nil {
return 0
}
@@ -818,7 +820,7 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
time.Sleep(time.Second) // and no pass of the second writes any again
for _, name := range names {
key, _ := delivery.WantKey("ace", name)
history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key)
history, err := b.buckets[buckets.Wants].History(b.ctx, key)
if err != nil {
t.Errorf("the want of %s was never written: %v", name, err)
continue
@@ -834,15 +836,15 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
t.Helper()
digest := b.manifest(t, name, manifest)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
b.putJSON(t, buckets.Modules, name, mod)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := delivery.PartGrantKey(name, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
b.putJSON(t, buckets.Grants, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.putJSON(t, buckets.Presence, "ace", on)
b.place(t, "ace", name)
}
@@ -853,7 +855,7 @@ func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AssignerWantsBucket, key, v)
b.putJSON(t, buckets.Wants, key, v)
}
// wantNow waits until the want of module on node satisfies ok and answers it.
@@ -862,7 +864,7 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
key, _ := delivery.WantKey(node, module)
var last delivery.Want
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil {
if e, err := b.buckets[buckets.Wants].Get(b.ctx, key); err == nil {
if w, err := delivery.DecodeWant(e.Value()); err == nil {
last = w
if ok(w) {
@@ -878,7 +880,7 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
func (b *bed) answer(t *testing.T, key, option string) {
t.Helper()
a, err := decodeAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
a, err := decodeAsk(key, b.get(t, buckets.Asks, key))
if err != nil {
t.Fatal(err)
}
@@ -892,7 +894,7 @@ func (b *bed) answerTo(t *testing.T, key, option, askID string) {
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AnswersBucket, key, v)
b.putJSON(t, buckets.Answers, key, v)
}
func (b *bed) hasKey(bucket, key string) bool {
@@ -908,10 +910,10 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.compose(t, "ace", 'a')
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld })
@@ -919,7 +921,7 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
t.Fatalf("the set-back want = %+v", w)
}
hold, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerAsksBucket, hold)
b.get(t, buckets.Asks, hold)
b.answer(t, hold, "release")
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
if w.Composed != "" || !w.Self || w.Rule != 0 {
@@ -941,7 +943,7 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
func (b *bed) askNow(t *testing.T, key string, asking int) decodedAsk {
t.Helper()
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil {
if e, err := b.buckets[buckets.Asks].Get(b.ctx, key); err == nil {
if a, err := decodeAsk(key, e.Value()); err == nil && a.Asking == asking {
return a
}
@@ -960,8 +962,8 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Self: true}) // a placed node's want names the build alone
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"})
b.putJSON(t, buckets.Have, "ace.web", held)
b.start(t)
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
a := b.askNow(t, akey, 0)
@@ -980,8 +982,8 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
t.Errorf("the want = %+v", w1)
}
// the node retries and the backup fails again: a new have value, asked again under the next number
again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, delivery.HaveBucket, "ace.web", again)
again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, buckets.Have, "ace.web", again)
a2 := b.askNow(t, akey, 2)
if a2.Ask.ID == a.Ask.ID {
t.Fatalf("asked again under the same id")
@@ -1003,18 +1005,18 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerWantsBucket, hkey)
b.get(t, delivery.AssignerAsksBucket, akey)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
b.get(t, buckets.Wants, hkey)
b.get(t, buckets.Asks, akey)
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
if b.hasKey(buckets.Wants, hkey) || b.hasKey(buckets.Asks, akey) {
continue
}
for k, c := range b.conditionsSaid(t) {
@@ -1026,7 +1028,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
}
}
}
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(buckets.Wants, hkey), b.hasKey(buckets.Asks, akey))
}
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
@@ -1036,19 +1038,19 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
_, stop := b.startStoppable(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
first, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
first, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the hold was not lifted")
}
@@ -1056,17 +1058,17 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
stop()
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putJSON(t, buckets.Have, "ace.web", failed)
b.start(t)
again, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
again, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil {
t.Fatal(err)
}
if again.Ask.ID == first.Ask.ID {
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
}
@@ -1135,11 +1137,11 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}})
b.putJSON(t, delivery.HaveBucket, node+".node", rec)
rec, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node}})
b.putJSON(t, buckets.Have, node+".node", rec)
}
b.putJSON(t, delivery.ModulesBucket, "web", []byte(`{"v":1}`))
b.putJSON(t, delivery.ModulesBucket, "shell", []byte(`{"v":1}`))
b.putJSON(t, buckets.Modules, "web", []byte(`{"v":1}`))
b.putJSON(t, buckets.Modules, "shell", []byte(`{"v":1}`))
// place writes a node's placement as another writer would, in its node and module keys (research 072 row 37), and
// answers its node key's revision; legacy writes it in the form before, placement.<node>, alone.
place := func(node string, legacy bool, modules ...string) uint64 {
@@ -1219,9 +1221,9 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
if err != nil {
t.Fatalf("no candidate %s: %v", key, err)
}
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}, Candidates: map[string]delivery.CandidateAnswer{
p.ID: {Proposal: p.ID, Revision: e.Revision(), Verdict: delivery.CandidateSame, Same: []string{"node"}}}})
b.putJSON(t, delivery.HaveBucket, node+".node", have)
have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node}, Candidates: map[string]nodeengine.CandidateAnswer{
p.ID: {Proposal: p.ID, Revision: int64(e.Revision()), Verdict: nodeengine.CandidateAnswerVerdictSame, Same: []string{"node"}}}})
b.putJSON(t, buckets.Have, node+".node", have)
}
}
@@ -1323,7 +1325,7 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
if ace, err := b.buckets[plc.PlacementBucket].Get(b.ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), "10.9.0.2") {
t.Errorf("the joining node's key = %v, %v", ace, err)
}
if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(plc.ProposalsBucket, joined.(proposed).Proposal.ID) {
if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(buckets.Asks, akey) || b.hasKey(plc.ProposalsBucket, joined.(proposed).Proposal.ID) {
t.Error("an ask was raised for the joining node, or its proposal stays open")
}
// Its enrolment ends before it confirms: a remove-node on the same warrant deletes its key at once, with no ask, and
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+46 -45
View File
@@ -12,6 +12,7 @@ import (
"fmt"
"io"
"log"
"mesh-delivery-assigner/internal/buckets"
"os"
"os/signal"
"slices"
@@ -58,24 +59,24 @@ func main() {
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
buckets.Grants,
buckets.Answers,
buckets.Builds,
buckets.Presence,
buckets.Have,
provisions.ProvisionsBucket,
delivery.SourceBucket,
delivery.MergesBucket,
buckets.Source,
buckets.Merges,
}
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
var readFamilies = []string{delivery.ComposedRead}
var readFamilies = []string{buckets.ComposedRead}
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
// (ADR 0567 §1), named after the assigner as every module state is.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, plc.PlacementBucket, plc.ProposalsBucket,
var ownBuckets = []string{buckets.Asks, buckets.Differences,
buckets.Puts, buckets.Wants, plc.PlacementBucket, plc.ProposalsBucket,
plc.CandidatesBucket, plc.PulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, plc.SeatHoldersBucket,
engineReadsBucket}
@@ -106,25 +107,25 @@ var keptStates = []announce.State{
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
{Name: health.FindingsState, Readable: true},
{Name: delivery.AssignerPoliciesState, Readable: true},
{Name: buckets.PoliciesState, Readable: true},
{Name: mma.StateSeatHolders, Readable: true},
{Name: views.MarksState, Readable: true}, // what its views family said each state up to (views.go)
{Name: views.StatusState, Readable: true}, // its first views of the indexer's states (views.go)
{Name: engineReadsState, Readable: true},
}
// buckets is every bucket the assigner opened, by name.
type buckets map[string]jetstream.KeyValue
// kvSet is every bucket the assigner opened, by name.
type kvSet map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
out := buckets{}
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (kvSet, error) {
out := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name)
if name == delivery.AssignerPutsBucket && err != nil {
if name == buckets.Puts && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
@@ -157,7 +158,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil {
if (name == buckets.Source || name == buckets.Merges) && err != nil {
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
// a record module is asked of a person, as before.
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
@@ -169,7 +170,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err)
continue
}
if name == delivery.PartGrantsBucket && err != nil {
if name == buckets.Grants && err != nil {
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
continue
@@ -187,10 +188,10 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
func startsWithout(name string) bool {
switch name {
case plc.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, engineReadsBucket:
buckets.Puts, buckets.Source, buckets.Merges, engineReadsBucket:
return true
}
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
return slices.Contains(seatBuckets, name) || name == buckets.Grants
}
// run is the program: it returns when ctx ends or the bus cannot be used.
@@ -241,7 +242,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
func findingsOf(ctx context.Context, kvs kvSet, logf func(string, ...any)) assign.Findings {
var f assign.Findings
if kv := kvs[assign.FindingsBucket]; kv != nil {
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
@@ -268,29 +269,29 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
buckets.Grants,
buckets.Answers,
buckets.Builds,
plc.PlacementBucket,
plc.ProposalsBucket,
plc.CandidatesBucket,
plc.PulsesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
delivery.AssignerDifferencesBucket,
delivery.AssignerWantsBucket,
buckets.Presence,
buckets.Have,
buckets.Asks,
buckets.Differences,
buckets.Wants,
assign.PoliciesBucket,
provisions.ProvisionsBucket,
plc.SeatHoldersBucket,
delivery.SourceBucket,
delivery.MergesBucket,
buckets.Source,
buckets.Merges,
engineReadsBucket,
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *indexerViews, logger *log.Logger) error {
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *indexerViews, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -317,7 +318,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
}
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
optional := name == delivery.AssignerPutsBucket
optional := name == buckets.Puts
go func() {
defer kw.Stop()
for {
@@ -357,10 +358,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
// stops the puts, never the wants.
if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil {
if kvs[buckets.Puts] != nil {
if err := watch(buckets.Puts); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
delete(kvs, buckets.Puts)
}
}
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
@@ -369,24 +370,24 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// further, without a restart.
var triedPuts time.Time
reopenPuts := func(p *passer) {
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
if kvs[buckets.Puts] != nil || time.Since(triedPuts) < PutsReopenEvery {
return
}
triedPuts = time.Now()
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
kv, err := js.KeyValue(ctx, buckets.Puts)
if err != nil {
return
}
kvs[delivery.AssignerPutsBucket] = kv
if err := watch(delivery.AssignerPutsBucket); err != nil {
kvs[buckets.Puts] = kv
if err := watch(buckets.Puts); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
delete(kvs, buckets.Puts)
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
logger.Printf("the bucket %s is open: puts are taken further", buckets.Puts)
}
started := 0
for _, name := range watched {
@@ -395,7 +396,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
started++
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket || name == delivery.SourceBucket {
if name == buckets.Have || name == buckets.Source {
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
opts = append(opts, jetstream.IncludeHistory())
}
@@ -427,7 +428,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, plc.SeatHoldersBucket} {
for _, name := range []string{buckets.Grants, assign.PoliciesBucket, provisions.ProvisionsBucket, plc.SeatHoldersBucket} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
@@ -459,7 +460,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if js != nil {
hearReadsChanged(ctx, js, hub, reads, findings, logger.Printf)
}
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
if bus != nil && kvs[buckets.Puts] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
if bus != nil {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go
+9 -9
View File
@@ -2,6 +2,7 @@ package main
import (
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
plc "mesh-delivery-assigner/internal/placement"
"context"
@@ -23,7 +24,6 @@ import (
mmac "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -87,12 +87,12 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
starts bool
said string
}{
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
{buckets.Puts, jetstream.ErrBucketNotFound, true, "no put is taken further"},
{buckets.Puts, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{buckets.Puts, context.DeadlineExceeded, true, "no put is taken further"},
{buckets.Wants, errors.New("nats: permissions violation"), false, ""},
// Issue 774: the grants part's grants not there do not stop the start.
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
{buckets.Grants, jetstream.ErrBucketNotFound, true, "no grant is read"},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
@@ -108,7 +108,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := buckets{}
kvs := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
@@ -120,8 +120,8 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
kv := kvs[plc.PlacementBucket].(*watchKV)
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
kvs[buckets.Puts].(*watchKV).closed = true
kvs[buckets.Wants].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+41 -39
View File
@@ -5,7 +5,9 @@ import (
"context"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
"strings"
"time"
@@ -24,7 +26,7 @@ import (
// passer makes the assigner's passes: from what the world holds now, it writes each key whose value differs from the
// one written, and only those (the Same functions).
type passer struct {
kvs buckets
kvs kvSet
w *world
logf func(string, ...any)
sdk string
@@ -124,7 +126,7 @@ func (p *passer) pass(ctx context.Context) {
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
func (p *passer) nodeWants(ctx context.Context) {
written := p.w.read(delivery.AssignerWantsBucket)
written := p.w.read(buckets.Wants)
placed := p.w.placedNodes()
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
for node := range placed {
@@ -136,7 +138,7 @@ func (p *passer) nodeWants(ctx context.Context) {
continue
}
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
}
}
@@ -144,7 +146,7 @@ func (p *passer) nodeWants(ctx context.Context) {
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
func (p *passer) wants(ctx context.Context) {
get := p.w.getter(delivery.ManifestsBucket) // the indexer's manifests, as its first view and views family say them
get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them
assigned := p.w.assigned()
trunk := p.w.trunk()
manifests := map[string]catalogue.Manifest{}
@@ -202,7 +204,7 @@ func (p *passer) wants(ctx context.Context) {
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp),
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
Composed: func(node, fp string) (assign.ComposedDigest, bool) {
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
},
GrantedAt: func(fp string) bool {
@@ -224,7 +226,7 @@ func (p *passer) wants(ctx context.Context) {
hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant)
if raw := p.w.value(grants, grant); raw != nil {
if g, err := delivery.DecodeGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
@@ -264,7 +266,7 @@ func (p *passer) wants(ctx context.Context) {
key, err := delivery.HeldKey(name, fp)
if err == nil {
held := *out.Held
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
}
}
if out.Lifted {
@@ -287,7 +289,7 @@ func (p *passer) wants(ctx context.Context) {
continue
}
want := want
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
}
@@ -546,7 +548,7 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
// node (wants are only compared, ADR 0429 §7), and the options say so.
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
key, err := delivery.HoldAskKey(module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
if err != nil || p.w.has(buckets.Asks, key) {
return
}
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
@@ -555,7 +557,7 @@ func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery
asking := int(now.Unix())
h := p.heldHold(module, fp, fresh)
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
@@ -583,13 +585,13 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
p.said[c.Key+".lifted"] = true
delete(p.said, c.Key+".raised") // a hold written again is said again
}
p.remove(ctx, delivery.AssignerWantsBucket, held)
p.remove(ctx, delivery.AssignerAsksBucket, ask)
p.remove(ctx, buckets.Wants, held)
p.remove(ctx, buckets.Asks, ask)
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
// set back from it (issue 1906, note 28).
for key := range p.w.read(delivery.AssignerAsksBucket) {
for key := range p.w.read(buckets.Asks) {
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
p.remove(ctx, delivery.AssignerAsksBucket, key)
p.remove(ctx, buckets.Asks, key)
}
}
}
@@ -616,8 +618,8 @@ func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
// A node that composes its own asks about its own composition, which its have names as the one it waits on (yeses): that
// ask stands while the have waits on it, whatever the controller composes (issue 774, 2026-10-31: novox's asks of
// postgres, keycloak and mongodb were taken away and made again under a new id every pass, so no answer could count).
if hkey, err := delivery.HaveKey(k.Node, k.Module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, hkey)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits == k.Composed {
if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil {
if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, hkey)); err == nil && h.Phase == nodeengineclient.PhaseWaitingYes && h.Waits == k.Composed {
return false
}
}
@@ -647,13 +649,13 @@ func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
}
add(trunkFP)
if key, err := delivery.WantKey(node, module); err == nil {
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
if w, err := delivery.DecodeWant(p.w.value(buckets.Wants, key)); err == nil {
add(w.FP)
}
}
if key, err := delivery.HaveKey(node, module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
add(h.FP)
if key, err := nodeengineclient.HaveKey(node, module); err == nil {
if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, key)); err == nil {
add(h.Fp)
}
}
return out
@@ -666,7 +668,7 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module
if len(trunk) == 0 {
return
}
for key := range p.w.read(delivery.AssignerAsksBucket) {
for key := range p.w.read(buckets.Asks) {
if p.staleAsk(key, trunk) {
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
@@ -676,7 +678,7 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module
continue
}
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
p.remove(ctx, delivery.AssignerAsksBucket, key)
p.remove(ctx, buckets.Asks, key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
// raised again if the trunk returns to the fingerprint.
@@ -709,7 +711,7 @@ func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held
if err != nil {
return delivery.Held{}
}
h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key])
h, _ := delivery.DecodeHeld(p.w.read(buckets.Wants)[key])
return h
}
@@ -733,12 +735,12 @@ func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *de
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
key, err := delivery.RestoreAskKey(node, module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
if err != nil || p.w.has(buckets.Asks, key) {
return
}
words := p.heldHold(module, fp, nil).Words
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
@@ -750,7 +752,7 @@ func (p *passer) answeredWith(key, option string) bool {
if !ok || a.Option != option {
return false
}
_, ask, err := assign.DecodeAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks, key))
if err != nil {
return false
}
@@ -776,11 +778,11 @@ func (p *passer) clearHeld(module, fp string) bool {
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
key, err := delivery.RecordAskKey(need.Node, module, need.Digest)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
if err != nil || p.w.has(buckets.Asks, key) {
return
}
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
@@ -794,13 +796,13 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
// in a new have value after the answer was written.
func (p *passer) backups(ctx context.Context) {
wanted := p.w.wantsOf()
for key, value := range p.w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
for key, value := range p.w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := delivery.DecodeHave(value)
if err != nil || have.Phase != delivery.PhaseHeld {
have, err := nodeengineclient.DecodeHave(value)
if err != nil || have.Phase != nodeengineclient.PhaseHeld {
continue
}
want, ok := wanted[module][node]
@@ -811,7 +813,7 @@ func (p *passer) backups(ctx context.Context) {
if err != nil {
continue
}
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks)[akey])
if there != nil || assign.BackupAskDigest(current) != want.Composed {
after := -1
if there == nil {
@@ -826,12 +828,12 @@ func (p *passer) backups(ctx context.Context) {
continue // not answered yet: the ask stands
}
words := assign.BackupAnswerWords(answer.Option, current.Asking)
haveChanges := p.w.changed(delivery.HaveBucket, key)
haveChanges := p.w.changed(buckets.Have, key)
if want.Backup != answer.Option || want.Words != words {
want.Backup, want.Words = answer.Option, words
if wkey, err := delivery.WantKey(node, module); err == nil {
w := want
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
p.put(ctx, buckets.Wants, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
p.answeredAt[currentAsk.ID] = haveChanges
}
continue
@@ -851,7 +853,7 @@ func (p *passer) backups(ctx context.Context) {
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
c := assign.BackupCondition(node, module, want.FP, words, true)
if p.say == nil || p.said[c.Key] {
return
@@ -889,15 +891,15 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo
if had && was.FP == w.FP {
yes, release = was.Yes, was.Release
}
key, err := delivery.HaveKey(node, module)
key, err := nodeengineclient.HaveKey(node, module)
if err != nil {
continue
}
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, key)); err == nil && h.Phase == nodeengineclient.PhaseWaitingYes && h.Waits != yes {
switch {
case m.Allowed != nil && m.Allowed(node, h.Waits):
yes, release = h.Waits, ""
case h.BuildAlone && h.FP != w.FP && m.Release != nil:
case h.BuildAlone && h.Fp != w.FP && m.Release != nil:
// A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1),
// named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked.
if r, ok := m.Release(w.FP); ok {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go
+88 -86
View File
@@ -6,6 +6,8 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"mesh-delivery-assigner/internal/buckets"
"slices"
"strings"
"testing"
@@ -53,7 +55,7 @@ func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt
}
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
func newPasser(kvs buckets, at time.Time) *passer {
func newPasser(kvs kvSet, at time.Time) *passer {
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
now: func() time.Time { return at }}
}
@@ -78,12 +80,12 @@ func placeOn(t *testing.T, w *world, node string, modules ...string) {
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
func composes(t *testing.T, w *world, node, digest string) {
t.Helper()
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
raw, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
w.put(delivery.HaveBucket, key, raw)
key, _ := nodeengineclient.HaveNodeKey(node)
w.put(buckets.Have, key, raw)
}
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
@@ -93,7 +95,7 @@ var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
// an id it keeps).
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
kvs := kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}
key, _ := delivery.HoldAskKey("web", fpTest)
held := &delivery.Held{Node: "ace", Words: "health failed"}
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
@@ -129,19 +131,19 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
}
}
key, _ := delivery.HoldAskKey("web", fpTest)
p := newPasser(buckets{}, time.Now())
p := newPasser(kvSet{}, time.Now())
put := func(asking int) string {
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
v, err := assign.EncodeAsk(key, a)
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.AssignerAsksBucket, key, v)
p.w.put(buckets.Asks, key, v)
return mustAskID(a)
}
first := put(100)
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
p.w.put(delivery.AnswersBucket, key, v)
p.w.put(buckets.Answers, key, v)
if !p.answeredWith(key, "release") {
t.Fatal("the answer to the ask that stands is not taken")
}
@@ -152,7 +154,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
if p.answeredWith(key, "release") {
t.Error("the answer to an earlier hold carried over to a later one")
}
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
p.w.put(buckets.Asks, key, nil) // the ask taken away
if p.answeredWith(key, "release") {
t.Error("an answer to an ask taken away was taken")
}
@@ -162,12 +164,12 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
t.Run("the rules read what each node's engine reads", func(t *testing.T) {
w := newWorld()
for node, reads := range map[string][]string{"ace": {"word:module", "word:tool-runner"}, "g14": nil} {
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ManifestReads: reads}})
raw, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node, ManifestReads: reads}})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
w.put(delivery.HaveBucket, key, raw)
key, _ := nodeengineclient.HaveNodeKey(node)
w.put(buckets.Have, key, raw)
}
n := w.nodes(nil, nil)
if !slices.Equal(n.Reads["ace"], []string{"word:module", "word:tool-runner"}) || len(n.Reads["g14"]) != 0 || len(n.Reads["novox"]) != 0 {
@@ -186,7 +188,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
}
p := &passer{w: w, reads: &enginereads.Keeper{Call: call, Fallback: w.recordReads, Sync: true}}
on, _ := delivery.EncodePresence(delivery.Presence{Online: false})
w.put(delivery.PresenceBucket, "ace", on)
w.put(buckets.Presence, "ace", on)
n = w.nodes(nil, nil)
p.engineReads(n)
if !slices.Equal(n.Reads["ace"], []string{"form:new"}) || len(n.Reads["g14"]) != 0 || asked["ace"] != 1 {
@@ -197,7 +199,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
}
p.engineReads(w.nodes(nil, nil))
on, _ = delivery.EncodePresence(delivery.Presence{Online: true})
w.put(delivery.PresenceBucket, "ace", on)
w.put(buckets.Presence, "ace", on)
p.engineReads(w.nodes(nil, nil))
if asked["ace"] != 2 {
t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"])
@@ -317,7 +319,7 @@ func putOfTheBus(t *testing.T) {
digest := "sha256:" + strings.Repeat("d", 64)
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
puts, asksKV, wantsKV := mem(), mem(), mem()
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
kvs := kvSet{buckets.Puts: puts, buckets.Asks: asksKV, buckets.Wants: wantsKV}
p := newPasser(kvs, now)
bus := &fakeBus{calls: make(chan string, 8)}
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
@@ -339,14 +341,14 @@ func putOfTheBus(t *testing.T) {
placeOn(t, p.w, "novox", "nats")
mdigest := strings.Repeat("e", 64)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
world(delivery.ModulesBucket, "nats", mod)
world(buckets.Modules, "nats", mod)
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
bkey, _ := delivery.BuilderBuildKey("nats", fp)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
world(delivery.BuilderBuildsBucket, bkey, built)
hkey, _ := delivery.HaveKey("novox", "nats")
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, before)
world(buckets.Builds, bkey, built)
hkey, _ := nodeengineclient.HaveKey("novox", "nats")
before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
world(buckets.Have, hkey, before)
builds := mem()
builds.values[bkey] = built
@@ -376,7 +378,7 @@ func putOfTheBus(t *testing.T) {
pkey, _ := delivery.PutKey("novox", "nats")
step := func() delivery.Put {
t.Helper()
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
world(buckets.Puts, pkey, puts.values[pkey])
p.pass(ctx)
got, err := delivery.DecodePut(puts.values[pkey])
if err != nil {
@@ -395,7 +397,7 @@ func putOfTheBus(t *testing.T) {
composes(t, p.w, "novox", digest)
gkey, _ := delivery.PartGrantKey("nats", fp)
g, _ := delivery.EncodeGrant(delivery.Grant{})
world(delivery.PartGrantsBucket, gkey, g)
world(buckets.Grants, gkey, g)
got := step()
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
if got.Step != delivery.PutPerson || got.Ask != akey {
@@ -408,9 +410,9 @@ func putOfTheBus(t *testing.T) {
if _, there := wantsKV.values["want.novox.nats"]; there {
t.Fatal("the bus's want was written before the operator's warrant")
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
world(buckets.Asks, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes)
world(buckets.Answers, akey, yes)
if got := step(); got.Step != delivery.PutSnapshot {
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
}
@@ -423,9 +425,9 @@ func putOfTheBus(t *testing.T) {
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
t.Errorf("the warrant was recorded as %q", call)
}
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, running)
world(buckets.Wants, "want.novox.nats", wantsKV.values["want.novox.nats"])
running, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
world(buckets.Have, hkey, running)
if got := step(); got.Step != delivery.PutHolding {
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
}
@@ -444,7 +446,7 @@ func putOfTheBus(t *testing.T) {
}
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
world(delivery.HaveBucket, hkey, before)
world(buckets.Have, hkey, before)
s.now = func() time.Time { return now.Add(time.Minute) }
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
@@ -454,9 +456,9 @@ func putOfTheBus(t *testing.T) {
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
world(buckets.Asks, akey, asksKV.values[akey])
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes2)
world(buckets.Answers, akey, yes2)
if got := step(); got.Step != delivery.PutSnapshot {
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
}
@@ -464,7 +466,7 @@ func putOfTheBus(t *testing.T) {
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
wantsKV.values["want.novox.nats"] = oldWant
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
world(buckets.Wants, "want.novox.nats", oldWant)
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
@@ -483,7 +485,7 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// The verb edges a want waits on come from the graph and what runs (ADR 0614 §2, §3): folded here to hold the count.
aWantWaitsOnTheVerbEdgesTheGraphKeeps(t)
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []oc.Condition
failing := true
p.say = func(c oc.Condition) error {
@@ -561,7 +563,7 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
@@ -618,7 +620,7 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
}
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"rofi": {Fp: fpTest, Health: nodeengineclient.HealthFailed}}}}
for _, c := range []struct {
name string
known assign.Nodes
@@ -628,7 +630,7 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
} {
conds = nil
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
fresh := newPasser(kvSet{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
fresh.logf = func(string, ...any) {}
fresh.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
@@ -650,19 +652,19 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now())
var said []oc.Condition
p.say = func(c oc.Condition) error { said = append(said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
p.w.put(buckets.Modules, "web", mod)
placeOn(t, p.w, "g14", "web")
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
p.w.put(delivery.HaveBucket, "g14.web", have)
p.w.put(buckets.Presence, "g14", on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, "g14.web", have)
key, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
p.w.put(delivery.AssignerWantsBucket, key, want)
p.w.put(buckets.Wants, key, want)
p.wants(context.Background())
cleared := map[string]bool{}
for _, c := range said {
@@ -688,19 +690,19 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
// raised (the first pass after a start), each once.
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
setup := func() (*passer, *[]oc.Condition) {
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
p := newPasser(kvSet{buckets.Wants: &memKV{values: map[string][]byte{}}, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now())
said := &[]oc.Condition{}
p.say = func(c oc.Condition) error { *said = append(*said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
p.w.put(buckets.Modules, "systemd-resolved", mod)
placeOn(t, p.w, "shanks", "systemd-resolved")
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "shanks", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
p.w.put(buckets.Presence, "shanks", on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed})
p.w.put(buckets.Have, "shanks.systemd-resolved", have)
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
p.w.put(delivery.AssignerWantsBucket, hkey, held)
p.w.put(buckets.Wants, hkey, held)
return p, said
}
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
@@ -767,7 +769,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
} {
t.Run(c.name, func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
var said []oc.Condition
p.say = func(c oc.Condition) error { said = append(said, c); return nil }
for _, key := range all {
@@ -788,7 +790,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Fatal(err)
}
asksKV.values[key] = v
p.w.put(delivery.AssignerAsksBucket, key, v)
p.w.put(buckets.Asks, key, v)
}
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
if !c.noDigit {
@@ -796,7 +798,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
}
heldOld, _ := delivery.HeldKey("web", fpOld)
wantsKV.values[heldOld] = []byte("x")
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
p.w.put(buckets.Wants, heldOld, []byte("x"))
firstAsk, _ := decodeAsk(holdOld, asksKV.values[holdOld])
p.staleAsks(context.Background(), c.trunk)
// the held record and condition of a fingerprint whose hold ask went are taken away with it
@@ -831,7 +833,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
var said []oc.Condition
fail := true
p.say = func(c oc.Condition) error {
@@ -848,7 +850,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Fatal(err)
}
asksKV.values[holdOld] = v
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
p.w.put(buckets.Asks, holdOld, v)
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
@@ -861,12 +863,12 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
asksKV.values[key] = []byte("x")
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
p.w.put(buckets.Asks, key, []byte("x"))
}
p.lift(context.Background(), "web", fpOld)
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
@@ -920,31 +922,31 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
{"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true},
} {
asksKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now())
p.logf = t.Logf
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FPSrc: c.fpSrc, Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
p.w.put(buckets.Modules, "web", mod)
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/web", Heads: map[string]string{"novox/web": commit}})
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, src, 7, time.Now())
p.w.putEntry(buckets.Source, delivery.SourceMainKey, src, 7, time.Now())
later, _ := delivery.EncodeSource(delivery.Source{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}})
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
p.w.putEntry(buckets.Source, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
if c.mergedAt != "" {
merge, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)}, State: delivery.MergeMerged, Commit: c.mergedAt}}})
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.MergesBucket, "feat/web", merge)
p.w.put(buckets.Merges, "feat/web", merge)
other, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40)}, State: delivery.MergeMerged, Commit: commit}}})
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.MergesBucket, "feat/other", other)
p.w.put(buckets.Merges, "feat/other", other)
}
have, _ := delivery.EncodeHave(delivery.Have{FP: c.at, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
p.w.put(delivery.HaveBucket, "g14.web", have)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: c.at, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK,
Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
p.w.put(buckets.Have, "g14.web", have)
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true },
Release: func(fp string) (string, bool) { return p.w.release("web", fp) }}
out := assign.Outcome{Wants: map[string]delivery.Want{"g14": {FP: fpTest, Self: true}}}
@@ -963,7 +965,7 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
// module in an assignment only is no module of its node, and before any placement is read nothing is placed.
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
if got := p.w.assigned(); len(got) != 0 {
t.Fatalf("before any placement: assigned %v", got)
@@ -982,25 +984,25 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
put(buckets.Modules, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
placeOn(t, p.w, "g14", "web")
pl, _ := mmac.EncodePlacement(mma.Placement{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
put(plc.PlacementBucket, "placement.g14", pl)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
put(buckets.Presence, "g14", on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
put(buckets.Have, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
put(buckets.Wants, wkey, want)
sync := func() {
for k, v := range wantsKV.values {
put(delivery.AssignerWantsBucket, k, v)
put(buckets.Wants, k, v)
}
}
@@ -1016,9 +1018,9 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
}
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
put(delivery.HaveBucket, "g14.web", waits)
waits, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK,
Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
put(buckets.Have, "g14.web", waits)
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
@@ -1039,7 +1041,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
t.Fatalf("a pass that changed nothing put %d wants and %d asks", wantsKV.puts-wantPuts, asksKV.puts-askPuts)
}
// Whichever rule asks it, a want put with the value the bucket holds reaches no bucket: each put is a change its node hears.
p.put(context.Background(), delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
p.put(context.Background(), buckets.Wants, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
if wantsKV.puts != wantPuts {
t.Fatalf("a want put unchanged reached the bucket")
}
@@ -1047,17 +1049,17 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
if err != nil {
t.Fatal(err)
}
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
put(buckets.Asks, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
put(buckets.Answers, akey, yes)
p.wants(context.Background())
sync()
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
}
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", ran)
ran, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: waiting, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
put(buckets.Have, "g14.web", ran)
p.wants(context.Background())
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
t.Errorf("a pass after the move dropped the yes: %+v", got)
@@ -1068,7 +1070,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
// cleared once, and said again at a later pass while the findings cannot be read.
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
bus := &fakeBus{calls: make(chan string, 8)}
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
var said []string
@@ -1103,7 +1105,7 @@ func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
// module serves; a verb is served once a running healthy build's manifest (the builder's build, the indexer's manifest)
// serves it, and a build whose manifest cannot be read counts what the graph says its module serves. No graph read: no wait.
func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
p := newPasser(buckets{}, time.Now())
p := newPasser(kvSet{}, time.Now())
fp := func(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
if p.served(context.Background(), nil, graph.Graph{}, assign.Nodes{}) != nil {
t.Error("with no graph read, a want waits on a verb")
@@ -1135,10 +1137,10 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
}
build, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
key, _ := delivery.BuilderBuildKey("server", fp("f"))
p.w.put(delivery.BuilderBuildsBucket, key, build)
running := delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
at := func(f string) delivery.Have { h := running; h.FP = f; return h }
served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]delivery.Have{
p.w.put(buckets.Builds, key, build)
running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h }
served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"g14": {"server": at(fp("f")), "late": at(fp("2"))}}})
for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} {
if served(v) != want {
@@ -1150,7 +1152,7 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}}
for _, online := range []bool{false, true} {
offering := p.offering(context.Background(), get, bus, assign.Nodes{Online: map[string]bool{"n2": online},
Haves: map[string]map[string]delivery.Have{"n2": {"bus": at(fp("2"))}}})
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"n2": {"bus": at(fp("2"))}}})
if offering(graph.BusState) != online {
t.Errorf("with the bus module's node online %v, bus-state offered %v", online, offering(graph.BusState))
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go
+17 -15
View File
@@ -5,6 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"mesh-delivery-assigner/internal/buckets"
"sort"
"strings"
"sync"
@@ -70,7 +72,7 @@ func identity(key string, p delivery.Put) string {
// putsByModule is every put the assigner holds, by module then node.
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
out := map[string]map[string]delivery.Put{}
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
for key, value := range p.w.read(buckets.Puts) {
node, module, err := delivery.ParsePutKey(key)
if err != nil {
continue
@@ -99,7 +101,7 @@ func (p *passer) puts(ctx context.Context) {
}
assigned := p.w.assigned()
trunk := p.w.trunk()
get := p.w.getter(delivery.ManifestsBucket) // the indexer's manifests, as its first view and views family say them
get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them
wanted := p.w.wantsOf()
modules := make([]string, 0, len(all))
for m := range all {
@@ -150,7 +152,7 @@ func (p *passer) puts(ctx context.Context) {
if wkey, err := delivery.WantKey(node, module); err == nil {
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
p.put(ctx, buckets.Wants, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
if next.Warranted {
@@ -175,7 +177,7 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
}
}
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
if raw := p.w.value(buckets.Builds, bkey); raw != nil {
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
@@ -195,7 +197,7 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
w.Recorded = policy == catalogue.PolicyRecord
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
if a, ask, err := assign.DecodeAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
if a, ask, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks, akey)); err == nil && a.Asking == assign.PutAsking(put) {
switch {
case p.answeredWith(akey, assign.OptionYes):
w.Answer = assign.OptionYes
@@ -208,9 +210,9 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
}
}
}
if hkey, err := delivery.HaveKey(node, module); err == nil {
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
if h, err := delivery.DecodeHave(raw); err == nil {
if hkey, err := nodeengineclient.HaveKey(node, module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
w.Have = &h
}
}
@@ -221,8 +223,8 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, bool) {
var out *assign.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
out = &d
}
@@ -243,13 +245,13 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
// raises it anew under its own id.
akey := next.Put.Ask
current, _, err := assign.DecodeAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
current, _, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks, akey))
if err != nil || current.Asking != assign.PutAsking(next.Put) {
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
if err != nil {
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
} else {
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.put(ctx, buckets.Asks, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
}
}
}
@@ -260,7 +262,7 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
// performs checks that the act about to be performed is the one the yes of the ask at key bound (occ.Performs).
func (p *passer) performs(key string, act occ.Act) error {
_, ask, err := assign.DecodeAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks, key))
if err != nil {
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
}
@@ -280,7 +282,7 @@ func samePut(a, b delivery.Put) bool {
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
// released it since; and says it as the event put. It says whether it wrote.
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
kv := p.kvs[delivery.AssignerPutsBucket]
kv := p.kvs[buckets.Puts]
if kv == nil {
return false
}
@@ -304,7 +306,7 @@ func (p *passer) writePut(ctx context.Context, key string, read, next delivery.P
}
return false
}
p.w.put(delivery.AssignerPutsBucket, key, data)
p.w.put(buckets.Puts, key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
p.acts.say(key, next)
return true
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go
+13 -10
View File
@@ -5,7 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"maps"
"mesh-delivery-assigner/internal/buckets"
"reflect"
"slices"
"sort"
@@ -166,14 +169,14 @@ func (p *passer) candidatesOf(id string) (map[string]mma.CandidatesByProposalNod
}
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
out := map[string]*delivery.CandidateAnswer{}
for key, value := range p.w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
func (p *passer) answersTo(id string) map[string]*nodeengine.CandidateAnswer {
out := map[string]*nodeengine.CandidateAnswer{}
for key, value := range p.w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
h, err := delivery.DecodeHaveNode(value)
h, err := nodeengineclient.DecodeHaveNode(value)
if err != nil {
continue
}
@@ -251,7 +254,7 @@ func (p *passer) ask(ctx context.Context, prop mma.Proposal, akey, brief string)
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
return prop, false
}
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.put(ctx, buckets.Asks, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
if pk, err := plc.PulseAskKey(prop.ID); err == nil {
p.pulse(ctx, pk, assign.ExpiresAt(placed).Sub(p.clock()))
}
@@ -449,7 +452,7 @@ func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, cu
// nodes with a record.
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
held := map[string]bool{}
for key := range p.w.read(delivery.ModulesBucket) {
for key := range p.w.read(buckets.Modules) {
if name, err := delivery.ParseModulesKey(key); err == nil {
held[name] = true
}
@@ -623,9 +626,9 @@ func (p *passer) sayChanged(prop mma.Proposal, answer delivery.Answer, changes [
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
// whole with why it ended.
func (p *passer) endProposal(ctx context.Context, prop mma.Proposal, why string) {
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
p.w.put(delivery.AssignerAsksBucket, akey, nil)
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(buckets.Asks, akey) {
if err := p.kvs[buckets.Asks].Delete(ctx, akey); err == nil {
p.w.put(buckets.Asks, akey, nil)
}
}
p.deleteCandidates(ctx, prop.ID)
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go
+11 -9
View File
@@ -4,6 +4,8 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"slices"
"sort"
"strings"
@@ -111,12 +113,12 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
}
recorded := map[string]bool{}
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Record != nil {
recorded[node] = true
}
}
@@ -404,9 +406,9 @@ func (s *seatTools) doPlacement(args map[string]any) (any, error) {
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
// every node's answer now.
type openProposal struct {
Proposal mma.Proposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
Proposal mma.Proposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]nodeengine.CandidateAnswer `json:"answers,omitempty"`
}
// doProposals is the verb proposals (§10).
@@ -423,9 +425,9 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
if err != nil {
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
}
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
@@ -433,13 +435,13 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
if err != nil || !found {
continue
}
h, err := delivery.DecodeHaveNode(raw)
h, err := nodeengineclient.DecodeHaveNode(raw)
if err != nil {
continue
}
for id, a := range h.Candidates {
if answers[id] == nil {
answers[id] = map[string]delivery.CandidateAnswer{}
answers[id] = map[string]nodeengine.CandidateAnswer{}
}
answers[id][node] = a
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go
+24 -20
View File
@@ -5,6 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/buckets"
"os"
"reflect"
"slices"
@@ -42,7 +46,7 @@ func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := mma.Placement{V: delivery.Version, Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}
anchor := mma.Placement{V: keys.Version, Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}
st := memStore{}
for _, state := range []string{mma.StatePlacement, mma.StateProposals, mma.StateCandidates, modulesState, haveState} {
st[state] = &memKV{values: map[string][]byte{}}
@@ -51,7 +55,7 @@ func TestTheSeatsVerbs(t *testing.T) {
st[modulesState].values[m] = []byte("{}")
}
for _, n := range []string{"anchor", "laptop"} {
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
raw, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: n}})
st[haveState].values[n+".node"] = raw
}
var asked []string
@@ -164,7 +168,7 @@ func TestTheSeatsVerbs(t *testing.T) {
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
// of a node running modules, or with another node's warrant, it is refused.
joiner, _ := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: "joiner", Decided: mma.NodeDecided{Address: "10.9.0.7"}, Modules: []mma.PlacedModule{}})
joiner, _ := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: "joiner", Decided: mma.NodeDecided{Address: "10.9.0.7"}, Modules: []mma.PlacedModule{}})
st[mma.StatePlacement].values["placement.joiner"] = joiner
removeNode := func(node, warrant string) (any, error) {
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
@@ -191,13 +195,13 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
w := newWorld()
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
pl, _ := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: "ace", Modules: []mma.PlacedModule{{Module: "web"}}})
pl, _ := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: "ace", Modules: []mma.PlacedModule{{Module: "web"}}})
pkey, _ := mmac.LegacyKey("ace")
w.putEntry(plc.PlacementBucket, pkey, pl, 1, time.Now())
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
hk, _ := delivery.HaveNodeKey(node)
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
hn, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
hk, _ := nodeengineclient.HaveNodeKey(node)
w.putEntry(buckets.Have, hk, hn, 1, time.Now())
}
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
t.Errorf("ace: %+v %v", d, ok)
@@ -213,7 +217,7 @@ func TestTheSeatsVerbs(t *testing.T) {
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
pl2, _ := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
pl2, _ := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
pkey2, _ := mmac.LegacyKey("g14")
w.putEntry(plc.PlacementBucket, pkey2, pl2, 1, time.Now())
p.unplacedFindings()
@@ -224,9 +228,9 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) {
writes := 0
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{plc.SeatHoldersBucket: kv}}
p := &passer{w: newWorld(), logf: t.Logf, kvs: kvSet{plc.SeatHoldersBucket: kv}}
place := func(node string, holds ...mma.Holding) {
pl, err := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: node, Modules: []mma.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
pl, err := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: node, Modules: []mma.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
if err != nil {
t.Fatal(err)
}
@@ -259,7 +263,7 @@ func TestTheSeatsVerbs(t *testing.T) {
ctx := context.Background()
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
var logs []string
p := &passer{w: newWorld(), kvs: buckets{plc.PlacementBucket: kv}, now: time.Now,
p := &passer{w: newWorld(), kvs: kvSet{plc.PlacementBucket: kv}, now: time.Now,
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
put := func(key string, value []byte, rev uint64) {
p.w.putEntry(plc.PlacementBucket, key, value, rev, time.Now())
@@ -319,7 +323,7 @@ func TestTheSeatsVerbs(t *testing.T) {
})
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil }}}
failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
@@ -370,29 +374,29 @@ func TestTheSeatsVerbs(t *testing.T) {
func wantsVerbs(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", delivery.ComposedRead} {
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", buckets.ComposedRead} {
st[state] = &memKV{values: map[string][]byte{}}
}
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
// a fingerprint, and laptop's web refused there.
webKey, _ := delivery.ComposedKey("anchor", "web", fpNew)
refusedKey, _ := delivery.ComposedKey("laptop", "web", fpNew)
st[delivery.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
st[delivery.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
st[delivery.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
st[buckets.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
st[buckets.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
st[buckets.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
manifest := func(name, extra string) {
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
digest := delivery.ManifestDigest([]byte(raw))
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw})
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: keys.Version, Module: name, JSON: raw})
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: keys.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
}
manifest("web", "")
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
st[haveState].values[node+".nats"], _ = nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
}
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
st[haveState].values["anchor.node"], _ = nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: "sha256:" + strings.Repeat("c", 64)})
var asked []string
var said []string
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go
+20 -4
View File
@@ -2,6 +2,9 @@ package main
import (
"context"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"mesh-delivery-assigner/internal/buckets"
"slices"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -62,7 +65,7 @@ func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign
if len(g) == 0 {
return nil
}
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
served, unread := graph.Served(graphHaves(n.Haves), func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
func(module string) []string { return g[module].Serves })
if len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
@@ -81,10 +84,10 @@ func (p *passer) offering(ctx context.Context, get getter, trunk map[string]cata
continue
}
for module, h := range haves {
if h.Phase != delivery.PhaseRunning || h.Health != delivery.HealthOK {
if h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthOK {
continue
}
m, ok := p.manifestOf(ctx, get, module, h.FP)
m, ok := p.manifestOf(ctx, get, module, h.Fp)
if !ok {
m = trunk[module]
}
@@ -112,7 +115,7 @@ func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string)
if err != nil {
return catalogue.Manifest{}, false
}
raw := p.w.value(delivery.BuilderBuildsBucket, key)
raw := p.w.value(buckets.Builds, key)
if raw == nil {
return catalogue.Manifest{}, false
}
@@ -126,3 +129,16 @@ func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string)
}
return got.manifest, true
}
// graphHaves is the haves as the module graph's Served reads them, which keeps the earlier have form for what it reads: the
// build, the phase and the health of each module on each node.
func graphHaves(haves map[string]map[string]nodeengine.HaveByNodeModuleValue) map[string]map[string]delivery.Have {
out := map[string]map[string]delivery.Have{}
for node, modules := range haves {
out[node] = map[string]delivery.Have{}
for module, h := range modules {
out[node][module] = delivery.Have{FP: h.Fp, Build: h.Build, Phase: delivery.Phase(h.Phase), Health: delivery.Health(h.Health)}
}
}
return out
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go
+6 -5
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"mesh-delivery-assigner/internal/buckets"
"slices"
"time"
@@ -53,10 +54,10 @@ var readTools = []viewedTool{
var viewedStates = map[string]string{
mma.StatePlacement: plc.PlacementBucket,
mma.StateSeatHolders: plc.SeatHoldersBucket,
"wants": delivery.AssignerWantsBucket,
"wants": buckets.Wants,
health.FindingsState: assign.FindingsBucket,
"asks": delivery.AssignerAsksBucket,
"differences": delivery.AssignerDifferencesBucket,
"asks": buckets.Asks,
"differences": buckets.Differences,
}
// viewTools are the assigner's first-view and read tools, reading only its own states, so they answer as soon as its
@@ -78,7 +79,7 @@ func keptSource(name string) views.Source { return views.KeptSource{State: stdio
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs buckets, findings assign.Findings, errs chan<- error, logf func(string, ...any)) {
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, findings assign.Findings, errs chan<- error, logf func(string, ...any)) {
tooLarge := func(state, key string, size int) {
if findings.Put == nil {
return
@@ -139,7 +140,7 @@ type indexerViews struct {
// indexerFirstViews are the indexer's states the assigner takes by first view and the views family, each by the name of
// the indexer's bucket the world keeps it under: manifests and modules (ADR 0621 §2), and the module graph, whose verb
// edges a want waits on (ADR 0614 §2, §3; ADR 0631 §1).
var indexerFirstViews = map[string]string{"manifests": delivery.ManifestsBucket, "modules": delivery.ModulesBucket, graph.State: graph.Bucket}
var indexerFirstViews = map[string]string{"manifests": buckets.Manifests, "modules": buckets.Modules, graph.State: graph.Bucket}
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
// between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go
+10 -8
View File
@@ -4,8 +4,10 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
"git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
"sort"
"strconv"
"time"
@@ -27,7 +29,7 @@ import (
// The states the verbs read, as the runtime names them.
const (
policiesState = delivery.AssignerPoliciesState
policiesState = buckets.PoliciesState
manifestsState = delivery.IndexerModule + ".manifests"
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
PolicyChangedEvent = "policy-changed"
@@ -153,15 +155,15 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
composed := map[string]string{}
for key, raw := range haves {
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
if node, _, isNode, err := nodeengineclient.ParseHaveKey(key); err == nil && isNode {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Composed != "" {
composed[node] = h.Composed
}
}
}
out := map[string]any{"composed": composed}
// And what every node published (ADR 0593 §8), read only to compare.
if values, err := s.values(delivery.ComposedRead); err != nil {
if values, err := s.values(buckets.ComposedRead); err != nil {
out["published-unread"] = err.Error()
} else {
out["published"] = published(values)
@@ -275,15 +277,15 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
}
var nodes []busNode
for key, raw := range haves {
node, module, isNode, err := delivery.ParseHaveKey(key)
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode || module != bus {
continue
}
h, err := delivery.DecodeHave(raw)
if err != nil || h.Phase == delivery.PhaseRemoved {
h, err := nodeengineclient.DecodeHave(raw)
if err != nil || h.Phase == nodeengineclient.PhaseRemoved {
continue
}
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
nodes = append(nodes, busNode{Node: node, FP: h.Fp, Behind: newest != "" && h.Fp != newest})
}
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go
+42 -38
View File
@@ -3,7 +3,11 @@ package main
import (
"context"
"errors"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-sdk/go/keys"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sort"
"sync"
@@ -78,7 +82,7 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
if bucket == plc.PlacementBucket {
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
}
if bucket == delivery.SourceBucket && key == delivery.SourceMainKey {
if bucket == buckets.Source && key == delivery.SourceMainKey {
if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" {
w.sourceGroups[revision] = s.Group
}
@@ -102,7 +106,7 @@ func (w *world) release(module, fp string) (string, bool) {
if group == "" || err != nil {
return "", false
}
raw := w.value(delivery.MergesBucket, key)
raw := w.value(buckets.Merges, key)
if raw == nil {
return "", false
}
@@ -146,17 +150,17 @@ func (w *world) put(bucket, key string, value []byte) {
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
_, _ = w.placement.Put(key, value, 0, false)
}
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
w.healthy[key] = append(list, h.FP)
if bucket == buckets.Have {
if h, err := nodeengineclient.DecodeHave(value); err == nil && h.Fp != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.Fp })
w.healthy[key] = append(list, h.Fp)
}
}
}
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
func (w *world) lastHealthy(node, module, fp string) string {
key, err := delivery.HaveKey(node, module)
key, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return ""
}
@@ -290,7 +294,7 @@ func (w *world) placedNodes() map[string]bool {
// trunk is every module the indexer holds at the trunk, with its value.
func (w *world) trunk() map[string]delivery.Module {
out := map[string]delivery.Module{}
for key, value := range w.read(delivery.ModulesBucket) {
for key, value := range w.read(buckets.Modules) {
name, err := delivery.ParseModulesKey(key)
if err != nil {
continue
@@ -304,14 +308,14 @@ func (w *world) trunk() map[string]delivery.Module {
// records is each node's record in its own have key (`<node>.node`): the nodes a placement proposal is answered by,
// with those placement names (ADR 0593 §8).
func (w *world) records() map[string]delivery.NodeRecord {
out := map[string]delivery.NodeRecord{}
for key, value := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
func (w *world) records() map[string]nodeengine.NodeRecord {
out := map[string]nodeengine.NodeRecord{}
for key, value := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
out[node] = *h.Record
}
}
@@ -321,7 +325,7 @@ func (w *world) records() map[string]delivery.NodeRecord {
// online is, by node, whether the watchman holds the node online.
func (w *world) online() map[string]bool {
out := map[string]bool{}
for key, value := range w.read(delivery.PresenceBucket) {
for key, value := range w.read(buckets.Presence) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
@@ -336,7 +340,7 @@ func (w *world) online() map[string]bool {
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]delivery.Want {
out := map[string]map[string]delivery.Want{}
for key, value := range w.read(delivery.AssignerWantsBucket) {
for key, value := range w.read(buckets.Wants) {
node, module, err := delivery.ParseWantKey(key)
if err != nil {
continue
@@ -355,9 +359,9 @@ func (w *world) wantsOf() map[string]map[string]delivery.Want {
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{},
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
for key, value := range w.read(delivery.PresenceBucket) {
for key, value := range w.read(buckets.Presence) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
@@ -366,21 +370,21 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
n.Online[node] = p.Online
}
}
for key, value := range w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
for key, value := range w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil {
continue
}
if isNode {
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
n.Reads[node] = h.Record.ManifestReads
}
continue
}
if h, err := delivery.DecodeHave(value); err == nil {
if h, err := nodeengineclient.DecodeHave(value); err == nil {
if n.Haves[node] == nil {
n.Haves[node] = map[string]delivery.Have{}
n.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
}
n.Haves[node][module] = h
}
@@ -411,7 +415,7 @@ func (w *world) value(bucket, key string) []byte {
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
// manifest but the controller's reads the controller's grants.
func (w *world) grantsBucket() string { return delivery.PartGrantsBucket }
func (w *world) grantsBucket() string { return buckets.Grants }
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
@@ -428,7 +432,7 @@ func (w *world) built(module, fp string) bool {
return false
}
w.mu.Lock()
value, ok := w.values[delivery.BuilderBuildsBucket][key]
value, ok := w.values[buckets.Builds][key]
w.mu.Unlock()
if !ok {
return false
@@ -444,7 +448,7 @@ func (w *world) buildNeeds(module, fp string) []string {
if err != nil {
return nil
}
b, err := delivery.DecodeBuilderBuild(w.value(delivery.BuilderBuildsBucket, key))
b, err := delivery.DecodeBuilderBuild(w.value(buckets.Builds, key))
if err != nil || b.State != delivery.BuildDone {
return nil
}
@@ -476,9 +480,9 @@ func (w *world) knownNodes() []string {
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
// controller's copy (mesh-controller_composed) is read no more (issue 774).
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
func (w *world) composed(node, module, fp string) (assign.ComposedDigest, bool) {
if !w.placedNodes()[node] {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
return w.ownComposed(node)
}
@@ -486,7 +490,7 @@ func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
func (w *world) answer(key string) (delivery.Answer, bool) {
w.mu.Lock()
value, ok := w.values[delivery.AnswersBucket][key]
value, ok := w.values[buckets.Answers][key]
w.mu.Unlock()
if !ok {
return delivery.Answer{}, false
@@ -512,20 +516,20 @@ func (w *world) changed(bucket, key string) uint64 {
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
key, err := delivery.HaveNodeKey(node)
func (w *world) ownComposed(node string) (assign.ComposedDigest, bool) {
key, err := nodeengineclient.HaveNodeKey(node)
if err != nil {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
raw, ok := w.read(delivery.HaveBucket)[key]
raw, ok := w.read(buckets.Have)[key]
if !ok {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
h, err := delivery.DecodeHaveNode(raw)
h, err := nodeengineclient.DecodeHaveNode(raw)
if err != nil || h.Composed == "" {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
return assign.ComposedDigest{V: keys.Version, Digest: h.Composed, Judgement: assign.JudgedNeither,
Words: "the node composes its own"}, true
}
@@ -534,8 +538,8 @@ func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
func (w *world) unplacedNodes() []string {
placed := w.placedNodes()
var out []string
for key := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
for key := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err == nil && isNode && !placed[node] {
out = append(out, node)
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go
+9 -8
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"mesh-delivery-assigner/internal/buckets"
"slices"
"strings"
"testing"
@@ -45,13 +46,13 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(buckets.Have, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
}
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
if got := w.changed(buckets.Have, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
@@ -66,11 +67,11 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
{"the part's grant", true, true},
{"no grant", false, false},
} {
p := newPasser(buckets{}, time.Now())
p := newPasser(kvSet{}, time.Now())
if row.inPart {
p.w.put(delivery.PartGrantsBucket, gkey, grant)
p.w.put(buckets.Grants, gkey, grant)
}
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
if got := p.w.grantsBucket(); got != buckets.Grants {
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
}
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
modules/mesh-delivery-assigner/go.mod
+17
View File
@@ -37,3 +37,20 @@ require (
)
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds
replace git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate => ../mesh-grants/grantsstate
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient => ../../contracts/go/clients/meshcatalogclient
require git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient => ../../contracts/go/clients/meshdeliveryclient
require git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog => ../../contracts/go/seats/meshcatalog
require git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery => ../../contracts/go/seats/meshdelivery
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
modules/mesh-delivery-assigner/internal/assign/asks.go
+4 -4
View File
@@ -157,10 +157,10 @@ const (
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + need.Words,
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + need.Words,
}[need.Judgement]
return assignerAsk(0, oc.Ask{
ID: delivery.AskingID(key, 0),
modules/mesh-delivery-assigner/internal/assign/asks_test.go
+2 -2
View File
@@ -178,7 +178,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
if err != nil {
t.Fatal(err)
}
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
@@ -222,7 +222,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
if err != nil {
modules/mesh-delivery-assigner/internal/assign/composed.go
+45
View File
@@ -0,0 +1,45 @@
package assign
import nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
// Judgement is the assigner's judgement of a move from the composition a node runs, as its have names it, to a new one
// (ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests). The three the node-engine names are its have's words (a have's
// judgement); neither and unchanged are the assigner's own.
type Judgement = string
// The judgements of a move.
const (
// JudgedRecreates is a move that recreates a container.
JudgedRecreates = nec.JudgedRecreates
// JudgedMovesData is a move that gives the module's data another place.
JudgedMovesData = nec.JudgedMovesData
// JudgedNeither is a move that does neither, and the judgement of a composition the node never ran.
JudgedNeither = "neither"
// JudgedCannot is a move that cannot be judged, because the composition the node runs is gone or cannot be read.
JudgedCannot = nec.JudgedCannot
// JudgedUnchanged is a composition equal to the module's slice of the declaration the node last applied, for a node
// whose have names no composition yet (the first want at the switch, novox/hq ADR 0540 clause 9): applying it
// changes nothing on the node, so it needs no person's yes.
JudgedUnchanged = "unchanged"
)
// ComposedDigest is what the assigner knows of a composition beside the node's have: its Digest, and the Judgement of the
// move to it from the composition the node runs now, whose digest is Against (empty when the node runs no composition of
// the module). Words say why, and are required for a move that cannot be judged.
//
// Policy is the module's upgrade policy as the assigner holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom where it
// comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it.
type ComposedDigest struct {
V int `json:"v"`
Digest string `json:"digest"`
Judgement Judgement `json:"judgement"`
Against string `json:"against,omitempty"`
Words string `json:"words,omitempty"`
// UserListOnly says the composition differs from the one the node's have names only in the bus's user list, whatever
// the judgement says of its mount: the assigner writes such a revision into the bus node's want at once, outside the
// record hold (ADR 0539 clause 7).
UserListOnly bool `json:"user-list-only,omitempty"`
Policy string `json:"policy,omitempty"`
PolicyFrom string `json:"policy-from,omitempty"`
PolicyWhy string `json:"policy-why,omitempty"`
}
modules/mesh-delivery-assigner/internal/assign/policy.go
+2 -1
View File
@@ -3,6 +3,7 @@ package assign
import (
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"sort"
"strings"
"time"
@@ -19,7 +20,7 @@ import (
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
// PoliciesBucket is the assigner's state `policies`.
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
var PoliciesBucket = delivery.AssignerModule + "_" + buckets.PoliciesState
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
modules/mesh-delivery-assigner/internal/assign/put.go
+17 -15
View File
@@ -2,6 +2,8 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
@@ -31,7 +33,7 @@ type PutWorld struct {
Newest string
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
Composed *delivery.ComposedDigest
Composed *ComposedDigest
Granted bool
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
// that ask expired unanswered.
@@ -43,7 +45,7 @@ type PutWorld struct {
Snapshotting bool
SnapshotFailed string
// Have is what the node says it runs of the module; nil when it says nothing.
Have *delivery.Have
Have *nodeengine.HaveByNodeModuleValue
Now time.Time
}
@@ -151,13 +153,13 @@ func StepPut(w PutWorld) PutNext {
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
func applied(w PutWorld, p delivery.Put) delivery.Put {
h := w.Have
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
if h == nil || h.Fp != p.FP || h.Composed != p.Composed {
return p
}
switch {
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK && p.Hold:
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK:
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case failed(*h):
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
@@ -168,8 +170,8 @@ func applied(w PutWorld, p delivery.Put) delivery.Put {
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
// container, move data, or that cannot be judged (§14).
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
runs := w.Have != nil && w.Have.FP == w.Put.FP
func needsPerson(w PutWorld, d ComposedDigest) bool {
runs := w.Have != nil && w.Have.Fp == w.Put.FP
if runs && w.Have.Composed == d.Digest {
return false // the node runs this composition already: nothing moves
}
@@ -179,7 +181,7 @@ func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
if !w.Recorded {
return false
}
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
}
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
@@ -218,14 +220,14 @@ func PutAct(node, module, fp, composed string) occ.Act {
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
// channel that proves who answered takes it (ADR 0259), never the view.
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
func PutAsk(key, node, module string, p delivery.Put, d ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
yes := oc.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: occ.Acknowledge}
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + d.Words,
delivery.JudgedNeither: "recreates no container and moves no data",
delivery.JudgedUnchanged: "changes nothing the node runs",
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + d.Words,
JudgedNeither: "recreates no container and moves no data",
JudgedUnchanged: "changes nothing the node runs",
}[d.Judgement]
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
expires := occ.FormatTime(now.Add(7 * 24 * time.Hour))
modules/mesh-delivery-assigner/internal/assign/seat_test.go
+4 -4
View File
@@ -2,6 +2,7 @@ package assign
import (
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/keys"
"slices"
"strings"
"testing"
@@ -10,7 +11,6 @@ import (
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
mmac "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/delivery"
plc "mesh-delivery-assigner/internal/placement"
)
@@ -19,9 +19,9 @@ import (
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
func TestThePlacementSeatDecides(t *testing.T) {
anchor := mma.Placement{V: delivery.Version, Node: "anchor", Decided: mma.NodeDecided{Hub: true, Address: "10.0.0.1"},
anchor := mma.Placement{V: keys.Version, Node: "anchor", Decided: mma.NodeDecided{Hub: true, Address: "10.0.0.1"},
Modules: []mma.PlacedModule{{Module: "store", Ports: map[string]int64{"5432": 5432}}, {Module: "web"}}}
laptop := mma.Placement{V: delivery.Version, Node: "laptop", Decided: mma.NodeDecided{Address: "10.0.0.2"}, Modules: []mma.PlacedModule{{Module: "shell"}}}
laptop := mma.Placement{V: keys.Version, Node: "laptop", Decided: mma.NodeDecided{Address: "10.0.0.2"}, Modules: []mma.PlacedModule{{Module: "shell"}}}
in := func(changes ...mma.ProposalChange) ProposeIn {
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
Current: map[string]mma.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
@@ -50,7 +50,7 @@ func TestThePlacementSeatDecides(t *testing.T) {
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
// act without its lease.
"the controller placed again": {in(mma.ProposalChange{Op: plc.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
"the controller seeded whole": {in(mma.ProposalChange{Op: plc.OpSeed, Node: "anchor", Whole: &mma.Placement{V: delivery.Version, Node: "anchor",
"the controller seeded whole": {in(mma.ProposalChange{Op: plc.OpSeed, Node: "anchor", Whole: &mma.Placement{V: keys.Version, Node: "anchor",
Decided: anchor.Decided, Modules: []mma.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
"a pin of nobody placed": {in(mma.ProposalChange{Op: plc.OpPin, Node: "laptop", Provision: "db", Provider: &mma.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
} {
modules/mesh-delivery-assigner/internal/assign/wants.go
+38 -36
View File
@@ -2,6 +2,8 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"maps"
"slices"
"sort"
@@ -59,7 +61,7 @@ type Module struct {
Nodes []string
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
Composed func(node, fp string) (ComposedDigest, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
@@ -93,9 +95,9 @@ type Module struct {
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
func (m Module) composition(node, fp string) (ComposedDigest, bool) {
if m.Composed == nil {
return delivery.ComposedDigest{}, false
return ComposedDigest{}, false
}
return m.Composed(node, fp)
}
@@ -103,17 +105,17 @@ func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
type Nodes struct {
Online map[string]bool
Haves map[string]map[string]delivery.Have
Haves map[string]map[string]nodeengine.HaveByNodeModuleValue
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
Providers map[string]map[string][]string
// Reads is, by node, the reader capabilities its node-engine states in its node record (delivery.NodeRecord.ManifestReads,
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
Reads map[string][]string
}
// healthy says whether a have runs fp healthy.
func healthy(h delivery.Have, fp string) bool {
return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
}
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
@@ -289,7 +291,7 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
if !slices.Contains(nodes, node) {
at = Canary(nodes, n.Online)
}
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
return true, ""
}
}
@@ -351,7 +353,7 @@ type Outcome struct {
type RecordNeed struct {
Node string
Digest string
Judgement delivery.Judgement
Judgement Judgement
Words string
}
@@ -381,7 +383,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
h, ok := n.Haves[node][m.Name]
if !ok || h.FP != m.FP || !failed(h) {
if !ok || h.Fp != m.FP || !failed(h) {
continue
}
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
@@ -450,15 +452,15 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
older := ran && runs(have) && have.Fp != m.FP && strings.HasPrefix(have.Fp, delivery.SourcePrefix2)
if older {
_, composed := m.composition(node, have.FP)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
_, composed := m.composition(node, have.Fp)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.Fp)
}
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
oldForm := ran && runs(have) && have.Fp != m.FP && !strings.HasPrefix(have.Fp, delivery.SourcePrefix2)
switch {
case older:
ok, target = true, have.FP
ok, target = true, have.Fp
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
@@ -472,7 +474,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
n.Haves[node][m.Name].FP != m.FP {
n.Haves[node][m.Name].Fp != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
// it would be on neither list and its node-engine would give the module up (issue 1945).
@@ -498,7 +500,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if before.Removed {
asked, hadLive = delivery.Want{}, false
}
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.Fp, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
@@ -533,9 +535,9 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name]
switch {
case there && h.Phase != delivery.PhaseRemoved && !held:
case there && h.Phase != nodeengineclient.PhaseRemoved && !held:
out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == delivery.PhaseRemoved):
case had && (!there || h.Phase == nodeengineclient.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
@@ -557,7 +559,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
at := m
at.FP = before.FP
running, ran := n.Haves[node][m.Name]
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
need, wait := at.mustAsk(node, before, true, ran, running.Fp, running.Composed == "", d)
if need != nil {
out.Record = append(out.Record, *need)
}
@@ -609,7 +611,7 @@ func (m Module) selfDerived(node string, before delivery.Want, had bool, next de
at := m
at.FP = next.FP
running, ran := n.Haves[node][m.Name]
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
if need, wait := at.mustAsk(node, before, had, ran, running.Fp, running.Composed == "", d); wait {
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
out.Record = append(out.Record, *need)
}
@@ -629,8 +631,8 @@ func NoWantWords(rule int, words string) string {
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
func failed(h nodeengine.HaveByNodeModuleValue) bool {
return h.Health == nodeengineclient.HealthFailed && (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseFailed)
}
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
@@ -650,7 +652,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string {
// the hold and the set-back stand at the second failure (ADR 0557 §2).
func failedElsewhere(m Module, n Nodes, node string) bool {
for _, other := range m.Nodes {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.Fp == m.FP && failed(h) {
return true
}
}
@@ -671,8 +673,8 @@ type NodeFailure struct {
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
func runs(h nodeengine.HaveByNodeModuleValue) bool {
return h.Phase == nodeengineclient.PhaseRunning
}
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
@@ -684,7 +686,7 @@ func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
continue
}
if !healthy(h, m.FP) {
@@ -697,7 +699,7 @@ func allRunHealthy(m Module, n Nodes) bool {
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h delivery.Have, at time.Time) string {
func HeldWords(h nodeengine.HaveByNodeModuleValue, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
@@ -731,7 +733,7 @@ func short(fp string) string {
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
// streams before it applies (§2); the assigner asks no snapshot for it.
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
return nil, false
}
@@ -739,10 +741,10 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == JudgedUnchanged {
return nil, false
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
if !m.recorded(d) || d.Judgement == JudgedNeither || d.Judgement == JudgedUnchanged {
return nil, false
}
if m.Allowed != nil && m.Allowed(node, d.Digest) {
@@ -754,13 +756,13 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
func (m Module) recorded(d ComposedDigest) bool {
if m.Chosen != nil {
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
return policy == catalogue.PolicyRecord
}
if d.Policy != "" {
return d.Policy == delivery.PolicyRecord
return d.Policy == catalogue.PolicyRecord
}
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
return policy == catalogue.PolicyRecord
@@ -774,14 +776,14 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
return delivery.Want{}, false
}
running, ran := n.Haves[node][m.Name]
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
if !ran || running.Fp == "" || (had && (before.Removed || before.FP != running.Fp)) {
return delivery.Want{}, false
}
d, ok := m.composition(node, running.FP)
d, ok := m.composition(node, running.Fp)
if !ok || !d.UserListOnly {
return delivery.Want{}, false
}
w := delivery.Want{FP: running.FP, Composed: d.Digest}
w := delivery.Want{FP: running.Fp, Composed: d.Digest}
if had && before.Composed == d.Digest {
w.Backup, w.Words = before.Backup, before.Words
}
modules/mesh-delivery-assigner/internal/assign/wants_test.go
+136 -134
View File
@@ -1,6 +1,8 @@
package assign
import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"slices"
"strings"
"testing"
@@ -27,9 +29,9 @@ func ready() (Module, Nodes) {
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
n := Nodes{
Online: map[string]bool{"ace": true, "g14": true},
Haves: map[string]map[string]delivery.Have{
"ace": {"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, "store": {FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
"g14": {"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"mail": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, "store": {Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}},
"g14": {"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}},
},
Providers: map[string]map[string][]string{"database": {"store": {"ace"}}},
}
@@ -40,8 +42,8 @@ func ready() (Module, Nodes) {
func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) }
// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither.
func composedEverywhere(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, true
func composedEverywhere(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedNeither}, true
}
func wantedOld() map[string]delivery.Want {
@@ -66,11 +68,11 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
delivery.RuleBuildDone: func(m *Module, _ *Nodes) { m.Built = false },
delivery.RuleNotHeld: func(m *Module, _ *Nodes) { m.Held = true },
delivery.RuleCanary: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
},
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
delivery.RuleComposed: func(m *Module, _ *Nodes) {
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
m.Composed = func(string, string) (ComposedDigest, bool) { return ComposedDigest{}, false }
},
// 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want.
delivery.RuleProvisions: func(m *Module, _ *Nodes) {
@@ -297,26 +299,26 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
// healthy held photos, and novox offline left a module with no canary.
for _, c := range []struct {
name, module, provision, provider string
have *delivery.Have // what novox runs of the module; nil: a new assignment
have *nodeengine.HaveByNodeModuleValue // what novox runs of the module; nil: a new assignment
before map[string]delivery.Want
online bool
want string
}{
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
&delivery.Have{FP: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}, nil, true, fNew},
&nodeengine.HaveByNodeModuleValue{Fp: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}, nil, true, fNew},
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
&nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, true, fNew},
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
&nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, false, fOld},
} {
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]delivery.Have{"novox": {
c.provider: {FP: fpOf('p'), Phase: delivery.PhaseFailed, Health: delivery.HealthFailed}}},
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {
c.provider: {Fp: fpOf('p'), Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed}}},
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
if c.have != nil {
n.Haves["novox"][c.module] = *c.have
@@ -345,7 +347,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
return map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fp, Self: true, Yes: dig(fp)}}
}
atF := func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseFailed, Health: delivery.HealthOK}
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthOK}
}
for _, c := range []struct {
name string
@@ -357,14 +359,14 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
}{
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
n.Haves["ace"]["store"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthWaits}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}},
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleCanary},
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}, want: delivery.Want{FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleCanary,
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: delivery.RuleCanary},
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
@@ -380,15 +382,15 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
atF(m, n)
recording(m)
m.Held = true
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld)}, true
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld)}, true
}
}, rule: delivery.RuleNotHeld, asked: true},
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if node == "g14" {
return delivery.ComposedDigest{}, false
return ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
@@ -396,9 +398,9 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Self = func(node string) bool { return node == "g14" }
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if node == "g14" {
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither}, true
return ComposedDigest{Judgement: JudgedNeither}, true
}
return composedEverywhere(node, fp)
}
@@ -459,9 +461,9 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
m, n := ready()
n.Online["ace"] = false
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
n.Providers["database"]["store"] = []string{"ace", "g14"}
n.Haves["g14"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["store"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('s'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if c := Canary(m.Nodes, n.Online); c != "g14" {
t.Fatalf("canary = %q, want g14", c)
}
@@ -477,7 +479,7 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
// 1297/B4: a failed health check of F holds F and sets the nodes that want it back to their last healthy fingerprint.
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
@@ -493,7 +495,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
if out.Held != nil || len(out.Wants) != 0 {
@@ -511,7 +513,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
failedHave := n.Haves["ace"]["mail"]
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
n.Online["novox"] = true
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}}
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if _, alone := out.NodeFailed["g14"]; !alone {
@@ -524,7 +526,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
}
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
delete(before, "novox")
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
@@ -537,15 +539,15 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
for _, c := range []struct {
name string
online map[string]bool
novox delivery.Health
novox string
}{
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, nodeengineclient.HealthOK},
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, nodeengineclient.HealthFailed},
} {
n.Online = c.online
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
if c.novox == delivery.HealthOK {
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: c.novox}}
if c.novox == nodeengineclient.HealthOK {
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
@@ -562,7 +564,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
func TestADataModuleIsNotSetBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
if out.Held == nil {
t.Fatal("F was not held")
@@ -579,7 +581,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
// 1906/D3: a module without data is set back and asked about nothing.
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
if len(out.Restore) != 0 {
t.Errorf("restore asks = %v", out.Restore)
@@ -605,9 +607,9 @@ func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
m, n := ready()
have := map[string]bool{}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if !have[node+"|"+fp] {
return delivery.ComposedDigest{}, false
return ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
@@ -639,8 +641,8 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
m, n := ready()
revised := "sha256:" + strings.Repeat("9", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedNeither}, true
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true
}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
out := Wants(m, n, running, nil)
@@ -662,8 +664,8 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
if w := out.Wants["g14"]; w.FP != fOld || w.Composed != revised || w.Newest != fNew {
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord,
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord,
Against: dig(fOld), Words: "recreates"}, true
}
out = Wants(m, n, wantedOld(), nil)
@@ -678,11 +680,11 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
m.Composed = func(node, fp string) (ComposedDigest, bool) {
if fp == fOld {
return delivery.ComposedDigest{}, false
return ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
@@ -698,11 +700,11 @@ func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
}
// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F).
func movingTo(j delivery.Judgement) (Module, Nodes, map[string]delivery.Want) {
func movingTo(j Judgement) (Module, Nodes, map[string]delivery.Want) {
m, n := ready()
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: j}
if j != delivery.JudgedNeither {
m.Composed = func(node, fp string) (ComposedDigest, bool) {
d := ComposedDigest{Digest: dig(fp), Judgement: j}
if j != JudgedNeither {
d.Against, d.Words = dig(fOld), "judged"
}
return d, true
@@ -717,7 +719,7 @@ func recording(m *Module) {
// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be
// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint.
func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) {
for _, j := range []delivery.Judgement{delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot} {
for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} {
m, n, wanted := movingTo(j)
recording(&m)
out := Wants(m, n, wanted, nil)
@@ -749,12 +751,12 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T)
// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module
// that rolls (not record) is never asked.
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
m, n, wanted := movingTo(delivery.JudgedNeither)
m, n, wanted := movingTo(JudgedNeither)
recording(&m)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("neither: %+v", out)
}
m, n, _ = movingTo(delivery.JudgedRecreates)
m, n, _ = movingTo(JudgedRecreates)
recording(&m)
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
delete(n.Haves["ace"], "mail")
@@ -762,13 +764,13 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
t.Errorf("a node that never ran it: %+v", out)
}
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
m, n, _ = movingTo(delivery.JudgedCannot)
m, n, _ = movingTo(JudgedCannot)
recording(&m)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew || len(out.Record) == 0 {
t.Errorf("a node that runs it with no want: %+v", out)
}
m, n, wanted = movingTo(delivery.JudgedRecreates)
m, n, wanted = movingTo(JudgedRecreates)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("a rolled module: %+v", out)
}
@@ -777,7 +779,7 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m, n, _ := movingTo(JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
// backs its streams up before it applies.
@@ -824,10 +826,10 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
}
}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: userList, Judgement: JudgedNeither, UserListOnly: true}, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
held := put
held.Step = delivery.PutHolding
m.Puts = map[string]delivery.Put{"ace": held}
@@ -844,8 +846,8 @@ func stepsOfAPut(t *testing.T) {
t.Helper()
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
d := ComposedDigest{Digest: dig(fNew), Judgement: JudgedCannot, Words: "the have names no composition"}
running := nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
bus := func() PutWorld {
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
Composed: &d, Granted: true, Have: &running, Now: now}
@@ -886,7 +888,7 @@ func stepsOfAPut(t *testing.T) {
w := bus()
ul := d
ul.UserListOnly = true
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
w.Composed, w.Have = &ul, &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, delivery.PutWanted, "held until released", false, false, false},
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
@@ -894,7 +896,7 @@ func stepsOfAPut(t *testing.T) {
w := bus()
w.Bus, w.Recorded = false, true
r := d
r.Judgement = delivery.JudgedRecreates
r.Judgement = JudgedRecreates
w.Composed = &r
return w
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
@@ -902,7 +904,7 @@ func stepsOfAPut(t *testing.T) {
w := bus()
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, delivery.PutHolding, "held there until released", false, false, false},
{"wanted without hold: done", func() PutWorld {
@@ -910,7 +912,7 @@ func stepsOfAPut(t *testing.T) {
w.Put.Hold = false
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
w.Put.Composed = d.Digest
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: d.Digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return w
}, delivery.PutDone, "the rules write its want again", false, false, false},
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
@@ -950,7 +952,7 @@ func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
// wants move by the rules again.
func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"}
m.Released = true
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleNotHeld}}, map[string]string{"ace": fOld})
if out.Held != nil {
@@ -966,7 +968,7 @@ func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
m.Restored = func(node string) bool { return false }
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.FP == fOld {
@@ -981,16 +983,16 @@ func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included;
// a composition with none (made by hand) falls back to what the manifest derives.
func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
m, n, wanted := movingTo(delivery.JudgedMovesData)
m, n, wanted := movingTo(JudgedMovesData)
with := func(policy, from string) {
inner := m.Composed
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
m.Composed = func(node, fp string) (ComposedDigest, bool) {
d, ok := inner(node, fp)
d.Policy, d.PolicyFrom = policy, from
return d, ok
}
}
with(delivery.PolicyRecord, "person") // a roll module the operator set to record
with(catalogue.PolicyRecord, "person") // a roll module the operator set to record
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].FP == fNew {
t.Errorf("the operator's record policy was not honoured: %+v", out)
}
@@ -1004,7 +1006,7 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
}
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
with(delivery.PolicyRecord, "person")
with(catalogue.PolicyRecord, "person")
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
@@ -1014,13 +1016,13 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
// A node that runs the bus with no readable want gets its want by the rules, at the fingerprint it runs or at a new one alike: the
// bus has no bar of its own (ADR 0601 §1).
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m, n, _ := movingTo(JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
}
@@ -1031,18 +1033,18 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against
// the composition the node runs, policy record, UserListOnly set.
func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedRecreates)
m, n, _ := movingTo(JudgedRecreates)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
Policy: delivery.PolicyRecord, PolicyFrom: "the bus"}
m.Composed = func(node, fp string) (ComposedDigest, bool) {
d := ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
Policy: catalogue.PolicyRecord, PolicyFrom: "the bus"}
if fp == fOld {
d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list
}
return d, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
for name, wanted := range map[string]map[string]delivery.Want{
"a want at the fingerprint it runs": {"ace": {FP: fOld, Composed: dig(fOld)}},
"no readable want": nil,
@@ -1069,17 +1071,17 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test
func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
for name, bus := range map[string]bool{"a record module": false, "the bus": true} {
// only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold
unchanged := func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedUnchanged}, true
unchanged := func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedUnchanged}, true
}
m, n, _ := movingTo(delivery.JudgedCannot)
m, n, _ := movingTo(JudgedCannot)
recording(&m)
m.Nodes = []string{"ace"}
if bus {
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
m.Composed = unchanged
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} // no composition named
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named
out := Wants(m, n, nil, nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
@@ -1089,16 +1091,16 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
}
// a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge,
// still wait for the ask
for what, j := range map[string]delivery.Judgement{"differing": delivery.JudgedRecreates, "unjudged": delivery.JudgedCannot, "judged neither": delivery.JudgedNeither} {
for what, j := range map[string]Judgement{"differing": JudgedRecreates, "unjudged": JudgedCannot, "judged neither": JudgedNeither} {
m2, n2, _ := movingTo(j)
recording(&m2)
m2.Nodes = []string{"ace"}
if bus {
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
n2.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m2, n2, nil, nil)
if j == delivery.JudgedNeither {
if j == JudgedNeither {
if out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("%s, %s: %+v", name, what, out)
}
@@ -1109,15 +1111,15 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
}
}
// a have that names a composition is judged as before
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
m.Composed = func(node, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew && len(out.Record) == 0 {
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
}
// a new fingerprint still asks (the bus is barred instead)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
@@ -1131,7 +1133,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
t.Fatalf("held = %+v", out.Held)
@@ -1145,7 +1147,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
everywhere := func() (Module, Nodes) {
m, n := ready()
m.Held = true
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
return m, n
}
m, n := everywhere()
@@ -1167,10 +1169,10 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
for name, change := range map[string]func(*Module, *Nodes){
"set back": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
},
"failing": func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
},
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
@@ -1193,11 +1195,11 @@ func x11() (Module, Nodes) {
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
n := Nodes{
Online: map[string]bool{"g14": true, "shanks": true},
Haves: map[string]map[string]delivery.Have{
"g14": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
"shanks": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"g14": {"screen-lock": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown},
"xorg": {Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}},
"shanks": {"screen-lock": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown},
"xorg": {Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}},
},
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
}
@@ -1222,14 +1224,14 @@ func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T)
}
// with the provider healthy, only the canary holds shanks back: still at what it runs
m, n = x11()
n.Haves["g14"]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.FP != fOld || w.Rule != delivery.RuleCanary {
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
}
// a node already running F gets its want at F; a removed want is no want either
m, n = x11()
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, map[string]delivery.Want{"g14": {Removed: true}}, nil)
if w := out.Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) || w.Removed {
t.Errorf("g14 running F with a removed want = %+v, want F", w)
@@ -1241,12 +1243,12 @@ func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T)
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
for what, change := range map[string]func(*Module, *Nodes){
"an old-form build": func(_ *Module, n *Nodes) {
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
},
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
"no composition": func(m *Module, _ *Nodes) {
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, fp == fNew
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedNeither}, fp == fNew
}
},
} {
@@ -1343,18 +1345,18 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm := func(n Nodes, node string) {
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
n.Haves[node]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-1792260876956849831", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
}
providersHealthy := func(n Nodes) {
for _, node := range []string{"g14", "shanks"} {
n.Haves[node]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves[node]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}
}
// the canary g14 runs F healthy: shanks's first want names F
m, n := x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.FP != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
@@ -1366,7 +1368,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
@@ -1377,7 +1379,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.FP != fNew || out.Unreached["shanks"] == "" {
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
@@ -1389,7 +1391,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
m, n = x11()
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
m.Granted = false
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
t.Errorf("F not granted: shanks got a want: %+v", w)
@@ -1407,21 +1409,21 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
// canary two seconds before the new build was healthy.
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
for _, phase := range []delivery.Phase{delivery.PhaseBackingUp, delivery.PhaseApplying, delivery.PhaseChecking} {
for _, phase := range []string{nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseApplying, nodeengineclient.PhaseChecking} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
}
}
for _, phase := range []delivery.Phase{delivery.PhaseRunning, delivery.PhaseFailed} {
for _, phase := range []string{nodeengineclient.PhaseRunning, nodeengineclient.PhaseFailed} {
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
@@ -1433,30 +1435,30 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
// fingerprint, or failed, still does.
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
setup := func(phase delivery.Phase, fp string, health delivery.Health) (Module, Nodes) {
setup := func(phase string, fp string, health string) (Module, Nodes) {
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]delivery.Have{
"ace": {"web": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
"g14": {"web": {FP: fp, Phase: phase, Health: health}}}}
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"web": {Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}},
"g14": {"web": {Fp: fp, Phase: phase, Health: health}}}}
return m, n
}
for _, phase := range []delivery.Phase{delivery.PhaseRemoved, delivery.PhaseHeld, delivery.PhaseApplying, delivery.PhaseBackingUp, delivery.PhaseChecking} {
m, n := setup(phase, fOld, delivery.HealthUnknown)
for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} {
m, n := setup(phase, fOld, nodeengineclient.HealthUnknown)
if out := Wants(m, n, nil, nil); !out.Lifted {
t.Errorf("g14's have in phase %s kept the hold up", phase)
}
}
for what, h := range map[string]delivery.Have{
"running another fingerprint": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
"failed": {FP: fNew, Phase: delivery.PhaseFailed, Health: delivery.HealthFailed},
for what, h := range map[string]nodeengine.HaveByNodeModuleValue{
"running another fingerprint": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK},
"failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed},
} {
m, n := setup(h.Phase, h.FP, h.Health)
m, n := setup(h.Phase, h.Fp, h.Health)
if out := Wants(m, n, nil, nil); out.Lifted {
t.Errorf("g14 %s: the hold was lifted", what)
}
}
// a failed health on a removed have holds nothing
m, n := setup(delivery.PhaseRemoved, fNew, delivery.HealthFailed)
m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed)
m.Held = false
if out := Wants(m, n, nil, nil); out.Held != nil {
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
@@ -1468,8 +1470,8 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
m, n := x11()
recording(&m)
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord}, true
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true
}
out := Wants(m, n, map[string]delivery.Want{"shanks": {Removed: true}}, nil)
if w, ok := out.Wants["shanks"]; ok {
@@ -1477,7 +1479,7 @@ func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
}
asked := false
for _, r := range out.Record {
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == delivery.JudgedRecreates)
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == JudgedRecreates)
}
if !asked {
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
@@ -1492,11 +1494,11 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
"held at F, running an old-form build": func(m *Module, n *Nodes) {
m.Held, m.Kept = true, true
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
},
"held by this pass": func(_ *Module, n *Nodes) {
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
n.Haves["shanks"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: "build-2", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
},
} {
m, n := x11()
modules/mesh-delivery-assigner/internal/buckets/buckets.go
+53
View File
@@ -0,0 +1,53 @@
// Package buckets holds the names of the bus buckets the assigner reads and the states it keeps beside the seats' own. A
// contract carries the local name of a state, not the module that holds it (ADR 0625 §4, §7), and the assigner opens
// buckets through grants its manifest declares, so the holder's name is kept here, in the assigner's own code, and held equal
// to the manifest by its test (ADR 0634 §1, finding F4).
package buckets
import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
wants "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
delivery "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
plc "mesh-delivery-assigner/internal/placement"
)
// The states the assigner keeps: the wants seat's, and its own asks, puts and policies.
const (
Wants = plc.Module + "_" + wants.StateWants
Differences = plc.Module + "_" + wants.StateDifferences
Asks = plc.Module + "_asks"
Puts = plc.Module + "_puts"
// PoliciesState is the state of the policies a person set for a module's upgrade; Policies its bucket.
PoliciesState = "policies"
Policies = plc.Module + "_" + PoliciesState
)
// AsksHistory is the revisions the bucket of asks keeps per key.
const AsksHistory = 5
// The states of the other seats' holders that the assigner reads, named by the holder's module as every state is.
const (
// Have holds a module's have on a node, `<node>.<module>`, and the node's own facts, `<node>.node`.
Have = "node-engine_" + nodeengine.StateHave
// ComposedRead is the family of the node-engine's composed states, one bucket per node.
ComposedRead = "mesh-host." + nodeengine.StateComposed
// Modules and Manifests are the indexer's.
Modules = "mesh-delivery-indexer_" + catalog.StateModules
Manifests = "mesh-delivery-indexer_" + catalog.StateManifests
// Presence holds each part's presence and the node's.
Presence = "mesh-delivery-watchman_" + watchdogs.StatePresence
// Source and Merges are the gate's.
Source = "mesh-delivery-gate_" + delivery.StateSource
Merges = "mesh-delivery-gate_" + delivery.StateMerges
// Answers holds the teller's answer to each ask.
Answers = "mesh-delivery-teller_answers"
// Builds is the builder's.
Builds = builds.BuilderBuildsBucket
// Grants is the grants part's.
Grants = grantsstate.PartGrantsBucket
)
modules/mesh-delivery-assigner/internal/buckets/buckets_test.go
+22
View File
@@ -0,0 +1,22 @@
package buckets
import "testing"
// The names are the ones the earlier shared form held (mesh-sdk's go/delivery), and the ones the manifest's reads and
// states say: a bucket is `<module>_<state>`.
func TestBucketsAreTheEarlierNames(t *testing.T) {
for name, c := range map[string][2]string{
"Have": {Have, "node-engine_have"}, "ComposedRead": {ComposedRead, "mesh-host.composed"},
"Modules": {Modules, "mesh-delivery-indexer_modules"}, "Manifests": {Manifests, "mesh-delivery-indexer_manifests"},
"Presence": {Presence, "mesh-delivery-watchman_presence"}, "Source": {Source, "mesh-delivery-gate_source"},
"Merges": {Merges, "mesh-delivery-gate_merges"}, "Answers": {Answers, "mesh-delivery-teller_answers"},
"Builds": {Builds, "mesh-delivery-builder_builds"}, "Grants": {Grants, "mesh-grants_grants"},
"Wants": {Wants, "mesh-delivery-assigner_wants"}, "Differences": {Differences, "mesh-delivery-assigner_differences"},
"Asks": {Asks, "mesh-delivery-assigner_asks"}, "Puts": {Puts, "mesh-delivery-assigner_puts"},
"Policies": {Policies, "mesh-delivery-assigner_policies"},
} {
if c[0] != c[1] {
t.Errorf("%s = %q, want %q", name, c[0], c[1])
}
}
}
modules/mesh-delivery-assigner/internal/judge/judge.go
+7 -6
View File
@@ -8,6 +8,7 @@ package judge
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"reflect"
"sort"
"strconv"
@@ -83,7 +84,7 @@ func Intended(current map[string]mma.Placement, laid map[string]*mma.Placement,
// on the bus) is named as giving no answer and the proposal decided on the answers it has (ADR 0593 §8).
// A difference fails nothing: one Intended computed is listed under Intended, one it did not under Unintended, both
// shown in the ask's brief. The seed alone fails on any difference.
func JudgeAnswers(p mma.Proposal, intended map[string]map[string]bool, answers map[string]*delivery.CandidateAnswer,
func JudgeAnswers(p mma.Proposal, intended map[string]map[string]bool, answers map[string]*nodeengine.CandidateAnswer,
revisions map[string]uint64, online map[string]bool, pulsing map[string]bool, now time.Time) mma.Verdict {
v := mma.Verdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{}, Carried: map[string][]string{},
Answers: map[string]map[string]any{}}
@@ -99,7 +100,7 @@ func JudgeAnswers(p mma.Proposal, intended map[string]map[string]bool, answers m
}
for _, node := range must {
a := answers[node]
if a == nil || a.Proposal != p.ID || a.Revision == 0 || a.Revision != revisions[node] {
if a == nil || a.Proposal != p.ID || a.Revision == 0 || uint64(a.Revision) != revisions[node] {
v.Silent = append(v.Silent, node)
continue
}
@@ -236,7 +237,7 @@ func Brief(p mma.Proposal, v mma.Verdict) string {
// differenceWords names one module's difference: the module, the first field that differs in it with its value as it
// runs and as the seed composes it, each cut to about 60 characters ("(value cut)" where a body shown is cut before the
// field), and how many fields more; or the resource and why when no body is shown.
func differenceWords(d delivery.ModuleDiffers) string {
func differenceWords(d nodeengine.CandidateDiffers) string {
for _, b := range d.Bodies {
if len(b.Paths) == 0 {
continue
@@ -336,7 +337,7 @@ func fieldName(running, composed, path string) string {
// answerBody is a node's answer as the verdict carries it: the seat's contract keeps a verdict's answers as bodies, since
// the answer's layout is the node-engine's (ADR 0593 §7).
func answerBody(a delivery.CandidateAnswer) map[string]any {
func answerBody(a nodeengine.CandidateAnswer) map[string]any {
raw, _ := json.Marshal(a)
var out map[string]any
_ = json.Unmarshal(raw, &out)
@@ -344,9 +345,9 @@ func answerBody(a delivery.CandidateAnswer) map[string]any {
}
// answerOf reads a verdict's answer body back; a body that does not fit is an answer with nothing in it.
func answerOf(body map[string]any) delivery.CandidateAnswer {
func answerOf(body map[string]any) nodeengine.CandidateAnswer {
raw, _ := json.Marshal(body)
var out delivery.CandidateAnswer
var out nodeengine.CandidateAnswer
_ = json.Unmarshal(raw, &out)
return out
}
modules/mesh-delivery-assigner/internal/judge/judge_test.go
+22 -21
View File
@@ -1,6 +1,7 @@
package judge
import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"slices"
"strings"
"testing"
@@ -32,38 +33,38 @@ func TestJudge(t *testing.T) {
Must: []string{"anchor", "laptop"}}
revisions := map[string]uint64{"anchor": 7, "laptop": 8}
intended := map[string]map[string]bool{"anchor": {"web": true}}
same := func(rev uint64) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
same := func(rev uint64) *nodeengine.CandidateAnswer {
return &nodeengine.CandidateAnswer{Proposal: "p-one", Revision: int64(rev), Verdict: nodeengine.CandidateAnswerVerdictSame, Same: []string{"mail"}}
}
differs := func(rev uint64, module string) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: module, Why: "the port differs"}}}
differs := func(rev uint64, module string) *nodeengine.CandidateAnswer {
return &nodeengine.CandidateAnswer{Proposal: "p-one", Revision: int64(rev), Verdict: nodeengine.CandidateAnswerVerdictDiffers,
Differs: []nodeengine.CandidateDiffers{{Module: module, Why: "the port differs"}}}
}
for _, row := range []struct {
name string
seed bool
answers map[string]*delivery.CandidateAnswer
answers map[string]*nodeengine.CandidateAnswer
pulsing map[string]bool
state string
words string
marked string
}{
{"all the same", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a module added where intended", false, map[string]*delivery.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: delivery.CandidateDiffers,
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a difference not computed passes, marked", false, map[string]*delivery.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
{"the seed carries a difference, named", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: delivery.CandidateDiffers, Differs: []delivery.ModuleDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
Bodies: []delivery.ResourceBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
{"all the same", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a module added where intended", false, map[string]*nodeengine.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: nodeengine.CandidateAnswerVerdictDiffers,
Added: []nodeengine.CandidateAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a difference not computed passes, marked", false, map[string]*nodeengine.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
{"the seed carries a difference, named", true, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: nodeengine.CandidateAnswerVerdictDiffers, Differs: []nodeengine.CandidateDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
Bodies: []nodeengine.CandidateBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}","MESH_TOOL_MODULES":"… (cut at 8192 bytes)`}}}}}}, nil, delivery.VerdictPass, "", ""},
{"a refusal", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: delivery.CandidateRefused,
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
{"a node not answering, within its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, plc.VerdictWaits, "laptop (offline)", ""},
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*delivery.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
{"a module held as today passes, shown in the brief", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: delivery.CandidateSame, Held: []delivery.ModuleRefused{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
{"its later answer", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a refusal", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: nodeengine.CandidateAnswerVerdictRefused,
Refusals: []nodeengine.CandidateRefusal{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
{"a node not answering, within its bound", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, plc.VerdictWaits, "laptop (offline)", ""},
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
{"a module held as today passes, shown in the brief", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: nodeengine.CandidateAnswerVerdictSame, Held: []nodeengine.CandidateRefusal{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
{"its later answer", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
} {
q := p
q.Seed = row.seed
@@ -77,7 +78,7 @@ func TestJudge(t *testing.T) {
}
if row.seed && row.state == delivery.VerdictPass {
whole := row.answers["laptop"].Differs[0]
whole.Bodies = []delivery.ResourceBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
whole.Bodies = []nodeengine.CandidateBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}"}}`}}
if got := differenceWords(whole); got != "node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → systemd-resolved (node-resolver)" {
modules/mesh-delivery-assigner/manifest_test.go
+11
View File
@@ -6,12 +6,14 @@ import (
"os"
"reflect"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/buckets"
plc "mesh-delivery-assigner/internal/placement"
)
@@ -48,6 +50,15 @@ func TestManifest(t *testing.T) {
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
// the buckets the program names are the manifest's reads, `<module>.<state>` as `<module>_<state>`
named := []string{buckets.Builds, buckets.Merges, buckets.Source, buckets.Answers, buckets.Presence, buckets.Grants,
buckets.ComposedRead, "mesh-provisioner_provisions", buckets.Have}
slices.Sort(named)
for i, r := range want {
if got := strings.Replace(named[i], "_", ".", 1); got != r {
t.Errorf("the buckets package names %q, the manifest reads %q", got, r)
}
}
states := map[string]int{}
for _, s := range m.State {
states[s.Name] = s.History