novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Merge branch 'b7f/registry' into b7/registry

Browse Source
This commit is contained in:
jschoubben
2026-11-05 16:03:33 +01:00
parent 8c2d61a2cf 70865842f2
commit bd6855e1b7
36 changed files with 1981 additions and 1444 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/composed.go
+11 -6
View File
@@ -4,11 +4,12 @@ import (
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
)
// Every node's published compositions, read only to compare (novox/hq ADR 0593 §8): each node's engine writes its own
// `composed-<node>` (delivery.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// `composed-<node>` (buckets.ComposedRead, one bucket per node), and `mesh-wants.upgrade` with compare names, for every
// node, the composition its own have runs beside the newest it published of each module and of the node, so a person sees
// where they part. The assigner composes nothing and never writes these.
@@ -24,7 +25,7 @@ type publishedRow struct {
}
// published is, by node, every composition it published, sorted by key, from every key of the per-node state (a key
// names its node first): each value decoded and digested here, as the node-engine digests it (delivery.ComposedDigestOf).
// names its node first): each value decoded and digested here, as the node-engine digests it (nodeengineclient.ComposedDigest).
// A value or key that does not read is named with why, never left out.
func published(values map[string][]byte) map[string][]publishedRow {
byNode := map[string]map[string][]byte{}
@@ -40,7 +41,11 @@ func published(values map[string][]byte) map[string][]publishedRow {
var rows []publishedRow
for key, raw := range keys {
row := publishedRow{Key: key}
_, module, fp, isNode, err := delivery.ParseComposedKey(key)
_, module, part, isNode, err := nodeengineclient.ParseComposedKey(key)
var fp string
if err == nil && !isNode {
fp, err = meshcatalogclient.ParseKeyPart(part)
}
switch {
case err != nil:
row.Unread = err.Error()
@@ -50,14 +55,14 @@ func published(values map[string][]byte) map[string][]publishedRow {
row.Module, row.FP = module, fp
}
if err == nil {
c, derr := delivery.DecodeComposed(raw)
c, derr := nodeengineclient.DecodeComposition(raw)
switch {
case derr != nil:
row.Unread = derr.Error()
case c.Refused != "":
row.Refused = c.Refused
default:
if d, err := delivery.ComposedDigestOf(c); err != nil {
if d, err := nodeengineclient.ComposedDigest(c); err != nil {
row.Unread = err.Error()
} else {
row.Digest = d
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+220 -201
View File
@@ -7,8 +7,18 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"io"
"maps"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"os"
"path/filepath"
"slices"
@@ -100,8 +110,8 @@ func startBed(t *testing.T, leaveOut string) *bed {
b.buckets[name] = kv
}
// The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog),
Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog),
Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
@@ -232,30 +242,30 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
was := PutsReopenEvery
PutsReopenEvery = 0
defer func() { PutsReopenEvery = was }()
b := startBed(t, delivery.AssignerPutsBucket)
b := startBed(t, buckets.Puts)
done := b.start(t)
select {
case err := <-done:
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
t.Fatalf("without %s the assigner stopped: %v", buckets.Puts, err)
case <-time.After(2 * time.Second):
}
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
putsKV, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: buckets.Puts, History: 5})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.PutKey("n1", "web")
asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
raw, _ := delivery.EncodePut(asked)
if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
pkey, _ := puts.Key("n1", "web")
asked := puts.Put{FP: meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), puts.Asked, "asked")
raw, _ := puts.Encode(asked)
if _, err := putsKV.Put(b.ctx, pkey, raw); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
if _, err := b.buckets[buckets.Presence].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
t.Fatal(err)
}
if e, err := puts.Get(b.ctx, pkey); err == nil {
if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
if e, err := putsKV.Get(b.ctx, pkey); err == nil {
if got, err := puts.Decode(e.Value()); err == nil && got.Step == puts.Refused && strings.Contains(got.Words, "not assigned") {
break
}
}
@@ -282,7 +292,7 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
t.Fatal(err)
}
if nw, err := delivery.DecodeNodeWant(pb.get(t, delivery.AssignerWantsBucket, "want.n1.node")); err != nil || !nw.Self {
if nw, err := mwc.DecodeNodeWant(pb.get(t, buckets.Wants, "want.n1.node")); err != nil || !nw.Self {
t.Fatalf("the placement state made after the start was not read: n1's own want = %+v, %v", nw, err)
}
}
@@ -322,29 +332,29 @@ func (b *bed) place(t *testing.T, node, module string) {
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
func (b *bed) compose(t *testing.T, node string, d byte) string {
t.Helper()
key, err := delivery.HaveNodeKey(node)
key, err := nodeengineclient.HaveNodeKey(node)
if err != nil {
t.Fatal(err)
}
digest := "sha256:" + strings.Repeat(string(d), 64)
v, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
v, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, key, v)
b.putJSON(t, buckets.Have, key, v)
return digest
}
// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest.
func (b *bed) manifest(t *testing.T, module, body string) string {
t.Helper()
digest := delivery.ManifestDigest([]byte(body))
key, _ := delivery.ManifestsKey(digest)
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
digest := meshcatalogclient.ManifestDigest([]byte(body))
key, _ := meshcatalogclient.ManifestsKey(digest)
value, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: body})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ManifestsBucket, key, value)
b.putJSON(t, buckets.Manifests, key, value)
return digest
}
@@ -381,28 +391,28 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
}
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
fNew, fOld := meshcatalogclient.SourcePrefix2+strings.Repeat("b", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "web", mod)
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putJSON(t, buckets.Modules, "web", mod)
built, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
if err != nil {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
bkey, _ := builds.BuilderBuildKey("web", fNew)
b.putJSON(t, buckets.Builds, bkey, built)
for _, fp := range []string{fNew, fOld} {
gkey, _ := delivery.PartGrantKey("web", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
}
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
for _, node := range []string{"ace", "g14", "shanks"} {
b.compose(t, node, 'c')
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, node, on)
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, node+".web", have)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, node, on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: runs[node], Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, node+".web", have)
b.place(t, node, "web")
}
b.compose(t, "shanks", 'a')
@@ -433,12 +443,12 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
}
wkey, _ := delivery.WantKey("ace", "web")
want, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey))
if err != nil || want.FP != fNew {
wkey, _ := mwc.WantKey("ace", "web")
want, err := mwc.DecodeWant(b.get(t, buckets.Wants, wkey))
if err != nil || want.Fp != fNew {
t.Fatalf("the canary ace's want = %+v, %v", want, err)
}
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w)
}
// DLV-O5 (ADR 0615 §5, ADR 0619 §2): each want is said on the assigner's views family, its value whole at its revision,
@@ -455,18 +465,18 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if neither.Subject.Machine != "g14" || !strings.HasPrefix(neither.Subject.ID, "g14.web") || neither.Severity != occ.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
}
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
if keys, err := b.buckets[buckets.Differences].Keys(b.ctx); err == nil && len(keys) != 0 {
t.Errorf("differences were written: %v", keys)
}
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
if e, err := b.buckets[buckets.Wants].Get(b.ctx, "want.g14.web"); err == nil {
t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
}
// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
// there by itself: the condition is cleared.
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
healthyF, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthyF)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w mwc.Want) bool { return w.Fp != "" }); w.Fp != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
t.Errorf("g14's want = %+v, want F, the node composing it", w)
}
cleared := false
@@ -489,14 +499,14 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
pkey, _ := mmac.LegacyKey(node)
b.putJSON(t, plc.PlacementBucket, pkey, pl)
}
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew {
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w mwc.Want) bool { return w.Self }); w.Composed != "" || w.Fp != fNew {
t.Errorf("ace's want once placed = %+v", w)
}
nkey, _ := delivery.WantNodeKey("ace")
nkey, _ := mwc.WantNodeKey("ace")
var self bool
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
nw, err := delivery.DecodeNodeWant(e.Value())
if e, err := b.buckets[buckets.Wants].Get(b.ctx, nkey); err == nil {
nw, err := mwc.DecodeNodeWant(e.Value())
self = err == nil && nw.Self
}
}
@@ -524,38 +534,38 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
// The provisioner keys it from the manifests, as soon as they are known and whether the provider is placed yet or not.
asked(provisions.Asked, "the provider's build is not done")
fpOf := func(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
fpOf := func(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) }
toolOld := fpOf('1')
for module, row := range map[string]struct {
fp, digest string
}{"db": {fpOf('2'), dbDigest}, "app": {fpOf('3'), appDigest}, "tool": {fpOf('4'), toolDigest}} {
mod, _ := delivery.EncodeModule(delivery.Module{FP2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, module, mod)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: meshcatalogclient.RootPath})
b.putJSON(t, buckets.Modules, module, mod)
fps := []string{row.fp}
if module == "tool" {
fps = append(fps, toolOld)
}
for _, fp := range fps {
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest})
bkey, _ := delivery.BuilderBuildKey(module, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.PartGrantKey(module, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest})
bkey, _ := builds.BuilderBuildKey(module, fp)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := grantsstate.PartGrantKey(module, fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
}
b.place(t, "ace", module)
}
runsOld, _ := delivery.EncodeHave(delivery.Have{FP: toolOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.tool", runsOld)
if w := b.wantNow(t, "ace", "tool", "the build module's want at the second build", func(w delivery.Want) bool { return w.FP == fpOf('4') }); w.FP != fpOf('4') {
runsOld, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: toolOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.tool", runsOld)
if w := b.wantNow(t, "ace", "tool", "the build module's want at the second build", func(w mwc.Want) bool { return w.Fp == fpOf('4') }); w.Fp != fpOf('4') {
t.Errorf("tool's want = %+v", w)
}
b.wantNow(t, "ace", "db", "the provider's want at its fingerprint", func(w delivery.Want) bool { return w.FP == fpOf('2') })
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.ace.app"); err == nil {
b.wantNow(t, "ace", "db", "the provider's want at its fingerprint", func(w mwc.Want) bool { return w.Fp == fpOf('2') })
if e, err := b.buckets[buckets.Wants].Get(b.ctx, "want.ace.app"); err == nil {
t.Errorf("app was wanted while its provision was asked: %s", e.Value())
}
asked(provisions.Provided, "")
if w := b.wantNow(t, "ace", "app", "the consumer's want once its provision is provided", func(w delivery.Want) bool { return w.FP == fpOf('3') }); w.FP != fpOf('3') {
if w := b.wantNow(t, "ace", "app", "the consumer's want once its provision is provided", func(w mwc.Want) bool { return w.Fp == fpOf('3') }); w.Fp != fpOf('3') {
t.Errorf("app's want = %+v", w)
}
@@ -588,29 +598,29 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
formsOld, formsNew := fpOf('5'), fpOf('6')
formsDigest := b.manifest(t, "forms", `{"module":"forms","version":"1"}`)
formsAt := func(fp string) {
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: formsDigest, Repository: "novox/forms", Commit: strings.Repeat("3", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "forms", mod)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: formsDigest, Repository: "novox/forms", Commit: strings.Repeat("3", 40), Path: meshcatalogclient.RootPath})
b.putJSON(t, buckets.Modules, "forms", mod)
}
for _, fp := range []string{formsOld, formsNew} {
build := delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: formsDigest}
build := builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: formsDigest}
if fp == formsNew {
build.Needs = []string{"form:new"}
}
built, err := delivery.EncodeBuilderBuild(build)
built, err := builds.EncodeBuilderBuild(build)
if err != nil {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey("forms", fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.PartGrantKey("forms", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
bkey, _ := builds.BuilderBuildKey("forms", fp)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := grantsstate.PartGrantKey("forms", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
}
runsFormsOld, _ := delivery.EncodeHave(delivery.Have{FP: formsOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.forms", runsFormsOld)
runsFormsOld, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: formsOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.forms", runsFormsOld)
b.place(t, "ace", "forms")
formsAt(formsOld)
b.wantNow(t, "ace", "forms", "the want at what ace runs, a build in the first form", func(w delivery.Want) bool { return w.FP == formsOld })
b.wantNow(t, "ace", "forms", "the want at what ace runs, a build in the first form", func(w mwc.Want) bool { return w.Fp == formsOld })
formsAt(formsNew)
statement := func(node string, ok func(enginereads.Statement) bool) enginereads.Statement {
var last enginereads.Statement
@@ -628,16 +638,16 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if s := statement("g14", func(s enginereads.Statement) bool { return !s.Answered }); s.From != enginereads.FromNone {
t.Errorf("g14, whose engine answers nothing and which has no record, reads %+v", s)
}
if w := b.wantNow(t, "ace", "forms", "the want held on ace, naming the form", func(w delivery.Want) bool {
if w := b.wantNow(t, "ace", "forms", "the want held on ace, naming the form", func(w mwc.Want) bool {
return w.Newest == formsNew && strings.Contains(w.Words, "form:new")
}); w.FP != formsOld || w.Rule != delivery.RuleComposed {
}); w.Fp != formsOld || w.Rule != mwc.RuleComposed {
t.Errorf("forms' want = %+v, want it held at what ace runs under rule 7", w)
}
statedMu.Lock()
stated = append(slices.Clone(stated), "form:new")
statedMu.Unlock()
say(enginereads.ReadsStarted)
b.wantNow(t, "ace", "forms", "the want moved once ace's engine said it reads the form", func(w delivery.Want) bool { return w.FP == formsNew })
b.wantNow(t, "ace", "forms", "the want moved once ace's engine said it reads the form", func(w mwc.Want) bool { return w.Fp == formsNew })
// assigner_engine_reads asks ace's engine again through the state: the assigner answers there with what it states now.
again, err := enginereads.AskAgainOf(bedState{b, engineReadsBucket}, "ace", "n1.ada", "a person asks", time.Now(), 15*time.Second)
if err != nil || again.From != enginereads.FromVerb || !slices.Contains(again.ManifestReads, "form:new") || again.AskAgain != nil {
@@ -677,25 +687,25 @@ func (s bedState) Await(key string, after uint64, done func([]byte) bool, bound
// the one its manifest declares (ownBuckets, held equal to the manifest by TestOpensWhatItsManifestNames).
func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("c", 64)
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "web", mod)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putJSON(t, buckets.Modules, "web", mod)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.place(t, "ace", "web")
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
hkey, _ := mwc.HeldKey("web", fNew)
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
if h, err := mwc.DecodeHeld(b.get(t, buckets.Wants, hkey)); err != nil || h.Node != "ace" ||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
t.Fatalf("held = %+v, %v", h, err)
}
akey, _ := delivery.HoldAskKey("web", fNew)
a, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
akey, _ := asks.HoldAskKey("web", fNew)
a, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil || a.Asking <= 0 || a.Ask.ID != meshdeliveryclient.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
t.Fatalf("the ask = %+v, %v", a, err)
}
}
@@ -735,26 +745,26 @@ func (b *bed) conditionsSaid(t *testing.T) map[string]oc.Condition {
// and the check's words.
func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64)
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "mail", mod)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.mail", failed)
wantKey, _ := delivery.WantKey("ace", "mail")
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putJSON(t, buckets.Modules, "mail", mod)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.mail", failed)
wantKey, _ := mwc.WantKey("ace", "mail")
want, _ := mwc.EncodeWant(mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
b.putJSON(t, buckets.Wants, wantKey, want)
b.place(t, "ace", "mail")
b.start(t)
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
a, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
akey, _ := asks.RestoreAskKey("ace", "mail", fNew)
a, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
t.Fatalf("the restore ask = %+v, %v", a, err)
}
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew {
if w, err := mwc.DecodeWant(b.get(t, buckets.Wants, wantKey)); err != nil || w.Fp != fNew {
t.Errorf("the data module was set back: want = %+v, %v", w, err)
}
key := ""
@@ -779,30 +789,30 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
b := startBed(t, "")
const modules = 40
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
names := make([]string, modules)
for i := range names {
name := fmt.Sprintf("mod%02d", i)
names[i] = name
fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name))
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putJSON(t, buckets.Modules, name, mod)
built, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
if err != nil {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.PartGrantKey(name, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
bkey, _ := builds.BuilderBuildKey(name, fp)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := grantsstate.PartGrantKey(name, fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
b.compose(t, "ace", 'c')
b.place(t, "ace", name)
}
count := func() int {
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
keys, err := b.buckets[buckets.Wants].Keys(b.ctx)
if err != nil {
return 0
}
@@ -817,8 +827,8 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
}
time.Sleep(time.Second) // and no pass of the second writes any again
for _, name := range names {
key, _ := delivery.WantKey("ace", name)
history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key)
key, _ := mwc.WantKey("ace", name)
history, err := b.buckets[buckets.Wants].History(b.ctx, key)
if err != nil {
t.Errorf("the want of %s was never written: %v", name, err)
continue
@@ -833,37 +843,37 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
t.Helper()
digest := b.manifest(t, name, manifest)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.PartGrantKey(name, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putJSON(t, buckets.Modules, name, mod)
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := builds.BuilderBuildKey(name, fp)
b.putJSON(t, buckets.Builds, bkey, built)
gkey, _ := grantsstate.PartGrantKey(name, fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
b.place(t, "ace", name)
}
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
func (b *bed) putWant(t *testing.T, node, module string, w mwc.Want) {
t.Helper()
key, _ := delivery.WantKey(node, module)
v, err := delivery.EncodeWant(w)
key, _ := mwc.WantKey(node, module)
v, err := mwc.EncodeWant(w)
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AssignerWantsBucket, key, v)
b.putJSON(t, buckets.Wants, key, v)
}
// wantNow waits until the want of module on node satisfies ok and answers it.
func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.Want) bool) delivery.Want {
func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(mwc.Want) bool) mwc.Want {
t.Helper()
key, _ := delivery.WantKey(node, module)
var last delivery.Want
key, _ := mwc.WantKey(node, module)
var last mwc.Want
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil {
if w, err := delivery.DecodeWant(e.Value()); err == nil {
if e, err := b.buckets[buckets.Wants].Get(b.ctx, key); err == nil {
if w, err := mwc.DecodeWant(e.Value()); err == nil {
last = w
if ok(w) {
return w
@@ -878,7 +888,7 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
func (b *bed) answer(t *testing.T, key, option string) {
t.Helper()
a, err := decodeAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
a, err := decodeAsk(key, b.get(t, buckets.Asks, key))
if err != nil {
t.Fatal(err)
}
@@ -892,7 +902,7 @@ func (b *bed) answerTo(t *testing.T, key, option, askID string) {
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AnswersBucket, key, v)
b.putJSON(t, buckets.Answers, key, v)
}
func (b *bed) hasKey(bucket, key string) bool {
@@ -904,24 +914,24 @@ func (b *bed) hasKey(bucket, key string) bool {
// and clears the condition.
func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
fNew, fOld := meshcatalogclient.SourcePrefix2+strings.Repeat("b", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.compose(t, "ace", 'a')
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld })
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w mwc.Want) bool { return w.Fp == fOld })
if w.Newest != fNew {
t.Fatalf("the set-back want = %+v", w)
}
hold, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerAsksBucket, hold)
hold, _ := asks.HoldAskKey("web", fNew)
b.get(t, buckets.Asks, hold)
b.answer(t, hold, "release")
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w mwc.Want) bool { return w.Fp == fNew })
if w.Composed != "" || !w.Self || w.Rule != 0 {
t.Errorf("the want = %+v, want F by the rules, the node composing it", w)
}
@@ -941,7 +951,7 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
func (b *bed) askNow(t *testing.T, key string, asking int) decodedAsk {
t.Helper()
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil {
if e, err := b.buckets[buckets.Asks].Get(b.ctx, key); err == nil {
if a, err := decodeAsk(key, e.Value()); err == nil && a.Asking == asking {
return a
}
@@ -956,38 +966,38 @@ func (b *bed) askNow(t *testing.T, key string, asking int) decodedAsk {
// value, so the node retries.
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Self: true}) // a placed node's want names the build alone
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Self: true}) // a placed node's want names the build alone
held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"})
b.putJSON(t, buckets.Have, "ace.web", held)
b.start(t)
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
akey, _ := asks.BackupAskKey("ace", "web", fNew)
a := b.askNow(t, akey, 0)
if !strings.Contains(a.Ask.Explanation, "no space left") {
t.Fatalf("the backup ask = %+v", a)
}
// an answer to another ask is no answer to this one
b.answerTo(t, akey, delivery.BackupSkip, "someone-else")
b.answerTo(t, akey, mwc.BackupSkip, "someone-else")
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
if w := b.wantNow(t, "ace", "web", "the want", func(mwc.Want) bool { return true }); w.Backup != "" {
t.Fatalf("an answer to another ask was taken: %+v", w)
}
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
if w1.FP != fNew || w1.Composed != "" || !w1.Self {
b.answerTo(t, akey, mwc.BackupAgain, a.Ask.ID)
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w mwc.Want) bool { return w.Backup == mwc.BackupAgain })
if w1.Fp != fNew || w1.Composed != "" || !w1.Self {
t.Errorf("the want = %+v", w1)
}
// the node retries and the backup fails again: a new have value, asked again under the next number
again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, delivery.HaveBucket, "ace.web", again)
again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, buckets.Have, "ace.web", again)
a2 := b.askNow(t, akey, 2)
if a2.Ask.ID == a.Ask.ID {
t.Fatalf("asked again under the same id")
}
b.answerTo(t, akey, delivery.BackupAgain, a2.Ask.ID)
w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain && w.Words != w1.Words })
b.answerTo(t, akey, mwc.BackupAgain, a2.Ask.ID)
w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w mwc.Want) bool { return w.Backup == mwc.BackupAgain && w.Words != w1.Words })
if w2.Words == w1.Words {
t.Errorf("the second answer left the want as it was: %+v", w2)
}
@@ -1000,55 +1010,55 @@ const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","wh
// cleared saying why, and the want stays at the fingerprint.
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerWantsBucket, hkey)
b.get(t, delivery.AssignerAsksBucket, akey)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
hkey, _ := mwc.HeldKey("web", fNew)
akey, _ := asks.HoldAskKey("web", fNew)
b.get(t, buckets.Wants, hkey)
b.get(t, buckets.Asks, akey)
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
if b.hasKey(buckets.Wants, hkey) || b.hasKey(buckets.Asks, akey) {
continue
}
for k, c := range b.conditionsSaid(t) {
if strings.HasPrefix(k, "module.web.") && c.State == occ.StateCleared && strings.HasSuffix(k, ".held") {
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w mwc.Want) bool { return w.Fp == fNew }); w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
return
}
}
}
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(buckets.Wants, hkey), b.hasKey(buckets.Asks, akey))
}
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
healthy, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
_, stop := b.startStoppable(t)
hkey, _ := delivery.HeldKey("web", fNew)
akey, _ := delivery.HoldAskKey("web", fNew)
first, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
hkey, _ := mwc.HeldKey("web", fNew)
akey, _ := asks.HoldAskKey("web", fNew)
first, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the hold was not lifted")
}
@@ -1056,17 +1066,17 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
stop()
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putJSON(t, buckets.Have, "ace.web", failed)
b.start(t)
again, err := decodeAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
again, err := decodeAsk(akey, b.get(t, buckets.Asks, akey))
if err != nil {
t.Fatal(err)
}
if again.Ask.ID == first.Ask.ID {
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
}
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
b.putJSON(t, buckets.Have, "ace.web", healthy)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(buckets.Wants, hkey); time.Sleep(100 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
}
@@ -1135,17 +1145,21 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}})
b.putJSON(t, delivery.HaveBucket, node+".node", rec)
rec, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node}})
b.putJSON(t, buckets.Have, node+".node", rec)
}
b.putJSON(t, delivery.ModulesBucket, "web", []byte(`{"v":1}`))
b.putJSON(t, delivery.ModulesBucket, "shell", []byte(`{"v":1}`))
b.putJSON(t, buckets.Modules, "web", []byte(`{"v":1}`))
b.putJSON(t, buckets.Modules, "shell", []byte(`{"v":1}`))
b.putJSON(t, buckets.Modules, "mesh-enrolment", []byte(`{"v":1}`))
// place writes a node's placement as another writer would, in its node and module keys (research 072 row 37), and
// answers its node key's revision; legacy writes it in the form before, placement.<node>, alone.
place := func(node string, legacy bool, modules ...string) uint64 {
p := mma.Placement{Node: node, Modules: []mma.PlacedModule{}}
for _, m := range modules {
p.Modules = append(p.Modules, mma.PlacedModule{Module: m})
if m == "mesh-enrolment" { // the module that holds the seat mesh-node-enroller: who may add a node is read from it
p.Holds = append(p.Holds, mma.Holding{Claim: "mesh-node-enroller", Scope: "mesh", Module: m})
}
}
values, _ := mmac.PlacementValues(p)
key, _ := mmac.NodeKey(node)
@@ -1219,9 +1233,9 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
if err != nil {
t.Fatalf("no candidate %s: %v", key, err)
}
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}, Candidates: map[string]delivery.CandidateAnswer{
p.ID: {Proposal: p.ID, Revision: e.Revision(), Verdict: delivery.CandidateSame, Same: []string{"node"}}}})
b.putJSON(t, delivery.HaveBucket, node+".node", have)
have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node}, Candidates: map[string]nodeengine.CandidateAnswer{
p.ID: {Proposal: p.ID, Revision: int64(e.Revision()), Verdict: nodeengine.CandidateAnswerVerdictSame, Same: []string{"node"}}}})
b.putJSON(t, buckets.Have, node+".node", have)
}
}
@@ -1231,8 +1245,8 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
t.Fatalf("must answer = %v, want both nodes", p.Must)
}
answerAll(p)
p = proposalNow(p.ID, "asked", func(p mma.Proposal) bool { return p.State == plc.ProposalAsked })
akey, _ := delivery.PlacementAskKey(p.ID)
p = proposalNow(p.ID, "asked", func(p mma.Proposal) bool { return p.State == mmac.ProposalAsked })
akey, _ := asks.PlacementAskKey(p.ID)
b.answer(t, akey, assign.OptionYes)
select {
case e := <-heard:
@@ -1275,8 +1289,8 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
return strings.Contains(p.Words, "laid again") && len(p.Must) > 0
})
answerAll(relaid)
proposalNow(moved.ID, "asked", func(p mma.Proposal) bool { return p.State == plc.ProposalAsked })
akey, _ = delivery.PlacementAskKey(moved.ID)
proposalNow(moved.ID, "asked", func(p mma.Proposal) bool { return p.State == mmac.ProposalAsked })
akey, _ = asks.PlacementAskKey(moved.ID)
b.answer(t, akey, assign.OptionNo)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(plc.ProposalsBucket, moved.ID); time.Sleep(50 * time.Millisecond) {
if time.Now().After(deadline) {
@@ -1295,7 +1309,7 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
t.Fatal(err)
}
silent := q.(proposed).Proposal
decided := proposalNow(silent.ID, "decided without its silent nodes", func(p mma.Proposal) bool { return p.State == plc.ProposalAsked })
decided := proposalNow(silent.ID, "decided without its silent nodes", func(p mma.Proposal) bool { return p.State == mmac.ProposalAsked })
if decided.Verdict == nil || decided.Verdict.State != delivery.VerdictPass ||
!strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "no answer from") || !strings.Contains(strings.Join(decided.Verdict.Notes, "; "), "laptop (offline)") {
t.Errorf("the silent node's verdict = %+v", decided.Verdict)
@@ -1307,6 +1321,11 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
for len(heard) > 0 {
<-heard
}
// The module holding the seat mesh-node-enroller is the one whose add-node is taken, whatever its name: it is placed
// now, on a node of its own.
hubRecord, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "hub"}})
b.putJSON(t, buckets.Have, "hub.node", hubRecord)
place("hub", false, "mesh-enrolment")
joined, err := verbs.doPropose(map[string]any{"why": "enrolment of ace", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`,
"changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"overlay":"dHVubmVs"}}]`}, "novox.mesh-enrolment")
if err != nil {
@@ -1323,7 +1342,7 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
if ace, err := b.buckets[plc.PlacementBucket].Get(b.ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), "10.9.0.2") {
t.Errorf("the joining node's key = %v, %v", ace, err)
}
if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(plc.ProposalsBucket, joined.(proposed).Proposal.ID) {
if akey, _ := asks.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(buckets.Asks, akey) || b.hasKey(plc.ProposalsBucket, joined.(proposed).Proposal.ID) {
t.Error("an ask was raised for the joining node, or its proposal stays open")
}
// Its enrolment ends before it confirms: a remove-node on the same warrant deletes its key at once, with no ask, and
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+46 -45
View File
@@ -12,6 +12,7 @@ import (
"fmt"
"io"
"log"
"mesh-delivery-assigner/internal/buckets"
"os"
"os/signal"
"slices"
@@ -58,24 +59,24 @@ func main() {
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
buckets.Grants,
buckets.Answers,
buckets.Builds,
buckets.Presence,
buckets.Have,
provisions.ProvisionsBucket,
delivery.SourceBucket,
delivery.MergesBucket,
buckets.Source,
buckets.Merges,
}
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
// node's compositions, which mesh-wants.upgrade compares (novox/hq ADR 0593 §8).
var readFamilies = []string{delivery.ComposedRead}
var readFamilies = []string{buckets.ComposedRead}
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
// (ADR 0567 §1), named after the assigner as every module state is.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, plc.PlacementBucket, plc.ProposalsBucket,
var ownBuckets = []string{buckets.Asks, buckets.Differences,
buckets.Puts, buckets.Wants, plc.PlacementBucket, plc.ProposalsBucket,
plc.CandidatesBucket, plc.PulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, plc.SeatHoldersBucket,
engineReadsBucket}
@@ -106,25 +107,25 @@ var keptStates = []announce.State{
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
{Name: health.FindingsState, Readable: true},
{Name: delivery.AssignerPoliciesState, Readable: true},
{Name: buckets.PoliciesState, Readable: true},
{Name: mma.StateSeatHolders, Readable: true},
{Name: views.MarksState, Readable: true}, // what its views family said each state up to (views.go)
{Name: views.StatusState, Readable: true}, // its first views of the indexer's states (views.go)
{Name: engineReadsState, Readable: true},
}
// buckets is every bucket the assigner opened, by name.
type buckets map[string]jetstream.KeyValue
// kvSet is every bucket the assigner opened, by name.
type kvSet map[string]jetstream.KeyValue
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
out := buckets{}
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (kvSet, error) {
out := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kv, err := js.KeyValue(ctx, name)
if name == delivery.AssignerPutsBucket && err != nil {
if name == buckets.Puts && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
@@ -157,7 +158,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil {
if (name == buckets.Source || name == buckets.Merges) && err != nil {
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
// a record module is asked of a person, as before.
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
@@ -169,7 +170,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err)
continue
}
if name == delivery.PartGrantsBucket && err != nil {
if name == buckets.Grants && err != nil {
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
continue
@@ -187,10 +188,10 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
func startsWithout(name string) bool {
switch name {
case plc.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, engineReadsBucket:
buckets.Puts, buckets.Source, buckets.Merges, engineReadsBucket:
return true
}
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
return slices.Contains(seatBuckets, name) || name == buckets.Grants
}
// run is the program: it returns when ctx ends or the bus cannot be used.
@@ -241,7 +242,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
func findingsOf(ctx context.Context, kvs kvSet, logf func(string, ...any)) assign.Findings {
var f assign.Findings
if kv := kvs[assign.FindingsBucket]; kv != nil {
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
@@ -268,29 +269,29 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
delivery.PartGrantsBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
buckets.Grants,
buckets.Answers,
buckets.Builds,
plc.PlacementBucket,
plc.ProposalsBucket,
plc.CandidatesBucket,
plc.PulsesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
delivery.AssignerDifferencesBucket,
delivery.AssignerWantsBucket,
buckets.Presence,
buckets.Have,
buckets.Asks,
buckets.Differences,
buckets.Wants,
assign.PoliciesBucket,
provisions.ProvisionsBucket,
plc.SeatHoldersBucket,
delivery.SourceBucket,
delivery.MergesBucket,
buckets.Source,
buckets.Merges,
engineReadsBucket,
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *indexerViews, logger *log.Logger) error {
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs kvSet, say func(oc.Condition) error, findings assign.Findings, bus caller, iv *indexerViews, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -317,7 +318,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
}
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
optional := name == delivery.AssignerPutsBucket
optional := name == buckets.Puts
go func() {
defer kw.Stop()
for {
@@ -357,10 +358,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
// stops the puts, never the wants.
if kvs[delivery.AssignerPutsBucket] != nil {
if err := watch(delivery.AssignerPutsBucket); err != nil {
if kvs[buckets.Puts] != nil {
if err := watch(buckets.Puts); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
delete(kvs, buckets.Puts)
}
}
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
@@ -369,24 +370,24 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// further, without a restart.
var triedPuts time.Time
reopenPuts := func(p *passer) {
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
if kvs[buckets.Puts] != nil || time.Since(triedPuts) < PutsReopenEvery {
return
}
triedPuts = time.Now()
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
kv, err := js.KeyValue(ctx, buckets.Puts)
if err != nil {
return
}
kvs[delivery.AssignerPutsBucket] = kv
if err := watch(delivery.AssignerPutsBucket); err != nil {
kvs[buckets.Puts] = kv
if err := watch(buckets.Puts); err != nil {
logger.Printf("%v: no put is taken further until it is opened again", err)
delete(kvs, delivery.AssignerPutsBucket)
delete(kvs, buckets.Puts)
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
logger.Printf("the bucket %s is open: puts are taken further", buckets.Puts)
}
started := 0
for _, name := range watched {
@@ -395,7 +396,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
started++
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket || name == delivery.SourceBucket {
if name == buckets.Have || name == buckets.Source {
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
opts = append(opts, jetstream.IncludeHistory())
}
@@ -427,7 +428,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, plc.SeatHoldersBucket} {
for _, name := range []string{buckets.Grants, assign.PoliciesBucket, provisions.ProvisionsBucket, plc.SeatHoldersBucket} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
@@ -459,7 +460,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if js != nil {
hearReadsChanged(ctx, js, hub, reads, findings, logger.Printf)
}
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
if bus != nil && kvs[buckets.Puts] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
if bus != nil {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go
+9 -9
View File
@@ -2,6 +2,7 @@ package main
import (
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
plc "mesh-delivery-assigner/internal/placement"
"context"
@@ -23,7 +24,6 @@ import (
mmac "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -87,12 +87,12 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
starts bool
said string
}{
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
{buckets.Puts, jetstream.ErrBucketNotFound, true, "no put is taken further"},
{buckets.Puts, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{buckets.Puts, context.DeadlineExceeded, true, "no put is taken further"},
{buckets.Wants, errors.New("nats: permissions violation"), false, ""},
// Issue 774: the grants part's grants not there do not stop the start.
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
{buckets.Grants, jetstream.ErrBucketNotFound, true, "no grant is read"},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
@@ -108,7 +108,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
wrote := make(chan string, 8)
kvs := buckets{}
kvs := kvSet{}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
@@ -120,8 +120,8 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
kv := kvs[plc.PlacementBucket].(*watchKV)
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
kvs[buckets.Puts].(*watchKV).closed = true
kvs[buckets.Wants].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(oc.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+95 -89
View File
@@ -5,7 +5,13 @@ import (
"context"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
"strings"
"time"
@@ -24,7 +30,7 @@ import (
// passer makes the assigner's passes: from what the world holds now, it writes each key whose value differs from the
// one written, and only those (the Same functions).
type passer struct {
kvs buckets
kvs kvSet
w *world
logf func(string, ...any)
sdk string
@@ -124,19 +130,19 @@ func (p *passer) pass(ctx context.Context) {
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
func (p *passer) nodeWants(ctx context.Context) {
written := p.w.read(delivery.AssignerWantsBucket)
written := p.w.read(buckets.Wants)
placed := p.w.placedNodes()
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
for node := range placed {
key, err := delivery.WantNodeKey(node)
key, err := mwc.WantNodeKey(node)
if err != nil {
continue
}
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
if before, err := mwc.DecodeNodeWant(written[key]); err == nil && before.Self {
continue
}
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
want := mwc.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeNodeWant(want) })
}
}
@@ -144,7 +150,7 @@ func (p *passer) nodeWants(ctx context.Context) {
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
func (p *passer) wants(ctx context.Context) {
get := p.w.getter(delivery.ManifestsBucket) // the indexer's manifests, as its first view and views family say them
get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them
assigned := p.w.assigned()
trunk := p.w.trunk()
manifests := map[string]catalogue.Manifest{}
@@ -198,35 +204,35 @@ func (p *passer) wants(ctx context.Context) {
served := p.served(ctx, get, moduleGraph, nodes)
offering := p.offering(ctx, get, manifests, nodes) // bus state (ADR 0614 §5)
for name := range names {
fp := trunk[name].FP2
fp := trunk[name].Fp2
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp),
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
Composed: func(node, fp string) (assign.ComposedDigest, bool) {
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
},
GrantedAt: func(fp string) bool {
key, err := delivery.PartGrantKey(name, fp)
key, err := grantsstate.PartGrantKey(name, fp)
return err == nil && p.w.has(grants, key)
},
Restored: func(node string) bool {
key, err := delivery.RestoreAskKey(node, name, fp)
key, err := asks.RestoreAskKey(node, name, fp)
return err == nil && p.answeredWith(key, assign.OptionRestore)
},
Allowed: func(node, digest string) bool {
key, err := delivery.RecordAskKey(node, name, digest)
key, err := asks.RecordAskKey(node, name, digest)
return err == nil && p.answeredWith(key, assign.OptionYes)
},
Release: func(fp string) (string, bool) { return p.w.release(name, fp) }}
if fp != "" {
held, _ := delivery.HeldKey(name, fp)
grant, _ := delivery.PartGrantKey(name, fp)
hold, _ := delivery.HoldAskKey(name, fp)
held, _ := mwc.HeldKey(name, fp)
grant, _ := grantsstate.PartGrantKey(name, fp)
hold, _ := asks.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant)
if raw := p.w.value(grants, grant); raw != nil {
if g, err := delivery.DecodeGrant(raw); err == nil {
if g, err := grantsstate.DecodeGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
}
}
@@ -261,10 +267,10 @@ func (p *passer) wants(ctx context.Context) {
}
}
if out.Held != nil {
key, err := delivery.HeldKey(name, fp)
key, err := mwc.HeldKey(name, fp)
if err == nil {
held := *out.Held
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeHeld(held) })
}
}
if out.Lifted {
@@ -281,13 +287,13 @@ func (p *passer) wants(ctx context.Context) {
p.recordAsk(ctx, name, fp, need)
}
for node, want := range out.Wants {
key, err := delivery.WantKey(node, name)
key, err := mwc.WantKey(node, name)
if err != nil {
p.logf("no want of %s on %s is written: %v", name, node, err)
continue
}
want := want
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeWant(want) })
}
}
}
@@ -295,7 +301,7 @@ func (p *passer) wants(ctx context.Context) {
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
func (p *passer) sayNoWant(module string, nodes []string, before map[string]mwc.Want, out assign.Outcome) {
if p.noWant == nil {
p.noWant = map[string]string{}
}
@@ -419,7 +425,7 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
// left it with none).
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
func leftNeither(into map[string]string, m assign.Module, before map[string]mwc.Want, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; on {
continue
@@ -544,9 +550,9 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
// node (wants are only compared, ADR 0429 §7), and the options say so.
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
key, err := delivery.HoldAskKey(module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *mwc.Held) {
key, err := asks.HoldAskKey(module, fp)
if err != nil || p.w.has(buckets.Asks, key) {
return
}
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
@@ -555,18 +561,18 @@ func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery
asking := int(now.Unix())
h := p.heldHold(module, fp, fresh)
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
func (p *passer) lift(ctx context.Context, module, fp string) {
held, err := delivery.HeldKey(module, fp)
held, err := mwc.HeldKey(module, fp)
if err != nil {
return
}
ask, err := delivery.HoldAskKey(module, fp)
ask, err := asks.HoldAskKey(module, fp)
if err != nil {
return
}
@@ -583,13 +589,13 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
p.said[c.Key+".lifted"] = true
delete(p.said, c.Key+".raised") // a hold written again is said again
}
p.remove(ctx, delivery.AssignerWantsBucket, held)
p.remove(ctx, delivery.AssignerAsksBucket, ask)
p.remove(ctx, buckets.Wants, held)
p.remove(ctx, buckets.Asks, ask)
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
// set back from it (issue 1906, note 28).
for key := range p.w.read(delivery.AssignerAsksBucket) {
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
p.remove(ctx, delivery.AssignerAsksBucket, key)
for key := range p.w.read(buckets.Asks) {
if k, err := asks.ParseAnyAskKey(key); err == nil && k.Kind == asks.AskKindRestore && k.Module == module && k.FP == fp {
p.remove(ctx, buckets.Asks, key)
}
}
}
@@ -599,16 +605,16 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
// nothing taken away.
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
k, err := delivery.ParseAnyAskKey(key)
func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModuleValue) bool {
k, err := asks.ParseAnyAskKey(key)
if err != nil {
return false
}
switch k.Kind {
case delivery.AskKindHold, delivery.AskKindRestore:
case asks.AskKindHold, asks.AskKindRestore:
m, on := trunk[k.Module]
return !on || (m.FP2 != "" && m.FP2 != k.FP)
case delivery.AskKindRecord:
return !on || (m.Fp2 != "" && m.Fp2 != k.FP)
case asks.AskKindRecord:
m, on := trunk[k.Module]
if !on {
return true
@@ -616,8 +622,8 @@ func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
// A node that composes its own asks about its own composition, which its have names as the one it waits on (yeses): that
// ask stands while the have waits on it, whatever the controller composes (issue 774, 2026-10-31: novox's asks of
// postgres, keycloak and mongodb were taken away and made again under a new id every pass, so no answer could count).
if hkey, err := delivery.HaveKey(k.Node, k.Module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, hkey)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits == k.Composed {
if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil {
if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, hkey)); err == nil && h.Phase == nodeengineclient.PhaseWaitingYes && h.Waits == k.Composed {
return false
}
}
@@ -625,13 +631,13 @@ func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
// trunk's, its want's, its have's), and only when every one of them is read.
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.Fp2) {
d, ok := p.w.composed(k.Node, k.Module, fp)
if !ok || d.Digest == k.Composed {
return false
}
}
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
return len(p.fingerprintsOf(k.Node, k.Module, m.Fp2)) > 0
}
return false
}
@@ -646,14 +652,14 @@ func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
}
}
add(trunkFP)
if key, err := delivery.WantKey(node, module); err == nil {
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
add(w.FP)
if key, err := mwc.WantKey(node, module); err == nil {
if w, err := mwc.DecodeWant(p.w.value(buckets.Wants, key)); err == nil {
add(w.Fp)
}
}
if key, err := delivery.HaveKey(node, module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
add(h.FP)
if key, err := nodeengineclient.HaveKey(node, module); err == nil {
if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, key)); err == nil {
add(h.Fp)
}
}
return out
@@ -662,21 +668,21 @@ func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
// read at all (no module), nothing is taken away.
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.ModulesByModuleValue) {
if len(trunk) == 0 {
return
}
for key := range p.w.read(delivery.AssignerAsksBucket) {
for key := range p.w.read(buckets.Asks) {
if p.staleAsk(key, trunk) {
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
k, kerr := delivery.ParseAnyAskKey(key)
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
k, kerr := asks.ParseAnyAskKey(key)
held := kerr == nil && (k.Kind == asks.AskKindHold || k.Kind == asks.AskKindRestore)
if held && !p.clearHeld(k.Module, k.FP) {
continue
}
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
p.remove(ctx, delivery.AssignerAsksBucket, key)
p.remove(ctx, buckets.Asks, key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
// raised again if the trunk returns to the fingerprint.
@@ -701,21 +707,21 @@ func (p *passer) remove(ctx context.Context, bucket, key string) {
}
// heldHold is the hold of fp as written: the node whose health check failed and its words.
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
func (p *passer) heldHold(module, fp string, fresh *mwc.Held) mwc.Held {
if fresh != nil {
return *fresh
}
key, err := delivery.HeldKey(module, fp)
key, err := mwc.HeldKey(module, fp)
if err != nil {
return delivery.Held{}
return mwc.Held{}
}
h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key])
h, _ := mwc.DecodeHeld(p.w.read(buckets.Wants)[key])
return h
}
// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the
// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask.
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *delivery.Held, data bool) {
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *mwc.Held, data bool) {
h := p.heldHold(module, fp, fresh)
c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data)
if p.say == nil || p.said[c.Key+".raised"] {
@@ -732,13 +738,13 @@ func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *de
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
key, err := delivery.RestoreAskKey(node, module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
key, err := asks.RestoreAskKey(node, module, fp)
if err != nil || p.w.has(buckets.Asks, key) {
return
}
words := p.heldHold(module, fp, nil).Words
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
@@ -750,7 +756,7 @@ func (p *passer) answeredWith(key, option string) bool {
if !ok || a.Option != option {
return false
}
_, ask, err := assign.DecodeAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks, key))
if err != nil {
return false
}
@@ -775,12 +781,12 @@ func (p *passer) clearHeld(module, fp string) bool {
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
key, err := delivery.RecordAskKey(need.Node, module, need.Digest)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
key, err := asks.RecordAskKey(need.Node, module, need.Digest)
if err != nil || p.w.has(buckets.Asks, key) {
return
}
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
}
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
@@ -794,24 +800,24 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
// in a new have value after the answer was written.
func (p *passer) backups(ctx context.Context) {
wanted := p.w.wantsOf()
for key, value := range p.w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
for key, value := range p.w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := delivery.DecodeHave(value)
if err != nil || have.Phase != delivery.PhaseHeld {
have, err := nodeengineclient.DecodeHave(value)
if err != nil || have.Phase != nodeengineclient.PhaseHeld {
continue
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
}
akey, err := delivery.BackupAskKey(node, module, want.FP)
akey, err := asks.BackupAskKey(node, module, want.Fp)
if err != nil {
continue
}
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks)[akey])
if there != nil || assign.BackupAskDigest(current) != want.Composed {
after := -1
if there == nil {
@@ -822,21 +828,21 @@ func (p *passer) backups(ctx context.Context) {
}
answer, answered := p.w.answer(akey)
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID ||
(answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) {
(answer.Option != mwc.BackupAgain && answer.Option != mwc.BackupSkip) {
continue // not answered yet: the ask stands
}
words := assign.BackupAnswerWords(answer.Option, current.Asking)
haveChanges := p.w.changed(delivery.HaveBucket, key)
haveChanges := p.w.changed(buckets.Have, key)
if want.Backup != answer.Option || want.Words != words {
want.Backup, want.Words = answer.Option, words
if wkey, err := delivery.WantKey(node, module); err == nil {
if wkey, err := mwc.WantKey(node, module); err == nil {
w := want
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
p.put(ctx, buckets.Wants, wkey, func() ([]byte, error) { return mwc.EncodeWant(w) })
p.answeredAt[currentAsk.ID] = haveChanges
}
continue
}
if answer.Option != delivery.BackupAgain {
if answer.Option != mwc.BackupAgain {
continue
}
// The answer is in the want. The node retried and the backup failed again when the have took another value since.
@@ -849,10 +855,10 @@ func (p *passer) backups(ctx context.Context) {
}
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
c := assign.BackupCondition(node, module, want.FP, words, true)
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want mwc.Want, words string, asking int) {
a := assign.BackupAsk(key, node, module, want.Fp, want.Composed, words, asking, time.Now())
p.put(ctx, buckets.Asks, key, func() ([]byte, error) { return assign.EncodeAsk(key, a) })
c := assign.BackupCondition(node, module, want.Fp, words, true)
if p.say == nil || p.said[c.Key] {
return
}
@@ -869,7 +875,7 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A
// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the
// want naming the release, with no ask (ADR 0601 §1).
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]mwc.Want, out *assign.Outcome) {
for _, node := range m.Nodes {
if m.Self == nil || !m.Self(node) {
continue
@@ -886,27 +892,27 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo
continue
}
yes, release := "", ""
if had && was.FP == w.FP {
if had && was.Fp == w.Fp {
yes, release = was.Yes, was.Release
}
key, err := delivery.HaveKey(node, module)
key, err := nodeengineclient.HaveKey(node, module)
if err != nil {
continue
}
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
if h, err := nodeengineclient.DecodeHave(p.w.value(buckets.Have, key)); err == nil && h.Phase == nodeengineclient.PhaseWaitingYes && h.Waits != yes {
switch {
case m.Allowed != nil && m.Allowed(node, h.Waits):
yes, release = h.Waits, ""
case h.BuildAlone && h.FP != w.FP && m.Release != nil:
case h.BuildAlone && h.Fp != w.Fp && m.Release != nil:
// A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1),
// named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked.
if r, ok := m.Release(w.FP); ok {
if r, ok := m.Release(w.Fp); ok {
yes, release = h.Waits, r
break
}
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
default:
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
}
}
// A want carried over from before the node composed its own may still name the controller's composition; a placed
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go
+203 -188
View File
@@ -6,6 +6,16 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"slices"
"strings"
"testing"
@@ -53,7 +63,7 @@ func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt
}
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
func newPasser(kvs buckets, at time.Time) *passer {
func newPasser(kvs kvSet, at time.Time) *passer {
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
now: func() time.Time { return at }}
}
@@ -78,24 +88,24 @@ func placeOn(t *testing.T, w *world, node string, modules ...string) {
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
func composes(t *testing.T, w *world, node, digest string) {
t.Helper()
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
raw, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
w.put(delivery.HaveBucket, key, raw)
key, _ := nodeengineclient.HaveNodeKey(node)
w.put(buckets.Have, key, raw)
}
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
var fpTest = meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
// an id it keeps).
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
key, _ := delivery.HoldAskKey("web", fpTest)
held := &delivery.Held{Node: "ace", Words: "health failed"}
kvs := kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}
key, _ := asks.HoldAskKey("web", fpTest)
held := &mwc.Held{Node: "ace", Words: "health failed"}
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
first.holdAsk(context.Background(), "web", fpTest, held)
a1, err := decodeAsk(key, asksKV.values[key])
@@ -128,20 +138,20 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
t.Errorf("offered(%s) = %v, want %v", provision, !want, want)
}
}
key, _ := delivery.HoldAskKey("web", fpTest)
p := newPasser(buckets{}, time.Now())
key, _ := asks.HoldAskKey("web", fpTest)
p := newPasser(kvSet{}, time.Now())
put := func(asking int) string {
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
v, err := assign.EncodeAsk(key, a)
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.AssignerAsksBucket, key, v)
p.w.put(buckets.Asks, key, v)
return mustAskID(a)
}
first := put(100)
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
p.w.put(delivery.AnswersBucket, key, v)
p.w.put(buckets.Answers, key, v)
if !p.answeredWith(key, "release") {
t.Fatal("the answer to the ask that stands is not taken")
}
@@ -152,7 +162,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
if p.answeredWith(key, "release") {
t.Error("the answer to an earlier hold carried over to a later one")
}
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
p.w.put(buckets.Asks, key, nil) // the ask taken away
if p.answeredWith(key, "release") {
t.Error("an answer to an ask taken away was taken")
}
@@ -162,12 +172,12 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
t.Run("the rules read what each node's engine reads", func(t *testing.T) {
w := newWorld()
for node, reads := range map[string][]string{"ace": {"word:module", "word:tool-runner"}, "g14": nil} {
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ManifestReads: reads}})
raw, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node, ManifestReads: reads}})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
w.put(delivery.HaveBucket, key, raw)
key, _ := nodeengineclient.HaveNodeKey(node)
w.put(buckets.Have, key, raw)
}
n := w.nodes(nil, nil)
if !slices.Equal(n.Reads["ace"], []string{"word:module", "word:tool-runner"}) || len(n.Reads["g14"]) != 0 || len(n.Reads["novox"]) != 0 {
@@ -185,8 +195,8 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)})
}
p := &passer{w: w, reads: &enginereads.Keeper{Call: call, Fallback: w.recordReads, Sync: true}}
on, _ := delivery.EncodePresence(delivery.Presence{Online: false})
w.put(delivery.PresenceBucket, "ace", on)
on, _ := presenceValue(false)
w.put(buckets.Presence, "ace", on)
n = w.nodes(nil, nil)
p.engineReads(n)
if !slices.Equal(n.Reads["ace"], []string{"form:new"}) || len(n.Reads["g14"]) != 0 || asked["ace"] != 1 {
@@ -196,8 +206,8 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
t.Errorf("the statements are %+v", s)
}
p.engineReads(w.nodes(nil, nil))
on, _ = delivery.EncodePresence(delivery.Presence{Online: true})
w.put(delivery.PresenceBucket, "ace", on)
on, _ = presenceValue(true)
w.put(buckets.Presence, "ace", on)
p.engineReads(w.nodes(nil, nil))
if asked["ace"] != 2 {
t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"])
@@ -313,11 +323,11 @@ func (b *fakeBus) Emit(eventType string, body any) error {
func putOfTheBus(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
fp := meshcatalogclient.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
digest := "sha256:" + strings.Repeat("d", 64)
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
puts, asksKV, wantsKV := mem(), mem(), mem()
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
putsKV, asksKV, wantsKV := mem(), mem(), mem()
kvs := kvSet{buckets.Puts: putsKV, buckets.Asks: asksKV, buckets.Wants: wantsKV}
p := newPasser(kvs, now)
bus := &fakeBus{calls: make(chan string, 8)}
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
@@ -338,21 +348,21 @@ func putOfTheBus(t *testing.T) {
}
placeOn(t, p.w, "novox", "nats")
mdigest := strings.Repeat("e", 64)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
world(delivery.ModulesBucket, "nats", mod)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
world(buckets.Modules, "nats", mod)
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
bkey, _ := delivery.BuilderBuildKey("nats", fp)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
world(delivery.BuilderBuildsBucket, bkey, built)
hkey, _ := delivery.HaveKey("novox", "nats")
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, before)
bkey, _ := builds.BuilderBuildKey("nats", fp)
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest})
world(buckets.Builds, bkey, built)
hkey, _ := nodeengineclient.HaveKey("novox", "nats")
before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
world(buckets.Have, hkey, before)
builds := mem()
builds.values[bkey] = built
buildKV := mem()
buildKV.values[bkey] = built
modules := mem()
modules.values["nats"] = mod
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
s := newPutTools(memStore{putsState: putsKV, builds.BuilderModule + ".builds": buildKV, delivery.IndexerModule + ".modules": modules})
s.now = func() time.Time { return now }
was := answerWithin
answerWithin = 0
@@ -370,21 +380,21 @@ func putOfTheBus(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
if a := ans.(putAnswer); a.Step != puts.Asked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
t.Fatalf("put answered %+v", a)
}
pkey, _ := delivery.PutKey("novox", "nats")
step := func() delivery.Put {
pkey, _ := puts.Key("novox", "nats")
step := func() puts.Put {
t.Helper()
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
world(buckets.Puts, pkey, putsKV.values[pkey])
p.pass(ctx)
got, err := delivery.DecodePut(puts.values[pkey])
got, err := puts.Decode(putsKV.values[pkey])
if err != nil {
t.Fatal(err)
}
return got
}
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
if got := step(); got.Step != puts.Compose || !strings.Contains(got.Words, "no composition") {
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
}
select {
@@ -393,12 +403,12 @@ func putOfTheBus(t *testing.T) {
default:
}
composes(t, p.w, "novox", digest)
gkey, _ := delivery.PartGrantKey("nats", fp)
g, _ := delivery.EncodeGrant(delivery.Grant{})
world(delivery.PartGrantsBucket, gkey, g)
gkey, _ := grantsstate.PartGrantKey("nats", fp)
g, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
world(buckets.Grants, gkey, g)
got := step()
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
if got.Step != delivery.PutPerson || got.Ask != akey {
akey, _ := asks.RecordAskKey("novox", "nats", digest)
if got.Step != puts.Person || got.Ask != akey {
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
}
a, err := decodeAsk(akey, asksKV.values[akey])
@@ -408,35 +418,35 @@ func putOfTheBus(t *testing.T) {
if _, there := wantsKV.values["want.novox.nats"]; there {
t.Fatal("the bus's want was written before the operator's warrant")
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
world(buckets.Asks, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes)
if got := step(); got.Step != delivery.PutSnapshot {
world(buckets.Answers, akey, yes)
if got := step(); got.Step != puts.Snapshot {
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
}
<-woke
got = step()
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
w, err := mwc.DecodeWant(wantsKV.values["want.novox.nats"])
if got.Step != puts.Wanted || err != nil || w.Fp != fp || w.Composed != digest || !w.Put || !w.Hold {
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
}
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
t.Errorf("the warrant was recorded as %q", call)
}
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
world(delivery.HaveBucket, hkey, running)
if got := step(); got.Step != delivery.PutHolding {
world(buckets.Wants, "want.novox.nats", wantsKV.values["want.novox.nats"])
running, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
world(buckets.Have, hkey, running)
if got := step(); got.Step != puts.Holding {
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
}
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
if got := step(); got.Step != delivery.PutReleased {
if got := step(); got.Step != puts.Released {
t.Fatalf("released, the put stands at %s", got.Step)
}
listed, _ := s.doPuts(map[string]any{"module": "nats"})
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != puts.Released || len(l[0].Steps) < 6 {
t.Errorf("puts lists %+v", l)
}
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
@@ -444,27 +454,27 @@ func putOfTheBus(t *testing.T) {
}
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
world(delivery.HaveBucket, hkey, before)
world(buckets.Have, hkey, before)
s.now = func() time.Time { return now.Add(time.Minute) }
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
again := step()
a2, err := decodeAsk(akey, asksKV.values[akey])
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
if again.Step != puts.Person || err != nil || a2.Ask.ID == a.Ask.ID {
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
}
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
world(buckets.Asks, akey, asksKV.values[akey])
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
world(delivery.AnswersBucket, akey, yes2)
if got := step(); got.Step != delivery.PutSnapshot {
world(buckets.Answers, akey, yes2)
if got := step(); got.Step != puts.Snapshot {
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
}
<-woke
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
oldWant, _ := mwc.EncodeWant(mwc.Want{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)})
wantsKV.values["want.novox.nats"] = oldWant
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
world(buckets.Wants, "want.novox.nats", oldWant)
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
@@ -483,7 +493,7 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// The verb edges a want waits on come from the graph and what runs (ADR 0614 §2, §3): folded here to hold the count.
aWantWaitsOnTheVerbEdgesTheGraphKeeps(t)
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []oc.Condition
failing := true
p.say = func(c oc.Condition) error {
@@ -497,7 +507,7 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
holding: assign.Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
p.sayNeither(one, nil, nil) // the say fails
p.sayNeither(one, nil, nil)
p.sayNeither(one, nil, nil)
@@ -518,7 +528,7 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
}
// 1945: a module that leaves neither list with no want is cleared in words saying so, never "has a want again". The bus is
// on neither list by a rule as any module (ADR 0601 §1), nothing for the operator to do.
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: delivery.RuleGrant, Words: "no credential"}}}
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: mwc.RuleGrant, Words: "no credential"}}}
p.sayNeither(bus, nil, nil)
if c := said[len(said)-1]; c.State != occ.StateRaised || !strings.HasPrefix(c.Explanation, occ.NothingToDo) || !strings.Contains(c.Explanation, "rule 6 holds its first want back: no credential") {
t.Errorf("the bus's neither condition = %+v", c)
@@ -535,23 +545,23 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
got := map[string]neitherOne{}
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: mwc.RuleGrant, Words: "w"}}})
gatherNeither(got, "rofi", assign.Outcome{})
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != mwc.RuleGrant {
t.Errorf("gathered = %+v", got)
}
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
// through its steps owns the want, so it is named, never "the node does not run it".
for _, c := range []struct {
name string
puts map[string]delivery.Put
before map[string]delivery.Want
puts map[string]puts.Put
before map[string]mwc.Want
why string
}{
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
{"a live want", nil, map[string]mwc.Want{"novox": {Fp: fpTest}}, ""},
{"a put on its steps", map[string]puts.Put{"novox": {FP: fpTest, Step: puts.Person}}, nil, "its put waits on its own steps and the operator's answer"},
{"nothing", nil, nil, "the node does not run it"},
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
{"an ended put", map[string]puts.Put{"novox": {FP: fpTest, Step: puts.Refused}}, nil, "the node does not run it"},
} {
left := map[string]string{}
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
@@ -561,22 +571,22 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
wanted := assign.Outcome{Wants: map[string]mwc.Want{"shanks": {Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
for i, c := range []struct {
before map[string]delivery.Want
before map[string]mwc.Want
out assign.Outcome
line string // the line said in this pass, "" for none
}{
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
{nil, notDone, ""},
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{map[string]mwc.Want{"shanks": {Removed: true}}, notDone, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
{map[string]mwc.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
@@ -618,7 +628,7 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
}
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"rofi": {Fp: fpTest, Health: nodeengineclient.HealthFailed}}}}
for _, c := range []struct {
name string
known assign.Nodes
@@ -628,7 +638,7 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
} {
conds = nil
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
fresh := newPasser(kvSet{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
fresh.logf = func(string, ...any) {}
fresh.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
@@ -650,19 +660,19 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now())
var said []oc.Condition
p.say = func(c oc.Condition) error { said = append(said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
p.w.put(buckets.Modules, "web", mod)
placeOn(t, p.w, "g14", "web")
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
p.w.put(delivery.HaveBucket, "g14.web", have)
key, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
p.w.put(delivery.AssignerWantsBucket, key, want)
on, _ := presenceValue(true)
p.w.put(buckets.Presence, "g14", on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, "g14.web", have)
key, _ := mwc.WantKey("g14", "web")
want, _ := mwc.EncodeWant(mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
p.w.put(buckets.Wants, key, want)
p.wants(context.Background())
cleared := map[string]bool{}
for _, c := range said {
@@ -688,19 +698,19 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
// raised (the first pass after a start), each once.
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
setup := func() (*passer, *[]oc.Condition) {
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
p := newPasser(kvSet{buckets.Wants: &memKV{values: map[string][]byte{}}, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now())
said := &[]oc.Condition{}
p.say = func(c oc.Condition) error { *said = append(*said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
p.w.put(buckets.Modules, "systemd-resolved", mod)
placeOn(t, p.w, "shanks", "systemd-resolved")
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "shanks", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
p.w.put(delivery.AssignerWantsBucket, hkey, held)
on, _ := presenceValue(true)
p.w.put(buckets.Presence, "shanks", on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed})
p.w.put(buckets.Have, "shanks.systemd-resolved", have)
hkey, _ := mwc.HeldKey("systemd-resolved", fpTest)
held, _ := mwc.EncodeHeld(mwc.Held{Node: "shanks", Words: "health failed"})
p.w.put(buckets.Wants, hkey, held)
return p, said
}
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
@@ -719,7 +729,7 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
}
// a standing neither this process raised
p, said := setup()
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: delivery.RuleNotHeld}}}
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: mwc.RuleNotHeld}}}
p.wants(context.Background())
p.wants(context.Background())
if r, c := neithers(*said); r != 0 || c != 1 {
@@ -738,24 +748,24 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
fpOld, fpNew := meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
trunkOf := func(fp string) map[string]delivery.Module {
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
trunkOf := func(fp string) map[string]meshcatalog.ModulesByModuleValue {
return map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}}
}
type ask struct{ name, key string }
holdOld, _ := delivery.HoldAskKey("web", fpOld)
holdNew, _ := delivery.HoldAskKey("web", fpNew)
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
holdOld, _ := asks.HoldAskKey("web", fpOld)
holdNew, _ := asks.HoldAskKey("web", fpNew)
holdGone, _ := asks.HoldAskKey("gone", fpOld)
restOld, _ := asks.RestoreAskKey("novox", "web", fpOld)
restNew, _ := asks.RestoreAskKey("novox", "web", fpNew)
recOld, _ := asks.RecordAskKey("novox", "web", digOld)
recNew, _ := asks.RecordAskKey("novox", "web", digNew)
backOld, _ := asks.BackupAskKey("novox", "web", fpOld)
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
for _, c := range []struct {
name string
trunk map[string]delivery.Module
trunk map[string]meshcatalog.ModulesByModuleValue
gone []string // the keys taken away
noDigit bool // the node's own composition is not read
}{
@@ -767,18 +777,18 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
} {
t.Run(c.name, func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
var said []oc.Condition
p.say = func(c oc.Condition) error { said = append(said, c); return nil }
for _, key := range all {
k, _ := delivery.ParseAnyAskKey(key)
var a delivery.AssignerAsk
k, _ := asks.ParseAnyAskKey(key)
var a asks.Ask
switch k.Kind {
case delivery.AskKindHold:
case asks.AskKindHold:
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
case delivery.AskKindRestore:
case asks.AskKindRestore:
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
case delivery.AskKindRecord:
case asks.AskKindRecord:
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
default:
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
@@ -788,19 +798,19 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Fatal(err)
}
asksKV.values[key] = v
p.w.put(delivery.AssignerAsksBucket, key, v)
p.w.put(buckets.Asks, key, v)
}
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
if !c.noDigit {
composes(t, p.w, "novox", digNew)
}
heldOld, _ := delivery.HeldKey("web", fpOld)
heldOld, _ := mwc.HeldKey("web", fpOld)
wantsKV.values[heldOld] = []byte("x")
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
p.w.put(buckets.Wants, heldOld, []byte("x"))
firstAsk, _ := decodeAsk(holdOld, asksKV.values[holdOld])
p.staleAsks(context.Background(), c.trunk)
// the held record and condition of a fingerprint whose hold ask went are taken away with it
heldOld, _ = delivery.HeldKey("web", fpOld)
heldOld, _ = mwc.HeldKey("web", fpOld)
_, heldThere := wantsKV.values[heldOld]
cleared := slices.ContainsFunc(said, func(c oc.Condition) bool {
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == occ.StateCleared
@@ -831,7 +841,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
var said []oc.Condition
fail := true
p.say = func(c oc.Condition) error {
@@ -848,7 +858,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Fatal(err)
}
asksKV.values[holdOld] = v
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
p.w.put(buckets.Asks, holdOld, v)
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
@@ -861,15 +871,15 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
keep, _ := asks.RestoreAskKey("ace", "web", fpNew)
keepOther, _ := asks.RestoreAskKey("ace", "other", fpOld)
for _, key := range []string{restOld, mustKey(asks.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
asksKV.values[key] = []byte("x")
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
p.w.put(buckets.Asks, key, []byte("x"))
}
p.lift(context.Background(), "web", fpOld)
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
for key, gone := range map[string]bool{restOld: true, mustKey(asks.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
if _, there := asksKV.values[key]; there == gone {
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
}
@@ -897,7 +907,7 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing
// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a
// secret too, one at the build it runs, and one no approved release carries are asked of a person as before.
func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
commit := strings.Repeat("1", 40)
for _, c := range []struct {
@@ -905,7 +915,7 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
alone bool
at string // the fingerprint the have runs
mergedAt string // the commit the release merged web at; "" for no merge of the release
fpSrc uint64 // the source revision at which web's fingerprint last changed
fpSrc int64 // the source revision at which web's fingerprint last changed
yes, asked bool
}{
{"the build alone, its release approved", true, fpOld, commit, 7, true, false},
@@ -920,40 +930,40 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
{"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true},
} {
asksKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now())
p.logf = t.Logf
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FPSrc: c.fpSrc, Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: c.fpSrc, Path: meshcatalogclient.RootPath})
p.w.put(buckets.Modules, "web", mod)
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/web", Heads: map[string]string{"novox/web": commit}})
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, src, 7, time.Now())
p.w.putEntry(buckets.Source, delivery.SourceMainKey, src, 7, time.Now())
later, _ := delivery.EncodeSource(delivery.Source{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}})
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
p.w.putEntry(buckets.Source, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
if c.mergedAt != "" {
merge, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)}, State: delivery.MergeMerged, Commit: c.mergedAt}}})
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.MergesBucket, "feat/web", merge)
p.w.put(buckets.Merges, "feat/web", merge)
other, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40)}, State: delivery.MergeMerged, Commit: commit}}})
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.MergesBucket, "feat/other", other)
p.w.put(buckets.Merges, "feat/other", other)
}
have, _ := delivery.EncodeHave(delivery.Have{FP: c.at, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
p.w.put(delivery.HaveBucket, "g14.web", have)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: c.at, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK,
Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
p.w.put(buckets.Have, "g14.web", have)
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true },
Release: func(fp string) (string, bool) { return p.w.release("web", fp) }}
out := assign.Outcome{Wants: map[string]delivery.Want{"g14": {FP: fpTest, Self: true}}}
p.yeses(context.Background(), "web", m, map[string]delivery.Want{"g14": {FP: fpOld, Self: true}}, &out)
out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}}
p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out)
w := out.Wants["g14"]
if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) {
t.Errorf("%s: the want is %+v", c.name, w)
}
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
akey, _ := asks.RecordAskKey("g14", "web", waiting)
if _, there := asksKV.values[akey]; there != c.asked {
t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked)
}
@@ -963,7 +973,7 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
// module in an assignment only is no module of its node, and before any placement is read nothing is placed.
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
if got := p.w.assigned(); len(got) != 0 {
t.Fatalf("before any placement: assigned %v", got)
@@ -979,47 +989,47 @@ func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
// kept there while the want stays at its build.
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
put(buckets.Modules, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
placeOn(t, p.w, "g14", "web")
pl, _ := mmac.EncodePlacement(mma.Placement{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
put(plc.PlacementBucket, "placement.g14", pl)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
on, _ := presenceValue(true)
put(buckets.Presence, "g14", on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
put(buckets.Have, "g14.web", have)
wkey, _ := mwc.WantKey("g14", "web")
want, _ := mwc.EncodeWant(mwc.Want{Fp: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
put(buckets.Wants, wkey, want)
sync := func() {
for k, v := range wantsKV.values {
put(delivery.AssignerWantsBucket, k, v)
put(buckets.Wants, k, v)
}
}
p.wants(context.Background())
p.nodeWants(context.Background())
sync()
got, err := delivery.DecodeWant(wantsKV.values[wkey])
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
got, err := mwc.DecodeWant(wantsKV.values[wkey])
if err != nil || got.Fp != fpOld || got.Composed != "" || !got.Self {
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
}
nkey, _ := delivery.WantNodeKey("g14")
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
nkey, _ := mwc.WantNodeKey("g14")
if nw, err := mwc.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
}
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
put(delivery.HaveBucket, "g14.web", waits)
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
waits, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK,
Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
put(buckets.Have, "g14.web", waits)
akey, _ := asks.RecordAskKey("g14", "web", waiting)
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
t.Fatal("a have waiting for a yes raised no record ask")
@@ -1039,7 +1049,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
t.Fatalf("a pass that changed nothing put %d wants and %d asks", wantsKV.puts-wantPuts, asksKV.puts-askPuts)
}
// Whichever rule asks it, a want put with the value the bucket holds reaches no bucket: each put is a change its node hears.
p.put(context.Background(), delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
p.put(context.Background(), buckets.Wants, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
if wantsKV.puts != wantPuts {
t.Fatalf("a want put unchanged reached the bucket")
}
@@ -1047,19 +1057,19 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
if err != nil {
t.Fatal(err)
}
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
put(buckets.Asks, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
put(buckets.Answers, akey, yes)
p.wants(context.Background())
sync()
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
if got, err = mwc.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.Fp != fpOld {
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
}
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", ran)
ran, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: waiting, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
put(buckets.Have, "g14.web", ran)
p.wants(context.Background())
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
if got, _ = mwc.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
t.Errorf("a pass after the move dropped the yes: %+v", got)
}
}
@@ -1068,7 +1078,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
// cleared once, and said again at a later pass while the findings cannot be read.
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
p := newPasser(kvSet{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
bus := &fakeBus{calls: make(chan string, 8)}
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
var said []string
@@ -1103,8 +1113,8 @@ func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
// module serves; a verb is served once a running healthy build's manifest (the builder's build, the indexer's manifest)
// serves it, and a build whose manifest cannot be read counts what the graph says its module serves. No graph read: no wait.
func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
p := newPasser(buckets{}, time.Now())
fp := func(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
p := newPasser(kvSet{}, time.Now())
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
if p.served(context.Background(), nil, graph.Graph{}, assign.Nodes{}) != nil {
t.Error("with no graph read, a want waits on a verb")
}
@@ -1126,19 +1136,19 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
t.Errorf("the kept verb edges of caller = %v", got)
}
digest := strings.Repeat("d", 64)
man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "server", JSON: `{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`})
man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "server", Manifest: `{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`})
get := func(_ context.Context, key string) ([]byte, error) {
if key == digest {
return man, nil
}
return nil, errNotThere
}
build, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
key, _ := delivery.BuilderBuildKey("server", fp("f"))
p.w.put(delivery.BuilderBuildsBucket, key, build)
running := delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
at := func(f string) delivery.Have { h := running; h.FP = f; return h }
served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]delivery.Have{
build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest})
key, _ := builds.BuilderBuildKey("server", fp("f"))
p.w.put(buckets.Builds, key, build)
running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h }
served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"g14": {"server": at(fp("f")), "late": at(fp("2"))}}})
for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} {
if served(v) != want {
@@ -1150,7 +1160,7 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}}
for _, online := range []bool{false, true} {
offering := p.offering(context.Background(), get, bus, assign.Nodes{Online: map[string]bool{"n2": online},
Haves: map[string]map[string]delivery.Have{"n2": {"bus": at(fp("2"))}}})
Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"n2": {"bus": at(fp("2"))}}})
if offering(graph.BusState) != online {
t.Errorf("with the bus module's node online %v, bus-state offered %v", online, offering(graph.BusState))
}
@@ -1168,10 +1178,15 @@ func decodeAsk(key string, data []byte) (decodedAsk, error) {
return decodedAsk{Asking: a.Asking, Ask: ask}, err
}
func mustAskID(a delivery.AssignerAsk) string {
func mustAskID(a asks.Ask) string {
ask, err := assign.AskOf(a)
if err != nil {
panic(err)
}
return ask.ID
}
// presenceValue is a node's presence as the watchman writes it.
func presenceValue(online bool) ([]byte, error) {
return json.Marshal(watchdogs.PresenceByNodeValue{V: 1, Online: online})
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go
+56 -48
View File
@@ -5,6 +5,14 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strings"
"sync"
@@ -63,25 +71,25 @@ func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
}
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
func identity(key string, p delivery.Put) string {
func identity(key string, p puts.Put) string {
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
}
// putsByModule is every put the assigner holds, by module then node.
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
out := map[string]map[string]delivery.Put{}
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
node, module, err := delivery.ParsePutKey(key)
func (p *passer) putsByModule() map[string]map[string]puts.Put {
out := map[string]map[string]puts.Put{}
for key, value := range p.w.read(buckets.Puts) {
node, module, err := puts.ParseKey(key)
if err != nil {
continue
}
put, err := delivery.DecodePut(value)
put, err := puts.Decode(value)
if err != nil {
p.logf("the put %s cannot be read, and is left: %v", key, err)
continue
}
if out[module] == nil {
out[module] = map[string]delivery.Put{}
out[module] = map[string]puts.Put{}
}
out[module][node] = put
}
@@ -99,7 +107,7 @@ func (p *passer) puts(ctx context.Context) {
}
assigned := p.w.assigned()
trunk := p.w.trunk()
get := p.w.getter(delivery.ManifestsBucket) // the indexer's manifests, as its first view and views family say them
get := p.w.getter(buckets.Manifests) // the indexer's manifests, as its first view and views family say them
wanted := p.w.wantsOf()
modules := make([]string, 0, len(all))
for m := range all {
@@ -114,29 +122,29 @@ func (p *passer) puts(ctx context.Context) {
}
}
for node, put := range all[module] {
if delivery.PutEnded(put.Step) {
if puts.Ended(put.Step) {
continue
}
key, _ := delivery.PutKey(node, module)
key, _ := puts.Key(node, module)
id := identity(key, put)
world := p.putWorld(node, module, put, manifest, assigned[node])
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
next := assign.StepPut(world)
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
if w, ok := wanted[module][node]; ok && w.Put && w.Fp == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
(next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) {
if d, _ := p.composedFor(node, module, w.Fp); d != nil && d.UserListOnly && d.Digest == w.Composed {
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
next.Put.Composed = w.Composed
}
}
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted && world.Bus && next.Put.Ask != "" {
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
next = assign.PutNext{Put: next.Put.Said(world.Now, puts.Refused, "the bus's want is not written: "+err.Error())}
}
}
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == puts.Wanted || next.Put.Step == puts.Holding) &&
!strings.Contains(next.Put.Words, failed) {
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
}
@@ -145,38 +153,38 @@ func (p *passer) puts(ctx context.Context) {
if !written {
continue // released or put again meanwhile: nothing is done for the put that no longer stands
}
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
if next.Put.Step == puts.Wanted && put.Step != puts.Wanted {
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
if wkey, err := delivery.WantKey(node, module); err == nil {
if wkey, err := mwc.WantKey(node, module); err == nil {
want := assign.PutWant(next.Put)
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
if before, ok := wanted[module][node]; !ok || !mwc.SameWant(normalWant(before), normalWant(want)) {
p.put(ctx, buckets.Wants, wkey, func() ([]byte, error) { return mwc.EncodeWant(want) })
}
}
if next.Warranted {
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
p.acts.warrant(id, meshdeliveryclient.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
}
}
}
}
}
func normalWant(w delivery.Want) delivery.Want {
func normalWant(w mwc.Want) mwc.Want {
w.V = 0
return w
}
// putWorld is what one step of a put reads from the world.
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].Fp2}
for _, m := range assignedHere {
if m == module {
w.Assigned = true
}
}
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
if bkey, err := builds.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(buckets.Builds, bkey); raw != nil {
if b, err := builds.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
w.Build = "unreadable: " + err.Error()
@@ -193,9 +201,9 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
policy, _, _ = catalogue.DerivedUpgrade(manifest)
}
w.Recorded = policy == catalogue.PolicyRecord
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
if akey, err := asks.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
if a, ask, err := assign.DecodeAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
if a, ask, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks, akey)); err == nil && a.Asking == assign.PutAsking(put) {
switch {
case p.answeredWith(akey, assign.OptionYes):
w.Answer = assign.OptionYes
@@ -208,9 +216,9 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
}
}
}
if hkey, err := delivery.HaveKey(node, module); err == nil {
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
if h, err := delivery.DecodeHave(raw); err == nil {
if hkey, err := nodeengineclient.HaveKey(node, module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
w.Have = &h
}
}
@@ -221,15 +229,15 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
func (p *passer) composedFor(node, module, fp string) (*assign.ComposedDigest, bool) {
var out *assign.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
out = &d
}
granted := false
if gkey, err := delivery.PartGrantKey(module, fp); err == nil {
if gkey, err := grantsstate.PartGrantKey(module, fp); err == nil {
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
if g, err := delivery.DecodeGrant(raw); err == nil {
if g, err := grantsstate.DecodeGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
}
@@ -239,17 +247,17 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest,
// actOn does the acts a step asks for.
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
if next.Put.Step == puts.Person && next.Put.Ask != "" && w.Composed != nil {
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
// raises it anew under its own id.
akey := next.Put.Ask
current, _, err := assign.DecodeAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
current, _, err := assign.DecodeAsk(akey, p.w.value(buckets.Asks, akey))
if err != nil || current.Asking != assign.PutAsking(next.Put) {
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
if err != nil {
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
} else {
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.put(ctx, buckets.Asks, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
}
}
}
@@ -260,7 +268,7 @@ func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.Pu
// performs checks that the act about to be performed is the one the yes of the ask at key bound (occ.Performs).
func (p *passer) performs(key string, act occ.Act) error {
_, ask, err := assign.DecodeAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
_, ask, err := assign.DecodeAsk(key, p.w.value(buckets.Asks, key))
if err != nil {
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
}
@@ -273,14 +281,14 @@ func (p *passer) performs(key string, act occ.Act) error {
}
// samePut says two puts say the same: their step, words and what they found.
func samePut(a, b delivery.Put) bool {
func samePut(a, b puts.Put) bool {
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
}
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
// released it since; and says it as the event put. It says whether it wrote.
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
kv := p.kvs[delivery.AssignerPutsBucket]
func (p *passer) writePut(ctx context.Context, key string, read, next puts.Put) bool {
kv := p.kvs[buckets.Puts]
if kv == nil {
return false
}
@@ -289,11 +297,11 @@ func (p *passer) writePut(ctx context.Context, key string, read, next delivery.P
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
return false
}
now, err := delivery.DecodePut(e.Value())
now, err := puts.Decode(e.Value())
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
return false // asked again or released meanwhile: the next pass reads that
}
data, err := delivery.EncodePut(next)
data, err := puts.Encode(next)
if err != nil {
p.logf("the put %s is not written: %v", key, err)
return false
@@ -304,7 +312,7 @@ func (p *passer) writePut(ctx context.Context, key string, read, next delivery.P
}
return false
}
p.w.put(delivery.AssignerPutsBucket, key, data)
p.w.put(buckets.Puts, key, data)
p.logf("put %s: %s: %s", key, next.Step, next.Words)
p.acts.say(key, next)
return true
@@ -429,8 +437,8 @@ type putSaid struct {
}
// say says a put's step as the event put, once per step and words.
func (a *putActs) say(key string, p delivery.Put) {
node, module, err := delivery.ParsePutKey(key)
func (a *putActs) say(key string, p puts.Put) {
node, module, err := puts.ParseKey(key)
if err != nil || a.bus == nil {
return
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat.go
+24 -19
View File
@@ -5,7 +5,12 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"maps"
"mesh-delivery-assigner/internal/buckets"
"reflect"
"slices"
"sort"
@@ -166,14 +171,14 @@ func (p *passer) candidatesOf(id string) (map[string]mma.CandidatesByProposalNod
}
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
out := map[string]*delivery.CandidateAnswer{}
for key, value := range p.w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
func (p *passer) answersTo(id string) map[string]*nodeengine.CandidateAnswer {
out := map[string]*nodeengine.CandidateAnswer{}
for key, value := range p.w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
h, err := delivery.DecodeHaveNode(value)
h, err := nodeengineclient.DecodeHaveNode(value)
if err != nil {
continue
}
@@ -251,17 +256,17 @@ func (p *passer) ask(ctx context.Context, prop mma.Proposal, akey, brief string)
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
return prop, false
}
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.put(ctx, buckets.Asks, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
if pk, err := plc.PulseAskKey(prop.ID); err == nil {
p.pulse(ctx, pk, assign.ExpiresAt(placed).Sub(p.clock()))
}
prop.State, prop.Ask = plc.ProposalAsked, placed.ID
prop.State, prop.Ask = mmac.ProposalAsked, placed.ID
return prop, true
}
// stepProposal takes one proposal one step further.
func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64, current map[string]mma.Placement, revs map[string]uint64) {
if !p.saidProposed[prop.ID] && prop.State == plc.ProposalAnswering && prop.Verdict == nil {
if !p.saidProposed[prop.ID] && prop.State == mmac.ProposalAnswering && prop.Verdict == nil {
p.sayEvent(mma.EventPlacementProposed, mma.PlacementEvent{Proposal: prop}, prop.ID)
p.saidProposed[prop.ID] = true
}
@@ -278,9 +283,9 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
p.addNode(ctx, prop, revs)
return
}
akey, _ := delivery.PlacementAskKey(prop.ID)
akey, _ := asks.PlacementAskKey(prop.ID)
again := prop.Again != nil && !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
if prop.State == plc.ProposalAsked {
if prop.State == mmac.ProposalAsked {
askPulse, _ := plc.PulseAskKey(prop.ID)
switch {
case p.answeredWith(akey, assign.OptionYes):
@@ -324,11 +329,11 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
prop.Verdict = &v
switch v.State {
case delivery.VerdictFail:
prop.State, prop.Ask = plc.ProposalFailed, ""
prop.State, prop.Ask = mmac.ProposalFailed, ""
case plc.VerdictWaits:
prop.State, prop.Ask = plc.ProposalAnswering, ""
prop.State, prop.Ask = mmac.ProposalAnswering, ""
case delivery.VerdictPass:
if prop.State != plc.ProposalAsked || again {
if prop.State != mmac.ProposalAsked || again {
var ok bool
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
return
@@ -449,8 +454,8 @@ func (p *passer) layAgain(ctx context.Context, prop mma.Proposal, rev uint64, cu
// nodes with a record.
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
held := map[string]bool{}
for key := range p.w.read(delivery.ModulesBucket) {
if name, err := delivery.ParseModulesKey(key); err == nil {
for key := range p.w.read(buckets.Modules) {
if name, err := meshcatalogclient.ParseModulesKey(key); err == nil {
held[name] = true
}
}
@@ -532,7 +537,7 @@ func (p *passer) addNode(ctx context.Context, prop mma.Proposal, revs map[string
laid, findings := plc.LayChanges(current, prop.Changes)
findings = append(findings, assign.ControllerRefusals(prop.Changes, laid)...)
node := prop.Changes[0].Node
if prop.Changes[0].Op == plc.OpRemoveNode {
if prop.Changes[0].Op == mmac.OpRemoveNode {
p.removeNode(ctx, prop, current, findings)
return
}
@@ -623,9 +628,9 @@ func (p *passer) sayChanged(prop mma.Proposal, answer delivery.Answer, changes [
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
// whole with why it ended.
func (p *passer) endProposal(ctx context.Context, prop mma.Proposal, why string) {
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
p.w.put(delivery.AssignerAsksBucket, akey, nil)
if akey, err := asks.PlacementAskKey(prop.ID); err == nil && p.w.has(buckets.Asks, akey) {
if err := p.kvs[buckets.Asks].Delete(ctx, akey); err == nil {
p.w.put(buckets.Asks, akey, nil)
}
}
p.deleteCandidates(ctx, prop.ID)
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools.go
+24 -18
View File
@@ -4,6 +4,9 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"slices"
"sort"
"strings"
@@ -101,7 +104,7 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
}
held := map[string]bool{}
for _, k := range mods {
if name, err := delivery.ParseModulesKey(k); err == nil {
if name, err := meshcatalogclient.ParseModulesKey(k); err == nil {
held[name] = true
}
}
@@ -111,12 +114,12 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
}
recorded := map[string]bool{}
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Record != nil {
recorded[node] = true
}
}
@@ -207,7 +210,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
return nil, err
}
for _, c := range changes {
if c.Op == plc.OpSeed {
if c.Op == mmac.OpSeed {
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
}
}
@@ -223,10 +226,11 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
if err != nil {
return nil, err
}
enrollers := plc.EnrollersOf(current)
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: plc.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant})
if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by), delivery.EnrolmentModule); refusal != "" {
if refusal := plc.AddNodeRefusal(p, moduleOfCaller(by, enrollers), enrollers); refusal != "" {
findings = append(findings, refusal)
}
if len(findings) > 0 {
@@ -264,12 +268,14 @@ func warrantOf(v any) (*mma.Warrant, error) {
return &out, nil
}
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven
// caller is that module's (its bus user `<node>.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller
// as proven, which the refusal names.
func moduleOfCaller(by string) string {
if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) {
return delivery.EnrolmentModule
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: the holder of the seat
// mesh-node-enroller when the proven caller is that module's (its bus user `<node>.<module>`, never a person's
// `person.<module>`), else the caller as proven, which the refusal names.
func moduleOfCaller(by string, enrollers []string) string {
for _, module := range enrollers {
if (stdio.Callers{module}).Allows(by) {
return module
}
}
return by
}
@@ -404,9 +410,9 @@ func (s *seatTools) doPlacement(args map[string]any) (any, error) {
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
// every node's answer now.
type openProposal struct {
Proposal mma.Proposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
Proposal mma.Proposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]nodeengine.CandidateAnswer `json:"answers,omitempty"`
}
// doProposals is the verb proposals (§10).
@@ -423,9 +429,9 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
if err != nil {
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
}
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
@@ -433,13 +439,13 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
if err != nil || !found {
continue
}
h, err := delivery.DecodeHaveNode(raw)
h, err := nodeengineclient.DecodeHaveNode(raw)
if err != nil {
continue
}
for id, a := range h.Candidates {
if answers[id] == nil {
answers[id] = map[string]delivery.CandidateAnswer{}
answers[id] = map[string]nodeengine.CandidateAnswer{}
}
answers[id][node] = a
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go
+34 -27
View File
@@ -5,6 +5,12 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/buckets"
"os"
"reflect"
"slices"
@@ -16,7 +22,6 @@ import (
mma "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/provisions"
"github.com/nats-io/nats.go/jetstream"
@@ -42,16 +47,17 @@ func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := mma.Placement{V: delivery.Version, Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}
anchor := mma.Placement{V: keys.Version, Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}, {Module: "mesh-enrolment"}},
Holds: []mma.Holding{{Claim: "mesh-node-enroller", Scope: "mesh", Module: "mesh-enrolment"}}}
st := memStore{}
for _, state := range []string{mma.StatePlacement, mma.StateProposals, mma.StateCandidates, modulesState, haveState} {
st[state] = &memKV{values: map[string][]byte{}}
}
for _, m := range []string{"web", "shell"} {
for _, m := range []string{"web", "shell", "mesh-enrolment"} {
st[modulesState].values[m] = []byte("{}")
}
for _, n := range []string{"anchor", "laptop"} {
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
raw, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: n}})
st[haveState].values[n+".node"] = raw
}
var asked []string
@@ -164,7 +170,7 @@ func TestTheSeatsVerbs(t *testing.T) {
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
// of a node running modules, or with another node's warrant, it is refused.
joiner, _ := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: "joiner", Decided: mma.NodeDecided{Address: "10.9.0.7"}, Modules: []mma.PlacedModule{}})
joiner, _ := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: "joiner", Decided: mma.NodeDecided{Address: "10.9.0.7"}, Modules: []mma.PlacedModule{}})
st[mma.StatePlacement].values["placement.joiner"] = joiner
removeNode := func(node, warrant string) (any, error) {
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
@@ -190,14 +196,14 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
w := newWorld()
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
pl, _ := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: "ace", Modules: []mma.PlacedModule{{Module: "web"}}})
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("2", 64)
pl, _ := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: "ace", Modules: []mma.PlacedModule{{Module: "web"}}})
pkey, _ := mmac.LegacyKey("ace")
w.putEntry(plc.PlacementBucket, pkey, pl, 1, time.Now())
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
hk, _ := delivery.HaveNodeKey(node)
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
hn, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: digest})
hk, _ := nodeengineclient.HaveNodeKey(node)
w.putEntry(buckets.Have, hk, hn, 1, time.Now())
}
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
t.Errorf("ace: %+v %v", d, ok)
@@ -213,7 +219,7 @@ func TestTheSeatsVerbs(t *testing.T) {
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
pl2, _ := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
pl2, _ := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
pkey2, _ := mmac.LegacyKey("g14")
w.putEntry(plc.PlacementBucket, pkey2, pl2, 1, time.Now())
p.unplacedFindings()
@@ -224,9 +230,9 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) {
writes := 0
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{plc.SeatHoldersBucket: kv}}
p := &passer{w: newWorld(), logf: t.Logf, kvs: kvSet{plc.SeatHoldersBucket: kv}}
place := func(node string, holds ...mma.Holding) {
pl, err := mmac.EncodePlacement(mma.Placement{V: delivery.Version, Node: node, Modules: []mma.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
pl, err := mmac.EncodePlacement(mma.Placement{V: keys.Version, Node: node, Modules: []mma.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
if err != nil {
t.Fatal(err)
}
@@ -259,7 +265,7 @@ func TestTheSeatsVerbs(t *testing.T) {
ctx := context.Background()
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
var logs []string
p := &passer{w: newWorld(), kvs: buckets{plc.PlacementBucket: kv}, now: time.Now,
p := &passer{w: newWorld(), kvs: kvSet{plc.PlacementBucket: kv}, now: time.Now,
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
put := func(key string, value []byte, rev uint64) {
p.w.putEntry(plc.PlacementBucket, key, value, rev, time.Now())
@@ -319,7 +325,7 @@ func TestTheSeatsVerbs(t *testing.T) {
})
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil }}}
failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
@@ -368,31 +374,32 @@ func TestTheSeatsVerbs(t *testing.T) {
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
func wantsVerbs(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
fpOld, fpNew := meshcatalogclient.SourcePrefix2+strings.Repeat("1", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", delivery.ComposedRead} {
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings", buckets.ComposedRead} {
st[state] = &memKV{values: map[string][]byte{}}
}
// ADR 0593 §6, §8: what each node published, every node's bucket as one state: anchor's own composition and its web at
// a fingerprint, and laptop's web refused there.
webKey, _ := delivery.ComposedKey("anchor", "web", fpNew)
refusedKey, _ := delivery.ComposedKey("laptop", "web", fpNew)
st[delivery.ComposedRead].values["anchor.node"], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor"})
st[delivery.ComposedRead].values[webKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "anchor", Module: "web", FP: fpNew})
st[delivery.ComposedRead].values[refusedKey], _ = delivery.EncodeComposed(delivery.Composed{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
partNew, _ := meshcatalogclient.KeyPart(fpNew)
webKey, _ := nodeengineclient.ComposedKey("anchor", "web", partNew)
refusedKey, _ := nodeengineclient.ComposedKey("laptop", "web", partNew)
st[buckets.ComposedRead].values["anchor.node"], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "anchor"})
st[buckets.ComposedRead].values[webKey], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "anchor", Module: "web", FP: fpNew})
st[buckets.ComposedRead].values[refusedKey], _ = nodeengineclient.EncodeComposition(nodeengineclient.Composition{Node: "laptop", Module: "web", FP: fpNew, Refused: "a required value is unset"})
manifest := func(name, extra string) {
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
digest := delivery.ManifestDigest([]byte(raw))
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw})
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
digest := meshcatalogclient.ManifestDigest([]byte(raw))
st[manifestsState].values[digest], _ = meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: name, Manifest: raw})
st[modulesState].values[name], _ = meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{V: keys.Version, Fp2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
}
manifest("web", "")
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
st[haveState].values[node+".nats"], _ = nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
}
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
st[haveState].values["anchor.node"], _ = nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Composed: "sha256:" + strings.Repeat("c", 64)})
var asked []string
var said []string
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go
+26 -23
View File
@@ -4,6 +4,9 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/puts"
"os"
"regexp"
"slices"
@@ -81,26 +84,26 @@ func (s *putTools) version(module, version string) (string, error) {
if !found {
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
}
m, err := delivery.DecodeModule(raw)
m, err := meshcatalogclient.DecodeModule(raw)
if err != nil {
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
}
if m.FP2 == "" {
if m.Fp2 == "" {
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
}
return m.FP2, nil
return m.Fp2, nil
case fullFP.MatchString(v):
return v, nil
case prefix.MatchString(v):
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
start := strings.TrimPrefix(v, meshcatalogclient.SourcePrefix2)
keys, err := s.st.keys(builds.BuilderModule + ".builds")
if err != nil {
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
}
var found []string
for _, k := range keys {
m, fp, err := delivery.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
m, fp, err := builds.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2), start) {
found = append(found, fp)
}
}
@@ -120,14 +123,14 @@ func (s *putTools) version(module, version string) (string, error) {
type putAnswer struct {
Node string `json:"node"`
Module string `json:"module"`
delivery.Put
puts.Put
Follow string `json:"follow,omitempty"`
}
// doPut is the tool assigner_put.
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := delivery.PutKey(node, module)
key, err := puts.Key(node, module)
if err != nil {
return nil, err
}
@@ -151,15 +154,15 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
if raw, _, found, err := s.st.get(putsState, key); err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
} else if found {
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
if before, err := puts.Decode(raw); err == nil && !puts.Ended(before.Step) {
if before.FP == fp && before.Hold == hold {
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
}
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
}
}
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
data, err := delivery.EncodePut(p)
p := puts.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, puts.Asked, words)
data, err := puts.Encode(p)
if err != nil {
return nil, err
}
@@ -171,15 +174,15 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
}
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
func (s *putTools) await(key string, p puts.Put) puts.Put {
deadline := time.After(answerWithin)
for {
s.mu.Lock()
raw, changed := s.heard[key], s.changed
s.mu.Unlock()
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
if now, err := puts.Decode(raw); err == nil && now.At.Equal(p.At) {
p = now
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
if now.Step != puts.Asked && now.Step != puts.Build && now.Step != puts.Compose {
return p // it waits on a person or the node, or ended
}
}
@@ -200,7 +203,7 @@ func followPut(node, module string) string {
// doRelease is the tool assigner_release.
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := delivery.PutKey(node, module)
key, err := puts.Key(node, module)
if err != nil {
return nil, err
}
@@ -214,15 +217,15 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if !found {
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
}
p, err := delivery.DecodePut(raw)
p, err := puts.Decode(raw)
if err != nil {
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
}
if delivery.PutEnded(p.Step) {
if puts.Ended(p.Step) {
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
}
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := delivery.EncodePut(p)
p = p.Said(s.now(), puts.Released, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
data, err := puts.Encode(p)
if err != nil {
return nil, err
}
@@ -242,7 +245,7 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
sort.Strings(keys)
out := []putAnswer{}
for _, key := range keys {
node, module, err := delivery.ParsePutKey(key)
node, module, err := puts.ParseKey(key)
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
continue
}
@@ -250,7 +253,7 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
if err != nil || !found {
continue
}
p, err := delivery.DecodePut(raw)
p, err := puts.Decode(raw)
if err != nil {
continue
}
@@ -371,7 +374,7 @@ func serveTools() error {
// shortFP is a fingerprint's first twelve hex digits.
func shortFP(fp string) string {
part, err := delivery.FingerprintKeyPart(fp)
part, err := meshcatalogclient.KeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go
+23 -6
View File
@@ -2,6 +2,10 @@ package main
import (
"context"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/buckets"
"slices"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -62,7 +66,7 @@ func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign
if len(g) == 0 {
return nil
}
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
served, unread := graph.Served(graphHaves(n.Haves), func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
func(module string) []string { return g[module].Serves })
if len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
@@ -81,10 +85,10 @@ func (p *passer) offering(ctx context.Context, get getter, trunk map[string]cata
continue
}
for module, h := range haves {
if h.Phase != delivery.PhaseRunning || h.Health != delivery.HealthOK {
if h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthOK {
continue
}
m, ok := p.manifestOf(ctx, get, module, h.FP)
m, ok := p.manifestOf(ctx, get, module, h.Fp)
if !ok {
m = trunk[module]
}
@@ -108,15 +112,15 @@ func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([
// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests),
// and whether it could be read.
func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) (catalogue.Manifest, bool) {
key, err := delivery.BuilderBuildKey(module, fp)
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return catalogue.Manifest{}, false
}
raw := p.w.value(delivery.BuilderBuildsBucket, key)
raw := p.w.value(buckets.Builds, key)
if raw == nil {
return catalogue.Manifest{}, false
}
b, err := delivery.DecodeBuilderBuild(raw)
b, err := builds.DecodeBuilderBuild(raw)
if err != nil || b.Manifest == "" {
return catalogue.Manifest{}, false
}
@@ -126,3 +130,16 @@ func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string)
}
return got.manifest, true
}
// graphHaves is the haves as the module graph's Served reads them, which keeps the earlier have form for what it reads: the
// build, the phase and the health of each module on each node.
func graphHaves(haves map[string]map[string]nodeengine.HaveByNodeModuleValue) map[string]map[string]delivery.Have {
out := map[string]map[string]delivery.Have{}
for node, modules := range haves {
out[node] = map[string]delivery.Have{}
for module, h := range modules {
out[node][module] = delivery.Have{FP: h.Fp, Build: h.Build, Phase: delivery.Phase(h.Phase), Health: delivery.Health(h.Health)}
}
}
return out
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go
+6 -5
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"mesh-delivery-assigner/internal/buckets"
"slices"
"time"
@@ -53,10 +54,10 @@ var readTools = []viewedTool{
var viewedStates = map[string]string{
mma.StatePlacement: plc.PlacementBucket,
mma.StateSeatHolders: plc.SeatHoldersBucket,
"wants": delivery.AssignerWantsBucket,
"wants": buckets.Wants,
health.FindingsState: assign.FindingsBucket,
"asks": delivery.AssignerAsksBucket,
"differences": delivery.AssignerDifferencesBucket,
"asks": buckets.Asks,
"differences": buckets.Differences,
}
// viewTools are the assigner's first-view and read tools, reading only its own states, so they answer as soon as its
@@ -78,7 +79,7 @@ func keptSource(name string) views.Source { return views.KeptSource{State: stdio
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs buckets, findings assign.Findings, errs chan<- error, logf func(string, ...any)) {
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, findings assign.Findings, errs chan<- error, logf func(string, ...any)) {
tooLarge := func(state, key string, size int) {
if findings.Put == nil {
return
@@ -139,7 +140,7 @@ type indexerViews struct {
// indexerFirstViews are the indexer's states the assigner takes by first view and the views family, each by the name of
// the indexer's bucket the world keeps it under: manifests and modules (ADR 0621 §2), and the module graph, whose verb
// edges a want waits on (ADR 0614 §2, §3; ADR 0631 §1).
var indexerFirstViews = map[string]string{"manifests": delivery.ManifestsBucket, "modules": delivery.ModulesBucket, graph.State: graph.Bucket}
var indexerFirstViews = map[string]string{"manifests": buckets.Manifests, "modules": buckets.Modules, graph.State: graph.Bucket}
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
// between two failed asks; zero is the views package's (views.Bound, two minutes, ADR 0615 §6; and a first wait of five
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go
+22 -17
View File
@@ -4,8 +4,13 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
meshhandacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
"git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"sort"
"strconv"
"time"
@@ -27,7 +32,7 @@ import (
// The states the verbs read, as the runtime names them.
const (
policiesState = delivery.AssignerPoliciesState
policiesState = buckets.PoliciesState
manifestsState = delivery.IndexerModule + ".manifests"
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
PolicyChangedEvent = "policy-changed"
@@ -94,14 +99,14 @@ func (s *wantsTools) manifest(digest string) ([]byte, bool, error) {
}
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]meshcatalog.ModulesByModuleValue, error) {
mods, err := s.values(modulesState)
if err != nil {
return nil, nil, err
}
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
out, held := map[string]catalogue.Manifest{}, map[string]meshcatalog.ModulesByModuleValue{}
for name, raw := range mods {
m, err := delivery.DecodeModule(raw)
m, err := meshcatalogclient.DecodeModule(raw)
if err != nil {
continue
}
@@ -113,12 +118,12 @@ func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]deli
if err != nil || !found {
continue
}
kept, err := delivery.DecodeManifest(mraw)
kept, err := meshcatalogclient.DecodeManifest(mraw)
if err != nil {
continue
}
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
if man, err := catalogue.Decode([]byte(kept.JSON)); err == nil {
if man, err := catalogue.Decode([]byte(kept.Manifest)); err == nil {
out[name] = man
}
}
@@ -153,15 +158,15 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
composed := map[string]string{}
for key, raw := range haves {
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
if node, _, isNode, err := nodeengineclient.ParseHaveKey(key); err == nil && isNode {
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Composed != "" {
composed[node] = h.Composed
}
}
}
out := map[string]any{"composed": composed}
// And what every node published (ADR 0593 §8), read only to compare.
if values, err := s.values(delivery.ComposedRead); err != nil {
if values, err := s.values(buckets.ComposedRead); err != nil {
out["published-unread"] = err.Error()
} else {
out["published"] = published(values)
@@ -268,22 +273,22 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
if bus == "" {
return nil, errors.New("the indexer holds no module that provides the bus")
}
newest := held[bus].FP2
newest := held[bus].Fp2
haves, err := s.values(haveState)
if err != nil {
return nil, err
}
var nodes []busNode
for key, raw := range haves {
node, module, isNode, err := delivery.ParseHaveKey(key)
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode || module != bus {
continue
}
h, err := delivery.DecodeHave(raw)
if err != nil || h.Phase == delivery.PhaseRemoved {
h, err := nodeengineclient.DecodeHave(raw)
if err != nil || h.Phase == nodeengineclient.PhaseRemoved {
continue
}
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
nodes = append(nodes, busNode{Node: node, FP: h.Fp, Behind: newest != "" && h.Fp != newest})
}
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
@@ -292,13 +297,13 @@ func (s *wantsTools) doBus(map[string]any) (any, error) {
} else {
out["steps"] = steps
}
puts, err := s.values(putsState)
kept, err := s.values(putsState)
if err != nil {
out["puts-unread"] = err.Error()
} else {
mine := map[string]json.RawMessage{}
for k, v := range puts {
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
for k, v := range kept {
if _, module, err := puts.ParseKey(k); err == nil && module == bus {
mine[k] = v
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go
+90 -71
View File
@@ -3,9 +3,19 @@ package main
import (
"context"
"errors"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/keys"
"maps"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sort"
"strings"
"sync"
"time"
@@ -37,7 +47,7 @@ type world struct {
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
meta map[string]entryMeta
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
trunkBuilt map[string]delivery.BuildLogEntry
trunkBuilt map[string]builds.BuildLogEntry
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
placement *mmac.Keys
@@ -61,7 +71,7 @@ type parsed struct {
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: mmac.NewKeys(),
meta: map[string]entryMeta{}, trunkBuilt: map[string]builds.BuildLogEntry{}, placement: mmac.NewKeys(),
sourceGroups: map[uint64]string{}}
}
@@ -78,7 +88,7 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
if bucket == plc.PlacementBucket {
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
}
if bucket == delivery.SourceBucket && key == delivery.SourceMainKey {
if bucket == buckets.Source && key == delivery.SourceMainKey {
if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" {
w.sourceGroups[revision] = s.Group
}
@@ -92,17 +102,17 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
// unknown: none, and the move is asked of a person.
func (w *world) release(module, fp string) (string, bool) {
t, ok := w.trunk()[module]
if !ok || fp == "" || t.FP2 != fp || t.FPSrc == 0 || t.Commit == "" {
if !ok || fp == "" || t.Fp2 != fp || t.FpSrc == 0 || t.Commit == "" {
return "", false
}
w.mu.Lock()
group := w.sourceGroups[t.FPSrc]
group := w.sourceGroups[uint64(t.FpSrc)]
w.mu.Unlock()
key, err := delivery.MergesKey(group)
if group == "" || err != nil {
return "", false
}
raw := w.value(delivery.MergesBucket, key)
raw := w.value(buckets.Merges, key)
if raw == nil {
return "", false
}
@@ -146,17 +156,17 @@ func (w *world) put(bucket, key string, value []byte) {
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
_, _ = w.placement.Put(key, value, 0, false)
}
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
w.healthy[key] = append(list, h.FP)
if bucket == buckets.Have {
if h, err := nodeengineclient.DecodeHave(value); err == nil && h.Fp != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.Fp })
w.healthy[key] = append(list, h.Fp)
}
}
}
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
func (w *world) lastHealthy(node, module, fp string) string {
key, err := delivery.HaveKey(node, module)
key, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return ""
}
@@ -222,7 +232,7 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
if ok {
return p
}
key, err := delivery.ManifestsKey(digest)
key, err := meshcatalogclient.ManifestsKey(digest)
if err != nil {
return parsed{err: err.Error()}
}
@@ -233,10 +243,10 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
if err != nil {
return parsed{err: err.Error()}
}
m, err := delivery.DecodeManifest(value)
m, err := meshcatalogclient.DecodeManifest(value)
if err != nil {
p = parsed{err: err.Error()}
} else if man, err := catalogue.Decode([]byte(m.JSON)); err != nil {
} else if man, err := catalogue.Decode([]byte(m.Manifest)); err != nil {
// Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the
// module's value where it is read (world.mesh).
p = parsed{module: m.Module, err: "it does not decode: " + err.Error()}
@@ -288,14 +298,14 @@ func (w *world) placedNodes() map[string]bool {
}
// trunk is every module the indexer holds at the trunk, with its value.
func (w *world) trunk() map[string]delivery.Module {
out := map[string]delivery.Module{}
for key, value := range w.read(delivery.ModulesBucket) {
name, err := delivery.ParseModulesKey(key)
func (w *world) trunk() map[string]meshcatalog.ModulesByModuleValue {
out := map[string]meshcatalog.ModulesByModuleValue{}
for key, value := range w.read(buckets.Modules) {
name, err := meshcatalogclient.ParseModulesKey(key)
if err != nil {
continue
}
if m, err := delivery.DecodeModule(value); err == nil {
if m, err := meshcatalogclient.DecodeModule(value); err == nil {
out[name] = m
}
}
@@ -304,49 +314,62 @@ func (w *world) trunk() map[string]delivery.Module {
// records is each node's record in its own have key (`<node>.node`): the nodes a placement proposal is answered by,
// with those placement names (ADR 0593 §8).
func (w *world) records() map[string]delivery.NodeRecord {
out := map[string]delivery.NodeRecord{}
for key, value := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
func (w *world) records() map[string]nodeengine.NodeRecord {
out := map[string]nodeengine.NodeRecord{}
for key, value := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
out[node] = *h.Record
}
}
return out
}
// presenceOf reads one key of the watchman's presence: a node's key (`<node>`) with the node's online or not; the keys of
// the parts (`part.<part>.<instance>`) are not a node's and are none here.
func presenceOf(key string, value []byte) (node string, online, ok bool) {
if strings.Contains(key, ".") {
return "", false, false
}
node, ok = watchdogs.ParsePresenceByNodeKey(key)
if !ok {
return "", false, false
}
p, err := watchdogs.DecodePresenceByNodeValue(value)
if err != nil {
return "", false, false
}
return node, p.Online, true
}
// online is, by node, whether the watchman holds the node online.
func (w *world) online() map[string]bool {
out := map[string]bool{}
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if p, err := delivery.DecodePresence(value); err == nil {
out[node] = p.Online
for key, value := range w.read(buckets.Presence) {
if node, online, ok := presenceOf(key, value); ok {
out[node] = online
}
}
return out
}
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]delivery.Want {
out := map[string]map[string]delivery.Want{}
for key, value := range w.read(delivery.AssignerWantsBucket) {
node, module, err := delivery.ParseWantKey(key)
func (w *world) wantsOf() map[string]map[string]mwc.Want {
out := map[string]map[string]mwc.Want{}
for key, value := range w.read(buckets.Wants) {
node, module, err := mwc.ParseWantKey(key)
if err != nil {
continue
}
want, err := delivery.DecodeWant(value)
want, err := mwc.DecodeWant(value)
if err != nil {
continue
}
if out[module] == nil {
out[module] = map[string]delivery.Want{}
out[module] = map[string]mwc.Want{}
}
out[module][node] = want
}
@@ -355,32 +378,28 @@ func (w *world) wantsOf() map[string]map[string]delivery.Want {
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{},
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if p, err := delivery.DecodePresence(value); err == nil {
n.Online[node] = p.Online
for key, value := range w.read(buckets.Presence) {
if node, online, ok := presenceOf(key, value); ok {
n.Online[node] = online
}
}
for key, value := range w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
for key, value := range w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil {
continue
}
if isNode {
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
n.Reads[node] = h.Record.ManifestReads
}
continue
}
if h, err := delivery.DecodeHave(value); err == nil {
if h, err := nodeengineclient.DecodeHave(value); err == nil {
if n.Haves[node] == nil {
n.Haves[node] = map[string]delivery.Have{}
n.Haves[node] = map[string]nodeengine.HaveByNodeModuleValue{}
}
n.Haves[node][module] = h
}
@@ -411,7 +430,7 @@ func (w *world) value(bucket, key string) []byte {
// grantsBucket is the bucket the grants are read from: the grants part's state `grants`, always (ADR 0571 clause 9); no
// manifest but the controller's reads the controller's grants.
func (w *world) grantsBucket() string { return delivery.PartGrantsBucket }
func (w *world) grantsBucket() string { return buckets.Grants }
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
@@ -423,29 +442,29 @@ func (w *world) has(bucket, key string) bool {
// built says whether the builder's build of a module at a fingerprint is done.
func (w *world) built(module, fp string) bool {
key, err := delivery.BuilderBuildKey(module, fp)
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return false
}
w.mu.Lock()
value, ok := w.values[delivery.BuilderBuildsBucket][key]
value, ok := w.values[buckets.Builds][key]
w.mu.Unlock()
if !ok {
return false
}
b, err := delivery.DecodeBuilderBuild(value)
return err == nil && b.State == delivery.BuildDone
b, err := builds.DecodeBuilderBuild(value)
return err == nil && b.State == builds.BuildDone
}
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build key's needs, ADR
// 0618 §1): nil when it is not done or names none.
func (w *world) buildNeeds(module, fp string) []string {
key, err := delivery.BuilderBuildKey(module, fp)
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return nil
}
b, err := delivery.DecodeBuilderBuild(w.value(delivery.BuilderBuildsBucket, key))
if err != nil || b.State != delivery.BuildDone {
b, err := builds.DecodeBuilderBuild(w.value(buckets.Builds, key))
if err != nil || b.State != builds.BuildDone {
return nil
}
return b.Needs
@@ -476,9 +495,9 @@ func (w *world) knownNodes() []string {
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
// controller's copy (mesh-controller_composed) is read no more (issue 774).
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
func (w *world) composed(node, module, fp string) (assign.ComposedDigest, bool) {
if !w.placedNodes()[node] {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
return w.ownComposed(node)
}
@@ -486,7 +505,7 @@ func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
func (w *world) answer(key string) (delivery.Answer, bool) {
w.mu.Lock()
value, ok := w.values[delivery.AnswersBucket][key]
value, ok := w.values[buckets.Answers][key]
w.mu.Unlock()
if !ok {
return delivery.Answer{}, false
@@ -512,20 +531,20 @@ func (w *world) changed(bucket, key string) uint64 {
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
key, err := delivery.HaveNodeKey(node)
func (w *world) ownComposed(node string) (assign.ComposedDigest, bool) {
key, err := nodeengineclient.HaveNodeKey(node)
if err != nil {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
raw, ok := w.read(delivery.HaveBucket)[key]
raw, ok := w.read(buckets.Have)[key]
if !ok {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
h, err := delivery.DecodeHaveNode(raw)
h, err := nodeengineclient.DecodeHaveNode(raw)
if err != nil || h.Composed == "" {
return delivery.ComposedDigest{}, false
return assign.ComposedDigest{}, false
}
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
return assign.ComposedDigest{V: keys.Version, Digest: h.Composed, Judgement: assign.JudgedNeither,
Words: "the node composes its own"}, true
}
@@ -534,8 +553,8 @@ func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
func (w *world) unplacedNodes() []string {
placed := w.placedNodes()
var out []string
for key := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
for key := range w.read(buckets.Have) {
node, _, isNode, err := nodeengineclient.ParseHaveKey(key)
if err == nil && isNode && !placed[node] {
out = append(out, node)
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go
+18 -14
View File
@@ -2,6 +2,10 @@ package main
import (
"context"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"mesh-delivery-assigner/internal/buckets"
"slices"
"strings"
"testing"
@@ -15,8 +19,8 @@ type fakeManifests map[string][]byte
func (f fakeManifests) add(t *testing.T, module, body string) string {
t.Helper()
digest := delivery.ManifestDigest([]byte(body))
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
digest := meshcatalogclient.ManifestDigest([]byte(body))
value, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: body})
if err != nil {
t.Fatal(err)
}
@@ -31,7 +35,7 @@ func (f fakeManifests) get(_ context.Context, key string) ([]byte, error) {
return nil, errNotThere
}
func fp(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
func fp(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) }
func pull(repo string, number int, modules map[string]delivery.CandidateModule) delivery.CandidatePull {
p := delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: strings.Repeat("3", 40)}, Fps: modules}
@@ -45,19 +49,19 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.web", []byte(`{"v":1}`))
w.put(buckets.Have, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(buckets.Have, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
}
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
if got := w.changed(buckets.Have, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := delivery.PartGrantKey("web", fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{Accounts: []string{"ace.web"}})
fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{Accounts: []string{"ace.web"}})
for _, row := range []struct {
name string
inPart bool
@@ -66,11 +70,11 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
{"the part's grant", true, true},
{"no grant", false, false},
} {
p := newPasser(buckets{}, time.Now())
p := newPasser(kvSet{}, time.Now())
if row.inPart {
p.w.put(delivery.PartGrantsBucket, gkey, grant)
p.w.put(buckets.Grants, gkey, grant)
}
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
if got := p.w.grantsBucket(); got != buckets.Grants {
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
}
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
modules/mesh-delivery-assigner/go.mod
+27
View File
@@ -37,3 +37,30 @@ require (
)
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds
replace git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate => ../mesh-grants/grantsstate
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient => ../../contracts/go/clients/meshcatalogclient
require git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient => ../../contracts/go/clients/meshdeliveryclient
require git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog => ../../contracts/go/seats/meshcatalog
require git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery => ../../contracts/go/seats/meshdelivery
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshnodeenroller v0.0.0-00010101000000-000000000000
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks => ./asks
require git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate => ../mesh-delivery-view/viewstate
require git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
modules/mesh-delivery-assigner/internal/assign/askjson.go
+17 -16
View File
@@ -3,24 +3,25 @@ package assign
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (delivery.AssignerAsk.Ask).
func assignerAsk(asking int, a oc.Ask) delivery.AssignerAsk {
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (asks.Ask.Ask).
func assignerAsk(asking int, a oc.Ask) asks.Ask {
raw, err := json.Marshal(a)
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
}
return delivery.AssignerAsk{Asking: asking, Ask: raw}
return asks.Ask{Asking: asking, Ask: raw}
}
// AskOf is the router's ask an asks-state value carries.
func AskOf(a delivery.AssignerAsk) (oc.Ask, error) {
func AskOf(a asks.Ask) (oc.Ask, error) {
var ask oc.Ask
if err := json.Unmarshal(a.Ask, &ask); err != nil {
return oc.Ask{}, fmt.Errorf("assign: the value carries no router's ask: %w", err)
@@ -28,32 +29,32 @@ func AskOf(a delivery.AssignerAsk) (oc.Ask, error) {
return ask, nil
}
// EncodeAsk is delivery.EncodeAssignerAsk at key, after the checks every keeper of an ask owes (occ.Carried, ADR 0627 §4):
// the ask's id is delivery.AskingID(key, asking), and it offers at least one option.
func EncodeAsk(key string, a delivery.AssignerAsk) ([]byte, error) {
// EncodeAsk is asks.EncodeAsk at key, after the checks every keeper of an ask owes (occ.Carried, ADR 0627 §4):
// the ask's id is meshdeliveryclient.AskingID(key, asking), and it offers at least one option.
func EncodeAsk(key string, a asks.Ask) ([]byte, error) {
if _, err := checkedAsk(key, a); err != nil {
return nil, err
}
return delivery.EncodeAssignerAsk(key, a)
return asks.EncodeAsk(key, a)
}
func checkedAsk(key string, a delivery.AssignerAsk) (oc.Ask, error) {
ask, err := occ.Carried(a.Ask, delivery.AskingID(key, a.Asking))
func checkedAsk(key string, a asks.Ask) (oc.Ask, error) {
ask, err := occ.Carried(a.Ask, meshdeliveryclient.AskingID(key, a.Asking))
if err != nil {
return oc.Ask{}, fmt.Errorf("assign: %w", err)
}
return ask, nil
}
// DecodeAsk reads the ask at key (delivery.DecodeAssignerAsk) with the router's ask it carries.
func DecodeAsk(key string, data []byte) (delivery.AssignerAsk, oc.Ask, error) {
a, err := delivery.DecodeAssignerAsk(key, data)
// DecodeAsk reads the ask at key (asks.DecodeAsk) with the router's ask it carries.
func DecodeAsk(key string, data []byte) (asks.Ask, oc.Ask, error) {
a, err := asks.DecodeAsk(key, data)
if err != nil {
return delivery.AssignerAsk{}, oc.Ask{}, err
return asks.Ask{}, oc.Ask{}, err
}
ask, err := checkedAsk(key, a)
if err != nil {
return delivery.AssignerAsk{}, oc.Ask{}, err
return asks.Ask{}, oc.Ask{}, err
}
return a, ask, nil
}
modules/mesh-delivery-assigner/internal/assign/asks.go
+23 -19
View File
@@ -2,6 +2,10 @@ package assign
import (
"fmt"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"regexp"
"strings"
"time"
@@ -11,13 +15,13 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (asks.HoldAskKey): keep it held or
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: delivery.AskingID(key, asking),
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
@@ -42,9 +46,9 @@ const (
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
func RestoreAsk(key, node, module, fp, words string, now time.Time) asks.Ask {
return assignerAsk(0, oc.Ask{
ID: delivery.AskingID(key, 0),
ID: meshdeliveryclient.AskingID(key, 0),
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
@@ -64,7 +68,7 @@ func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.Ass
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) oc.Condition {
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
@@ -149,21 +153,21 @@ func LiftedCondition(module, fp string) oc.Condition {
const (
OptionYes = "yes"
OptionNo = "no"
OptionBackupAgain = delivery.BackupAgain
OptionBackupSkip = delivery.BackupSkip
OptionBackupAgain = mwc.BackupAgain
OptionBackupSkip = mwc.BackupSkip
)
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + need.Words,
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) asks.Ask {
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + need.Words,
}[need.Judgement]
return assignerAsk(0, oc.Ask{
ID: delivery.AskingID(key, 0),
ID: meshdeliveryclient.AskingID(key, 0),
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
@@ -179,7 +183,7 @@ func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) del
}
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
// give one id (meshdeliveryclient.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
if after < 0 {
return 0
@@ -192,7 +196,7 @@ var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a delivery.AssignerAsk) string {
func BackupAskDigest(a asks.Ask) string {
ask, err := AskOf(a)
if err != nil {
return ""
@@ -212,9 +216,9 @@ func BackupAnswerWords(option string, asking int) string {
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) asks.Ask {
return assignerAsk(asking, oc.Ask{
ID: delivery.AskingID(key, asking),
ID: meshdeliveryclient.AskingID(key, asking),
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
module, short(fp), node, words, module, digest),
@@ -232,7 +236,7 @@ func BackupAsk(key, node, module, fp, digest, words string, asking int, now time
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) oc.Condition {
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
subject := oc.ConditionSubject{Scope: "module", ID: module, Machine: node}
if !raised {
return oc.Condition{Key: key, Subject: subject, State: occ.StateCleared}
modules/mesh-delivery-assigner/internal/assign/asks_test.go
+26 -20
View File
@@ -2,6 +2,12 @@ package assign
import (
"fmt"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"mesh-delivery-assigner/internal/puts"
"slices"
"strings"
"testing"
@@ -21,7 +27,7 @@ func TestTheHoldAskIsValid(t *testing.T) {
}
func theHoldAskIsValid(t *testing.T) {
key, err := delivery.HoldAskKey("mail", fNew)
key, err := asks.HoldAskKey("mail", fNew)
if err != nil {
t.Fatal(err)
}
@@ -39,13 +45,13 @@ func theHoldAskIsValid(t *testing.T) {
}
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
if again.Asking != 7 || mustAsk(t, again).ID != delivery.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
if again.Asking != 7 || mustAsk(t, again).ID != meshdeliveryclient.AskingID(key, 7) || mustAsk(t, again).ID == mustAsk(t, a).ID {
t.Errorf("the asking after a lift = %d %s", again.Asking, mustAsk(t, again).ID)
}
if _, err := EncodeAsk(key, again); err != nil {
t.Errorf("the ask asked again cannot be written: %v", err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
if _, err := viewstate.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
// The join (A1d ruling 1): mesh-sdk no longer refuses another id, no option or no ask in an assigner's ask; EncodeAsk does.
@@ -57,7 +63,7 @@ func theHoldAskIsValid(t *testing.T) {
ask := mustAsk(t, good)
ask.Options = nil
empty.Ask = assignerAsk(0, ask).Ask
for name, a := range map[string]delivery.AssignerAsk{"another id": other, "no option": empty, "no ask": {}} {
for name, a := range map[string]asks.Ask{"another id": other, "no option": empty, "no ask": {}} {
if _, err := EncodeAsk(key, a); err == nil {
t.Errorf("%s: written", name)
}
@@ -68,7 +74,7 @@ func theHoldAskIsValid(t *testing.T) {
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
func theRestoreAskIsValid(t *testing.T) {
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
key, err := asks.RestoreAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
@@ -80,7 +86,7 @@ func theRestoreAskIsValid(t *testing.T) {
if _, err := EncodeAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
if _, err := viewstate.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
var ids []string
@@ -137,11 +143,11 @@ func saidConditionsAreFindings(t *testing.T) {
raised, cleared oc.Condition
key string
}{
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".held"},
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
} {
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
f := Findings{Put: func(k string, v []byte) error {
@@ -174,32 +180,32 @@ func saidConditionsAreFindings(t *testing.T) {
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
now := time.Now()
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
rkey, err := asks.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
if err != nil {
t.Fatal(err)
}
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := asks.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
dgst := "sha256:" + strings.Repeat("b", 64)
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
for key, a := range map[string]asks.Ask{rkey: r, bkey: b} {
if err := occ.Check(mustAsk(t, a), now); err != nil {
t.Errorf("%s: the router refuses the ask: %v", key, err)
}
if _, err := EncodeAsk(key, a); err != nil {
t.Errorf("%s: cannot be written: %v", key, err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
if _, err := viewstate.ViewAnswerKey(key); err != nil {
t.Errorf("%s: the view may not answer it: %v", key, err)
}
}
if !strings.Contains(mustAsk(t, r).Explanation, "gone") || mustAsk(t, r).Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", mustAsk(t, r))
}
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != delivery.BackupAgain || mustAsk(t, b).Options[1].ID != delivery.BackupSkip {
if !strings.Contains(mustAsk(t, b).Explanation, "no space left") || mustAsk(t, b).Options[0].ID != mwc.BackupAgain || mustAsk(t, b).Options[1].ID != mwc.BackupSkip {
t.Errorf("the backup ask = %+v", mustAsk(t, b))
}
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
@@ -221,8 +227,8 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
}
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
// so the view cannot answer it, and it lasts no longer than an approving ask may.
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
put := puts.Put{FP: fNew, Why: "the bus's user list", Hold: true}
d := ComposedDigest{Digest: dgst, Judgement: JudgedCannot, Words: "the have names no composition"}
for _, bus := range []bool{false, true} {
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
if err != nil {
@@ -234,7 +240,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
if _, err := EncodeAsk(rkey, a); err != nil {
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
}
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: mustAsk(t, a).ID, Option: OptionYes}, rkey, a)
viewed := viewstate.ViewAnswerFits(viewstate.ViewAnswer{Asking: mustAsk(t, a).ID, Option: OptionYes}, rkey, a)
yes := mustAsk(t, a).Options[0]
// mesh-sdk gives no refusal by level any more: the teller refuses an option a view cannot answer (its level is not
// acknowledge), as it did through the sdk before.
@@ -269,7 +275,7 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
// under the assigner's own name keyed by node and module, and is cleared under the same key.
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: mwc.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
if _, err := occ.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
@@ -283,7 +289,7 @@ func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
}
// mustAsk is the router's ask an asks-state value carries.
func mustAsk(t *testing.T, a delivery.AssignerAsk) oc.Ask {
func mustAsk(t *testing.T, a asks.Ask) oc.Ask {
t.Helper()
ask, err := AskOf(a)
if err != nil {
@@ -292,7 +298,7 @@ func mustAsk(t *testing.T, a delivery.AssignerAsk) oc.Ask {
return ask
}
func expiresAt(t *testing.T, a delivery.AssignerAsk) time.Time {
func expiresAt(t *testing.T, a asks.Ask) time.Time {
t.Helper()
return ExpiresAt(mustAsk(t, a))
}
modules/mesh-delivery-assigner/internal/assign/composed.go
+45
View File
@@ -0,0 +1,45 @@
package assign
import nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
// Judgement is the assigner's judgement of a move from the composition a node runs, as its have names it, to a new one
// (ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests). The three the node-engine names are its have's words (a have's
// judgement); neither and unchanged are the assigner's own.
type Judgement = string
// The judgements of a move.
const (
// JudgedRecreates is a move that recreates a container.
JudgedRecreates = nec.JudgedRecreates
// JudgedMovesData is a move that gives the module's data another place.
JudgedMovesData = nec.JudgedMovesData
// JudgedNeither is a move that does neither, and the judgement of a composition the node never ran.
JudgedNeither = "neither"
// JudgedCannot is a move that cannot be judged, because the composition the node runs is gone or cannot be read.
JudgedCannot = nec.JudgedCannot
// JudgedUnchanged is a composition equal to the module's slice of the declaration the node last applied, for a node
// whose have names no composition yet (the first want at the switch, novox/hq ADR 0540 clause 9): applying it
// changes nothing on the node, so it needs no person's yes.
JudgedUnchanged = "unchanged"
)
// ComposedDigest is what the assigner knows of a composition beside the node's have: its Digest, and the Judgement of the
// move to it from the composition the node runs now, whose digest is Against (empty when the node runs no composition of
// the module). Words say why, and are required for a move that cannot be judged.
//
// Policy is the module's upgrade policy as the assigner holds it (ADR 0236, ADR 0539 clause 14), with PolicyFrom where it
// comes from and PolicyWhy why: the assigner holds a move of a module of policy record from it.
type ComposedDigest struct {
V int `json:"v"`
Digest string `json:"digest"`
Judgement Judgement `json:"judgement"`
Against string `json:"against,omitempty"`
Words string `json:"words,omitempty"`
// UserListOnly says the composition differs from the one the node's have names only in the bus's user list, whatever
// the judgement says of its mount: the assigner writes such a revision into the bus node's want at once, outside the
// record hold (ADR 0539 clause 7).
UserListOnly bool `json:"user-list-only,omitempty"`
Policy string `json:"policy,omitempty"`
PolicyFrom string `json:"policy-from,omitempty"`
PolicyWhy string `json:"policy-why,omitempty"`
}
modules/mesh-delivery-assigner/internal/assign/policy.go
+2 -1
View File
@@ -3,6 +3,7 @@ package assign
import (
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/buckets"
"sort"
"strings"
"time"
@@ -19,7 +20,7 @@ import (
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
// PoliciesBucket is the assigner's state `policies`.
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
var PoliciesBucket = delivery.AssignerModule + "_" + buckets.PoliciesState
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
modules/mesh-delivery-assigner/internal/assign/put.go
+55 -49
View File
@@ -2,11 +2,17 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/puts"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
@@ -20,7 +26,7 @@ const ComposeWithin = 10 * time.Minute
// PutWorld is what one step of a put reads.
type PutWorld struct {
Node, Module string
Put delivery.Put
Put puts.Put
// Assigned says the module stands assigned to the node.
Assigned bool
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
@@ -31,7 +37,7 @@ type PutWorld struct {
Newest string
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
Composed *delivery.ComposedDigest
Composed *ComposedDigest
Granted bool
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
// that ask expired unanswered.
@@ -43,13 +49,13 @@ type PutWorld struct {
Snapshotting bool
SnapshotFailed string
// Have is what the node says it runs of the module; nil when it says nothing.
Have *delivery.Have
Have *nodeengine.HaveByNodeModuleValue
Now time.Time
}
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
type PutNext struct {
Put delivery.Put
Put puts.Put
// Ask writes the put's ask about the move (PutAsk).
Ask bool
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
@@ -61,7 +67,7 @@ type PutNext struct {
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
func StepPut(w PutWorld) PutNext {
p := w.Put
if delivery.PutEnded(p.Step) {
if puts.Ended(p.Step) {
return PutNext{Put: p}
}
say := func(step, format string, args ...any) PutNext {
@@ -69,21 +75,21 @@ func StepPut(w PutWorld) PutNext {
}
fp := short(p.FP)
if !w.Assigned {
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
return say(puts.Refused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
}
switch w.Build {
case delivery.BuildDone:
case delivery.BuildFailed:
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case builds.BuildDone:
case builds.BuildFailed:
return say(puts.Refused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
case "":
if p.FP != w.Newest {
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
return say(puts.Refused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
}
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
return say(puts.Build, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
default:
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
return say(puts.Build, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
}
if w.Composed == nil || !w.Granted {
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
@@ -91,33 +97,33 @@ func StepPut(w PutWorld) PutNext {
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
}
since := w.Now
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
if p.Step == puts.Compose && len(p.Steps) > 0 {
since = p.Steps[len(p.Steps)-1].At
}
if w.Now.Sub(since) > ComposeWithin {
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
return say(puts.Refused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
}
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
return say(puts.Compose, "waits up to %s: %s yet", ComposeWithin, missing)
}
d := *w.Composed
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
if p.Step == puts.Wanted || p.Step == puts.Holding {
return PutNext{Put: applied(w, p)}
}
p.Composed = d.Digest
if needsPerson(w, d) {
switch w.Answer {
case OptionNo:
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
return say(puts.Refused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
case OptionYes:
default:
if w.AskExpired {
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
return say(puts.Refused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
}
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
next := say(puts.Person, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
if p.Ask == "" {
next.Ask = true
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
next.Put.Ask, _ = asks.RecordAskKey(w.Node, w.Module, d.Digest)
}
return next
}
@@ -126,11 +132,11 @@ func StepPut(w PutWorld) PutNext {
case w.Snapshot != "":
p.Snapshot = w.Snapshot
case w.SnapshotFailed != "":
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
return say(puts.Refused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
case w.Snapshotting:
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
return say(puts.Snapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
default:
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next := say(puts.Snapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
next.Snapshot = true
return next
}
@@ -143,22 +149,22 @@ func StepPut(w PutWorld) PutNext {
if p.Snapshot != "" {
words += "; the streams were snapshotted: " + p.Snapshot
}
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
next := PutNext{Put: applied(w, p.Said(w.Now, puts.Wanted, words))}
next.Warranted = w.Bus && p.Ask != ""
return next
}
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
func applied(w PutWorld, p delivery.Put) delivery.Put {
func applied(w PutWorld, p puts.Put) puts.Put {
h := w.Have
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
if h == nil || h.Fp != p.FP || h.Composed != p.Composed {
return p
}
switch {
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK && p.Hold:
return p.Said(w.Now, puts.Holding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
case h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK:
return p.Said(w.Now, puts.Done, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
case failed(*h):
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
}
@@ -168,8 +174,8 @@ func applied(w PutWorld, p delivery.Put) delivery.Put {
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
// container, move data, or that cannot be judged (§14).
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
runs := w.Have != nil && w.Have.FP == w.Put.FP
func needsPerson(w PutWorld, d ComposedDigest) bool {
runs := w.Have != nil && w.Have.Fp == w.Put.FP
if runs && w.Have.Composed == d.Digest {
return false // the node runs this composition already: nothing moves
}
@@ -179,22 +185,22 @@ func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
if !w.Recorded {
return false
}
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
return d.Judgement == JudgedRecreates || d.Judgement == JudgedMovesData || d.Judgement == JudgedCannot
}
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
func PutWant(p delivery.Put) delivery.Want {
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
func PutWant(p puts.Put) mwc.Want {
return mwc.Want{Fp: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
}
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
func PutOwns(p puts.Put) (want *mwc.Want, owns bool) {
switch p.Step {
case delivery.PutWanted, delivery.PutHolding:
case puts.Wanted, puts.Holding:
w := PutWant(p)
return &w, true
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
case puts.Done, puts.Released, puts.Refused:
return nil, false
}
return nil, true
@@ -203,7 +209,7 @@ func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
func PutAsking(p delivery.Put) int {
func PutAsking(p puts.Put) int {
if n := p.At.UnixMilli(); n > 0 {
return int(n)
}
@@ -218,21 +224,21 @@ func PutAct(node, module, fp, composed string) occ.Act {
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
// channel that proves who answered takes it (ADR 0259), never the view.
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
func PutAsk(key, node, module string, p puts.Put, d ComposedDigest, bus bool, now time.Time) (asks.Ask, error) {
yes := oc.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: occ.Acknowledge}
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + d.Words,
delivery.JudgedNeither: "recreates no container and moves no data",
delivery.JudgedUnchanged: "changes nothing the node runs",
what := map[Judgement]string{
JudgedRecreates: "recreates a container",
JudgedMovesData: "moves the module's data",
JudgedCannot: "cannot be judged: " + d.Words,
JudgedNeither: "recreates no container and moves no data",
JudgedUnchanged: "changes nothing the node runs",
}[d.Judgement]
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
expires := occ.FormatTime(now.Add(7 * 24 * time.Hour))
if bus {
binds, err := occ.ActDigest(PutAct(node, module, p.FP, d.Digest))
if err != nil {
return delivery.AssignerAsk{}, err
return asks.Ask{}, err
}
yes.Level, yes.Binds = occ.Approve, binds
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
@@ -240,7 +246,7 @@ func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest,
expires = occ.FormatTime(now.Add(occ.ApproveLasts))
}
return assignerAsk(PutAsking(p), oc.Ask{
ID: delivery.AskingID(key, PutAsking(p)),
ID: meshdeliveryclient.AskingID(key, PutAsking(p)),
Headline: fmt.Sprintf("Put %s on %s?", module, node),
Explanation: explanation,
Options: []oc.Option{yes,
modules/mesh-delivery-assigner/internal/assign/seat.go
+10 -8
View File
@@ -2,6 +2,8 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"maps"
"slices"
"strings"
@@ -42,7 +44,7 @@ type ProposeIn struct {
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Placement, []string) {
p := mma.Proposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]int64{}, State: plc.ProposalAnswering}
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]int64{}, State: mmac.ProposalAnswering}
var findings []string
if p.By == "" {
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
@@ -54,7 +56,7 @@ func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Placement, []string) {
findings = append(findings, "a proposal changes something: it names no change")
}
for i, c := range in.Changes {
if c.Op == plc.OpSeed {
if c.Op == mmac.OpSeed {
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
}
}
@@ -99,7 +101,7 @@ func Propose(in ProposeIn) (mma.Proposal, map[string]*mma.Placement, []string) {
// this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2).
joining := map[string]bool{}
for _, c := range in.Changes {
if c.Op == plc.OpAddNode {
if c.Op == mmac.OpAddNode {
joining[c.Node] = true
}
}
@@ -127,7 +129,7 @@ func ControllerRefusals(changes []mma.ProposalChange, laid map[string]mma.Placem
var out []string
nodes := map[string]bool{}
for i, c := range changes {
places := c.Module == ControllerModule && c.Op != plc.OpTakeOff
places := c.Module == ControllerModule && c.Op != mmac.OpTakeOff
if c.Whole != nil && slices.Contains(mmac.Assigned(*c.Whole), ControllerModule) {
places = true
}
@@ -194,10 +196,10 @@ func PlacementAsking(p mma.Proposal, at time.Time) int {
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
func PlacementAsk(key string, p mma.Proposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
func PlacementAsk(key string, p mma.Proposal, brief string, asking int, now time.Time) (asks.Ask, error) {
binds, err := occ.ActDigest(PlacementAct(p))
if err != nil {
return delivery.AssignerAsk{}, err
return asks.Ask{}, err
}
nodes := plc.Nodes(p)
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
@@ -205,7 +207,7 @@ func PlacementAsk(key string, p mma.Proposal, brief string, asking int, now time
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
}
return assignerAsk(asking, oc.Ask{
ID: delivery.AskingID(key, asking),
ID: meshdeliveryclient.AskingID(key, asking),
Headline: headline,
Explanation: brief,
Options: []oc.Option{
@@ -235,6 +237,6 @@ func Rest(p mma.Proposal, written map[string]*mma.Placement) (mma.Proposal, bool
rest.Base[node] = base
}
}
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", plc.ProposalAnswering
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", mmac.ProposalAnswering
return rest, len(rest.Changes) > 0
}
modules/mesh-delivery-assigner/internal/assign/seat_test.go
+20 -20
View File
@@ -2,6 +2,7 @@ package assign
import (
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/keys"
"slices"
"strings"
"testing"
@@ -10,7 +11,6 @@ import (
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
mmac "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/delivery"
plc "mesh-delivery-assigner/internal/placement"
)
@@ -19,18 +19,18 @@ import (
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
func TestThePlacementSeatDecides(t *testing.T) {
anchor := mma.Placement{V: delivery.Version, Node: "anchor", Decided: mma.NodeDecided{Hub: true, Address: "10.0.0.1"},
anchor := mma.Placement{V: keys.Version, Node: "anchor", Decided: mma.NodeDecided{Hub: true, Address: "10.0.0.1"},
Modules: []mma.PlacedModule{{Module: "store", Ports: map[string]int64{"5432": 5432}}, {Module: "web"}}}
laptop := mma.Placement{V: delivery.Version, Node: "laptop", Decided: mma.NodeDecided{Address: "10.0.0.2"}, Modules: []mma.PlacedModule{{Module: "shell"}}}
laptop := mma.Placement{V: keys.Version, Node: "laptop", Decided: mma.NodeDecided{Address: "10.0.0.2"}, Modules: []mma.PlacedModule{{Module: "shell"}}}
in := func(changes ...mma.ProposalChange) ProposeIn {
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
Current: map[string]mma.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
Held: func(m string) bool { return m != "ghost" },
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
}
move := []mma.ProposalChange{{Op: plc.OpTakeOff, Node: "anchor", Module: "web"}, {Op: plc.OpPlace, Node: "laptop", Module: "web"}}
move := []mma.ProposalChange{{Op: mmac.OpTakeOff, Node: "anchor", Module: "web"}, {Op: mmac.OpPlace, Node: "laptop", Module: "web"}}
p, laid, findings := Propose(in(move...))
if len(findings) != 0 || p.State != plc.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
if len(findings) != 0 || p.State != mmac.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
!slices.Equal(mmac.Assigned(*laid["laptop"]), []string{"shell", "web"}) || !slices.Equal(mmac.Assigned(*laid["anchor"]), []string{"store"}) {
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
}
@@ -41,18 +41,18 @@ func TestThePlacementSeatDecides(t *testing.T) {
in ProposeIn
want string
}{
"a module not held": {in(mma.ProposalChange{Op: plc.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(mma.ProposalChange{Op: plc.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(mma.ProposalChange{Op: plc.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
"a module not held": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
"a seed by hand": {in(mma.ProposalChange{Op: plc.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
"a seed by hand": {in(mma.ProposalChange{Op: mmac.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
// act without its lease.
"the controller placed again": {in(mma.ProposalChange{Op: plc.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
"the controller seeded whole": {in(mma.ProposalChange{Op: plc.OpSeed, Node: "anchor", Whole: &mma.Placement{V: delivery.Version, Node: "anchor",
"the controller placed again": {in(mma.ProposalChange{Op: mmac.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
"the controller seeded whole": {in(mma.ProposalChange{Op: mmac.OpSeed, Node: "anchor", Whole: &mma.Placement{V: keys.Version, Node: "anchor",
Decided: anchor.Decided, Modules: []mma.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
"a pin of nobody placed": {in(mma.ProposalChange{Op: plc.OpPin, Node: "laptop", Provision: "db", Provider: &mma.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
"a pin of nobody placed": {in(mma.ProposalChange{Op: mmac.OpPin, Node: "laptop", Provision: "db", Provider: &mma.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
} {
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
@@ -61,12 +61,12 @@ func TestThePlacementSeatDecides(t *testing.T) {
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
for _, c := range []mma.ProposalChange{
{Op: plc.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
{Op: plc.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int64{"80": 8080}, Bindings: map[string]mma.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: plc.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: plc.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
{Op: plc.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: plc.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
{Op: mmac.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
{Op: mmac.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int64{"80": 8080}, Bindings: map[string]mma.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: mmac.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: mmac.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
{Op: mmac.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: mmac.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
} {
if _, _, f := Propose(in(c)); len(f) != 0 {
t.Errorf("%s is refused: %v", c.Op, f)
@@ -75,7 +75,7 @@ func TestThePlacementSeatDecides(t *testing.T) {
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
if rest, ok := Rest(p, map[string]*mma.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
len(rest.Base) != 1 || rest.State != plc.ProposalAnswering || rest.Must != nil {
len(rest.Base) != 1 || rest.State != mmac.ProposalAnswering || rest.Must != nil {
t.Errorf("the rest = %+v, %v", rest, ok)
}
if _, ok := Rest(p, laid); ok {
@@ -100,7 +100,7 @@ func TestThePlacementSeatDecides(t *testing.T) {
t.Fatalf("the ask = %+v, %v", a, err)
}
q := p
q.Changes = append(slices.Clone(p.Changes), mma.ProposalChange{Op: plc.OpUnpin, Node: "laptop", Provision: "x"})
q.Changes = append(slices.Clone(p.Changes), mma.ProposalChange{Op: mmac.OpUnpin, Node: "laptop", Provision: "x"})
if occ.Performs(mustAsk(t, a).Options[0], PlacementAct(q)) == nil {
t.Error("the ask's yes binds changes it was not asked about")
}
modules/mesh-delivery-assigner/internal/assign/wants.go
+96 -91
View File
@@ -2,14 +2,19 @@ package assign
import (
"fmt"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mwc "git.novox.be/novox/mesh-contracts/placement/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"maps"
"mesh-delivery-assigner/internal/puts"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-delivery/sdk/forms"
@@ -59,7 +64,7 @@ type Module struct {
Nodes []string
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
Composed func(node, fp string) (ComposedDigest, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
@@ -77,7 +82,7 @@ type Module struct {
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]delivery.Put
Puts map[string]puts.Put
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
Needs []string
@@ -93,9 +98,9 @@ type Module struct {
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
func (m Module) composition(node, fp string) (ComposedDigest, bool) {
if m.Composed == nil {
return delivery.ComposedDigest{}, false
return ComposedDigest{}, false
}
return m.Composed(node, fp)
}
@@ -103,17 +108,17 @@ func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
type Nodes struct {
Online map[string]bool
Haves map[string]map[string]delivery.Have
Haves map[string]map[string]nodeengine.HaveByNodeModuleValue
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
Providers map[string]map[string][]string
// Reads is, by node, the reader capabilities its node-engine states in its node record (delivery.NodeRecord.ManifestReads,
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
Reads map[string][]string
}
// healthy says whether a have runs fp healthy.
func healthy(h delivery.Have, fp string) bool {
return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
}
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
@@ -142,40 +147,40 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) {
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
case m.Held:
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
}
if ordering {
canary := Canary(m.Nodes, n.Online)
switch {
case canary == "":
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
}
if !m.Granted {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
}
if m.GrantedOn != nil && !m.GrantedOn(node) {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
return false, mwc.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
short(m.FP))
}
if words, lacking := m.readerLacking(node, n); lacking {
return false, delivery.RuleComposed, words
return false, mwc.RuleComposed, words
}
if _, ok := m.composition(node, m.FP); !ok {
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
return false, mwc.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
}
if words, waiting := m.provisionWaiting(node); waiting {
return false, delivery.RuleProvisions, words
return false, mwc.RuleProvisions, words
}
if words, waiting := m.verbWaiting(node); waiting {
return false, delivery.RuleProvisions, words
return false, mwc.RuleProvisions, words
}
if words, waiting := m.busStateWaiting(node); waiting {
return false, delivery.RuleProvisions, words
return false, mwc.RuleProvisions, words
}
return true, 0, ""
}
@@ -289,7 +294,7 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
if !slices.Contains(nodes, node) {
at = Canary(nodes, n.Online)
}
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK {
return true, ""
}
}
@@ -320,9 +325,9 @@ func keepsData(m catalogue.Manifest) bool {
// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
type Outcome struct {
Wants map[string]delivery.Want
Wants map[string]mwc.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *delivery.Held
Held *mwc.Held
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
@@ -351,7 +356,7 @@ type Outcome struct {
type RecordNeed struct {
Node string
Digest string
Judgement delivery.Judgement
Judgement Judgement
Words string
}
@@ -372,8 +377,8 @@ func (h Holding) Reason() string {
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]mwc.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
held := m.Held
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
out.Lifted, held = true, false
@@ -381,7 +386,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
h, ok := n.Haves[node][m.Name]
if !ok || h.FP != m.FP || !failed(h) {
if !ok || h.Fp != m.FP || !failed(h) {
continue
}
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
@@ -396,7 +401,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
continue
}
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
out.Held = &mwc.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
@@ -408,24 +413,24 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
break
}
before, had := wants[node]
var next delivery.Want
var next mwc.Want
p, put := m.Puts[node]
if !put && had && before.Put && before.Hold {
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
// without hold is not kept this way: its put takes it further once heard.
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: delivery.PutHolding}, true
p, put = puts.Put{FP: before.Fp, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
Step: puts.Holding}, true
}
if put {
if want, owns := PutOwns(p); owns {
if want != nil {
w := *want
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.Fp == w.Fp {
w.Composed = ul.Composed
}
if !had || !delivery.SameWant(normal(before), normal(w)) {
if !had || !mwc.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
}
@@ -436,29 +441,29 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
if w, ok := m.userListRevision(node, before, had, n); ok {
if !had || !delivery.SameWant(normal(before), normal(w)) {
if !had || !mwc.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
continue
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == mwc.RuleCanary {
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
older := ran && runs(have) && have.Fp != m.FP && strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
if older {
_, composed := m.composition(node, have.FP)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
_, composed := m.composition(node, have.Fp)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.Fp)
}
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
oldForm := ran && runs(have) && have.Fp != m.FP && !strings.HasPrefix(have.Fp, meshcatalogclient.SourcePrefix2)
switch {
case older:
ok, target = true, have.FP
ok, target = true, have.Fp
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
@@ -471,40 +476,40 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
n.Haves[node][m.Name].FP != m.FP {
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.Fp == m.FP) &&
n.Haves[node][m.Name].Fp != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
// it would be on neither list and its node-engine would give the module up (issue 1945).
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
ok, rule, words = false, mwc.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
}
switch {
case ok:
d, _ := m.composition(node, target)
next = delivery.Want{FP: target, Composed: d.Digest}
next = mwc.Want{Fp: target, Composed: d.Digest}
at := m
if target != m.FP {
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
next.Newest, next.Rule, next.Words = m.FP, rule, words
next.Newest, next.Rule, next.Words = m.FP, int64(rule), words
at.FP = target
}
// A placed node's want names the build alone: it is the same target while the build is.
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
if had && !before.Removed && !before.Put && before.Fp == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
// A removed want is no want: a first want after it goes through the record ask as a first want does.
asked, hadLive := before, had
if before.Removed {
asked, hadLive = delivery.Want{}, false
asked, hadLive = mwc.Want{}, false
}
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.Fp, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
continue
}
case held && had && before.FP == m.FP:
case held && had && before.Fp == m.FP:
out.Rules[node] = Holding{Rule: rule, Words: words}
last := lastHealthy[node]
switch {
@@ -522,7 +527,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
break
}
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
next = mwc.Want{Fp: last, Composed: d.Digest, Newest: m.FP, Rule: int64(rule), Words: words}
}
default:
out.Rules[node] = Holding{Rule: rule, Words: words}
@@ -533,31 +538,31 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
h, there := n.Haves[node][m.Name]
switch {
case there && h.Phase != delivery.PhaseRemoved && !held:
case there && h.Phase != nodeengineclient.PhaseRemoved && !held:
out.Neither[node] = Holding{Rule: rule, Words: words}
case had && (!there || h.Phase == delivery.PhaseRemoved):
case had && (!there || h.Phase == nodeengineclient.PhaseRemoved):
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
// would have its node agent remove the module and stop holding its composition's resources back.
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
if !delivery.SameWant(normal(before), normal(next)) {
next = mwc.Want{Removed: true, Words: NoWantWords(rule, words)}
if !mwc.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
continue
}
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
if before.FP == m.FP {
next = mwc.Want{Fp: before.Fp, Composed: before.Composed, Newest: m.FP, Rule: int64(rule), Words: words}
if before.Fp == m.FP {
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
next = delivery.Want{FP: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
next = mwc.Want{Fp: m.FP, Composed: before.Composed}
} else if d, there := m.composition(node, before.Fp); there && d.Digest != "" && d.Digest != before.Composed {
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
at := m
at.FP = before.FP
at.FP = before.Fp
running, ran := n.Haves[node][m.Name]
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
need, wait := at.mustAsk(node, before, true, ran, running.Fp, running.Composed == "", d)
if need != nil {
out.Record = append(out.Record, *need)
}
@@ -570,13 +575,13 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if !write {
continue
}
if !had || !delivery.SameWant(normal(before), normal(next)) {
if !had || !mwc.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
}
for node, before := range wants {
if !slices.Contains(m.Nodes, node) && !before.Removed {
out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
out.Wants[node] = mwc.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
}
}
return out
@@ -589,7 +594,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
// while its ask waits, the want written stays and the node's holding says why.
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
func (m Module) selfDerived(node string, before mwc.Want, had bool, next mwc.Want, n Nodes, out *Outcome) (mwc.Want, bool) {
switch {
case next.Removed:
return next, true
@@ -600,16 +605,16 @@ func (m Module) selfDerived(node string, before delivery.Want, had bool, next de
return next, true
}
next.Self, next.Yes = false, ""
d, there := m.composition(node, next.FP)
d, there := m.composition(node, next.Fp)
if !there || d.Digest == "" {
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
out.Rules[node] = Holding{Rule: mwc.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.Fp))}
return next, false
}
at := m
at.FP = next.FP
at.FP = next.Fp
running, ran := n.Haves[node][m.Name]
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
if need, wait := at.mustAsk(node, before, had, ran, running.Fp, running.Composed == "", d); wait {
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
out.Record = append(out.Record, *need)
}
@@ -629,8 +634,8 @@ func NoWantWords(rule int, words string) string {
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
// the world's lastHealthy) already asks for phase running.
func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
func failed(h nodeengine.HaveByNodeModuleValue) bool {
return h.Health == nodeengineclient.HealthFailed && (h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseFailed)
}
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
@@ -650,7 +655,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string {
// the hold and the set-back stand at the second failure (ADR 0557 §2).
func failedElsewhere(m Module, n Nodes, node string) bool {
for _, other := range m.Nodes {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.Fp == m.FP && failed(h) {
return true
}
}
@@ -671,8 +676,8 @@ type NodeFailure struct {
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
func runs(h nodeengine.HaveByNodeModuleValue) bool {
return h.Phase == nodeengineclient.PhaseRunning
}
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
@@ -684,7 +689,7 @@ func allRunHealthy(m Module, n Nodes) bool {
ran := 0
for _, node := range m.Nodes {
h, ok := n.Haves[node][m.Name]
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
continue
}
if !healthy(h, m.FP) {
@@ -697,7 +702,7 @@ func allRunHealthy(m Module, n Nodes) bool {
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
// it, and the node's own words. The node is the held record's own field.
func HeldWords(h delivery.Have, at time.Time) string {
func HeldWords(h nodeengine.HaveByNodeModuleValue, at time.Time) string {
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
if !at.IsZero() {
words += ", seen at " + at.UTC().Format(time.RFC3339)
@@ -714,13 +719,13 @@ func LiftedWords(module, fp string) string {
}
// normal is a want without its version, as a decoded and a composed one are compared.
func normal(w delivery.Want) delivery.Want {
func normal(w mwc.Want) mwc.Want {
w.V = 0
return w
}
func short(fp string) string {
part, err := delivery.FingerprintKeyPart(fp)
part, err := meshcatalogclient.KeyPart(fp)
if err != nil || len(part) < 12 {
return fp
}
@@ -731,18 +736,18 @@ func short(fp string) string {
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
// streams before it applies (§2); the assigner asks no snapshot for it.
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
func (m Module) mustAsk(node string, before mwc.Want, had, ran bool, running string, haveNamesNoComposition bool, d ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.Fp == m.FP && before.Composed == d.Digest) {
return nil, false
}
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == JudgedUnchanged {
return nil, false
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
if !m.recorded(d) || d.Judgement == JudgedNeither || d.Judgement == JudgedUnchanged {
return nil, false
}
if m.Allowed != nil && m.Allowed(node, d.Digest) {
@@ -754,13 +759,13 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
func (m Module) recorded(d ComposedDigest) bool {
if m.Chosen != nil {
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
return policy == catalogue.PolicyRecord
}
if d.Policy != "" {
return d.Policy == delivery.PolicyRecord
return d.Policy == catalogue.PolicyRecord
}
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
return policy == catalogue.PolicyRecord
@@ -769,19 +774,19 @@ func (m Module) recorded(d delivery.ComposedDigest) bool {
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
func (m Module) userListRevision(node string, before mwc.Want, had bool, n Nodes) (mwc.Want, bool) {
if !catalogue.ProvidesBus(m.Manifest) {
return delivery.Want{}, false
return mwc.Want{}, false
}
running, ran := n.Haves[node][m.Name]
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
return delivery.Want{}, false
if !ran || running.Fp == "" || (had && (before.Removed || before.Fp != running.Fp)) {
return mwc.Want{}, false
}
d, ok := m.composition(node, running.FP)
d, ok := m.composition(node, running.Fp)
if !ok || !d.UserListOnly {
return delivery.Want{}, false
return mwc.Want{}, false
}
w := delivery.Want{FP: running.FP, Composed: d.Digest}
w := mwc.Want{Fp: running.Fp, Composed: d.Digest}
if had && before.Composed == d.Digest {
w.Backup, w.Words = before.Backup, before.Words
}
@@ -790,6 +795,6 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
// put reads by too).
func GrantNamesNode(g delivery.Grant, node, module string) bool {
func GrantNamesNode(g grantsstate.Grant, node, module string) bool {
return g.NamesNode(node, module)
}
modules/mesh-delivery-assigner/internal/assign/wants_test.go
+315 -308
View File
File diff suppressed because it is too large Load Diff
modules/mesh-delivery-assigner/internal/buckets/buckets.go
+53
View File
@@ -0,0 +1,53 @@
// Package buckets holds the names of the bus buckets the assigner reads and the states it keeps beside the seats' own. A
// contract carries the local name of a state, not the module that holds it (ADR 0625 §4, §7), and the assigner opens
// buckets through grants its manifest declares, so the holder's name is kept here, in the assigner's own code, and held equal
// to the manifest by its test (ADR 0634 §1, finding F4).
package buckets
import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
wants "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
delivery "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
plc "mesh-delivery-assigner/internal/placement"
)
// The states the assigner keeps: the wants seat's, and its own asks, puts and policies.
const (
Wants = plc.Module + "_" + wants.StateWants
Differences = plc.Module + "_" + wants.StateDifferences
Asks = plc.Module + "_asks"
Puts = plc.Module + "_puts"
// PoliciesState is the state of the policies a person set for a module's upgrade; Policies its bucket.
PoliciesState = "policies"
Policies = plc.Module + "_" + PoliciesState
)
// AsksHistory is the revisions the bucket of asks keeps per key.
const AsksHistory = 5
// The states of the other seats' holders that the assigner reads, named by the holder's module as every state is.
const (
// Have holds a module's have on a node, `<node>.<module>`, and the node's own facts, `<node>.node`.
Have = "node-engine_" + nodeengine.StateHave
// ComposedRead is the family of the node-engine's composed states, one bucket per node.
ComposedRead = "mesh-host." + nodeengine.StateComposed
// Modules and Manifests are the indexer's.
Modules = "mesh-delivery-indexer_" + catalog.StateModules
Manifests = "mesh-delivery-indexer_" + catalog.StateManifests
// Presence holds each part's presence and the node's.
Presence = "mesh-delivery-watchman_" + watchdogs.StatePresence
// Source and Merges are the gate's.
Source = "mesh-delivery-gate_" + delivery.StateSource
Merges = "mesh-delivery-gate_" + delivery.StateMerges
// Answers holds the teller's answer to each ask.
Answers = "mesh-delivery-teller_answers"
// Builds is the builder's.
Builds = builds.BuilderBuildsBucket
// Grants is the grants part's.
Grants = grantsstate.PartGrantsBucket
)
modules/mesh-delivery-assigner/internal/buckets/buckets_test.go
+22
View File
@@ -0,0 +1,22 @@
package buckets
import "testing"
// The names are the ones the earlier shared form held (mesh-sdk's go/delivery), and the ones the manifest's reads and
// states say: a bucket is `<module>_<state>`.
func TestBucketsAreTheEarlierNames(t *testing.T) {
for name, c := range map[string][2]string{
"Have": {Have, "node-engine_have"}, "ComposedRead": {ComposedRead, "mesh-host.composed"},
"Modules": {Modules, "mesh-delivery-indexer_modules"}, "Manifests": {Manifests, "mesh-delivery-indexer_manifests"},
"Presence": {Presence, "mesh-delivery-watchman_presence"}, "Source": {Source, "mesh-delivery-gate_source"},
"Merges": {Merges, "mesh-delivery-gate_merges"}, "Answers": {Answers, "mesh-delivery-teller_answers"},
"Builds": {Builds, "mesh-delivery-builder_builds"}, "Grants": {Grants, "mesh-grants_grants"},
"Wants": {Wants, "mesh-delivery-assigner_wants"}, "Differences": {Differences, "mesh-delivery-assigner_differences"},
"Asks": {Asks, "mesh-delivery-assigner_asks"}, "Puts": {Puts, "mesh-delivery-assigner_puts"},
"Policies": {Policies, "mesh-delivery-assigner_policies"},
} {
if c[0] != c[1] {
t.Errorf("%s = %q, want %q", name, c[0], c[1])
}
}
}
modules/mesh-delivery-assigner/internal/judge/judge.go
+7 -6
View File
@@ -8,6 +8,7 @@ package judge
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"reflect"
"sort"
"strconv"
@@ -83,7 +84,7 @@ func Intended(current map[string]mma.Placement, laid map[string]*mma.Placement,
// on the bus) is named as giving no answer and the proposal decided on the answers it has (ADR 0593 §8).
// A difference fails nothing: one Intended computed is listed under Intended, one it did not under Unintended, both
// shown in the ask's brief. The seed alone fails on any difference.
func JudgeAnswers(p mma.Proposal, intended map[string]map[string]bool, answers map[string]*delivery.CandidateAnswer,
func JudgeAnswers(p mma.Proposal, intended map[string]map[string]bool, answers map[string]*nodeengine.CandidateAnswer,
revisions map[string]uint64, online map[string]bool, pulsing map[string]bool, now time.Time) mma.Verdict {
v := mma.Verdict{At: now.UTC(), Intended: map[string][]string{}, Unintended: map[string][]string{}, Carried: map[string][]string{},
Answers: map[string]map[string]any{}}
@@ -99,7 +100,7 @@ func JudgeAnswers(p mma.Proposal, intended map[string]map[string]bool, answers m
}
for _, node := range must {
a := answers[node]
if a == nil || a.Proposal != p.ID || a.Revision == 0 || a.Revision != revisions[node] {
if a == nil || a.Proposal != p.ID || a.Revision == 0 || uint64(a.Revision) != revisions[node] {
v.Silent = append(v.Silent, node)
continue
}
@@ -236,7 +237,7 @@ func Brief(p mma.Proposal, v mma.Verdict) string {
// differenceWords names one module's difference: the module, the first field that differs in it with its value as it
// runs and as the seed composes it, each cut to about 60 characters ("(value cut)" where a body shown is cut before the
// field), and how many fields more; or the resource and why when no body is shown.
func differenceWords(d delivery.ModuleDiffers) string {
func differenceWords(d nodeengine.CandidateDiffers) string {
for _, b := range d.Bodies {
if len(b.Paths) == 0 {
continue
@@ -336,7 +337,7 @@ func fieldName(running, composed, path string) string {
// answerBody is a node's answer as the verdict carries it: the seat's contract keeps a verdict's answers as bodies, since
// the answer's layout is the node-engine's (ADR 0593 §7).
func answerBody(a delivery.CandidateAnswer) map[string]any {
func answerBody(a nodeengine.CandidateAnswer) map[string]any {
raw, _ := json.Marshal(a)
var out map[string]any
_ = json.Unmarshal(raw, &out)
@@ -344,9 +345,9 @@ func answerBody(a delivery.CandidateAnswer) map[string]any {
}
// answerOf reads a verdict's answer body back; a body that does not fit is an answer with nothing in it.
func answerOf(body map[string]any) delivery.CandidateAnswer {
func answerOf(body map[string]any) nodeengine.CandidateAnswer {
raw, _ := json.Marshal(body)
var out delivery.CandidateAnswer
var out nodeengine.CandidateAnswer
_ = json.Unmarshal(raw, &out)
return out
}
modules/mesh-delivery-assigner/internal/judge/judge_test.go
+22 -21
View File
@@ -1,6 +1,7 @@
package judge
import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"slices"
"strings"
"testing"
@@ -32,38 +33,38 @@ func TestJudge(t *testing.T) {
Must: []string{"anchor", "laptop"}}
revisions := map[string]uint64{"anchor": 7, "laptop": 8}
intended := map[string]map[string]bool{"anchor": {"web": true}}
same := func(rev uint64) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
same := func(rev uint64) *nodeengine.CandidateAnswer {
return &nodeengine.CandidateAnswer{Proposal: "p-one", Revision: int64(rev), Verdict: nodeengine.CandidateAnswerVerdictSame, Same: []string{"mail"}}
}
differs := func(rev uint64, module string) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Proposal: "p-one", Revision: rev, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: module, Why: "the port differs"}}}
differs := func(rev uint64, module string) *nodeengine.CandidateAnswer {
return &nodeengine.CandidateAnswer{Proposal: "p-one", Revision: int64(rev), Verdict: nodeengine.CandidateAnswerVerdictDiffers,
Differs: []nodeengine.CandidateDiffers{{Module: module, Why: "the port differs"}}}
}
for _, row := range []struct {
name string
seed bool
answers map[string]*delivery.CandidateAnswer
answers map[string]*nodeengine.CandidateAnswer
pulsing map[string]bool
state string
words string
marked string
}{
{"all the same", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a module added where intended", false, map[string]*delivery.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: delivery.CandidateDiffers,
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a difference not computed passes, marked", false, map[string]*delivery.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
{"the seed carries a difference, named", true, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: delivery.CandidateDiffers, Differs: []delivery.ModuleDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
Bodies: []delivery.ResourceBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
{"all the same", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a module added where intended", false, map[string]*nodeengine.CandidateAnswer{"anchor": {Proposal: "p-one", Revision: 7, Verdict: nodeengine.CandidateAnswerVerdictDiffers,
Added: []nodeengine.CandidateAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a difference not computed passes, marked", false, map[string]*nodeengine.CandidateAnswer{"anchor": differs(7, "mail"), "laptop": same(8)}, nil, delivery.VerdictPass, "", "mail"},
{"the seed carries a difference, named", true, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: nodeengine.CandidateAnswerVerdictDiffers, Differs: []nodeengine.CandidateDiffers{{Module: "node-tools", Resource: "node-tools.runtime", Why: "the resource differs",
Bodies: []nodeengine.CandidateBodies{{ID: "node-tools.runtime", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf", "env.MESH_TOOL_MODULES"},
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/hosts\":\"hostname\",\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}","MESH_TOOL_MODULES":"… (cut at 8192 bytes)`}}}}}}, nil, delivery.VerdictPass, "", ""},
{"a refusal", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: delivery.CandidateRefused,
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
{"a node not answering, within its bound", false, map[string]*delivery.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, plc.VerdictWaits, "laptop (offline)", ""},
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*delivery.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
{"a module held as today passes, shown in the brief", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: delivery.CandidateSame, Held: []delivery.ModuleRefused{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
{"its later answer", false, map[string]*delivery.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
{"a refusal", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8, Verdict: nodeengine.CandidateAnswerVerdictRefused,
Refusals: []nodeengine.CandidateRefusal{{Module: "web", Why: "no own secret"}}}}, nil, delivery.VerdictFail, "laptop refuses", ""},
{"a node not answering, within its bound", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7)}, map[string]bool{"laptop": true}, plc.VerdictWaits, "laptop (offline)", ""},
{"an answer of an earlier revision, past its bound, is no answer and decided without it", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(6), "laptop": same(8)}, nil, delivery.VerdictPass, "no answer from anchor", ""},
{"a module held as today passes, shown in the brief", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": {Proposal: "p-one", Revision: 8,
Verdict: nodeengine.CandidateAnswerVerdictSame, Held: []nodeengine.CandidateRefusal{{Module: "spiegel", Why: "a required value is unset"}}}}, nil, delivery.VerdictPass, "", ""},
{"its later answer", false, map[string]*nodeengine.CandidateAnswer{"anchor": same(7), "laptop": same(8)}, nil, delivery.VerdictPass, "", ""},
} {
q := p
q.Seed = row.seed
@@ -77,7 +78,7 @@ func TestJudge(t *testing.T) {
}
if row.seed && row.state == delivery.VerdictPass {
whole := row.answers["laptop"].Differs[0]
whole.Bodies = []delivery.ResourceBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
whole.Bodies = []nodeengine.CandidateBodies{{ID: "r", Paths: []string{"env.MESH_OWNED_PATHS./etc/resolv.conf"},
Running: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"networkmanager (node-uplink)\"}"}}`,
Composed: `{"env":{"MESH_OWNED_PATHS":"{\"/etc/resolv.conf\":\"systemd-resolved (node-resolver)\"}"}}`}}
if got := differenceWords(whole); got != "node-tools: MESH_OWNED_PATHS[/etc/resolv.conf] networkmanager (node-uplink) → systemd-resolved (node-resolver)" {
modules/mesh-delivery-assigner/internal/placement/placement_test.go
+44 -38
View File
@@ -33,7 +33,13 @@ func TestPlacement(t *testing.T) {
Holds: []seat.Holding{{Claim: "mesh-store", Scope: "mesh", Module: "postgres"}}}
current := map[string]seat.Placement{"g14": g14, "novox": novox}
const id = "p-20261028t183000-1a2b3c"
const enroller = "mesh-enrolment"
// the enroller is whichever module holds the seat mesh-node-enroller: no module is named for it
const enroller = "enroll-lab"
enrollers := EnrollersOf(map[string]seat.Placement{"ace": {Node: "ace",
Holds: []seat.Holding{{Claim: "mesh-node-enroller", Scope: "mesh", Module: enroller}}}})
if !slices.Equal(enrollers, []string{enroller}) {
t.Fatalf("the holders of the enroller seat = %v", enrollers)
}
t.Run("what a change says", func(t *testing.T) {
after := g14
@@ -97,10 +103,10 @@ func TestPlacement(t *testing.T) {
t.Run("changes laid over the keys", func(t *testing.T) {
laid, findings := LayChanges(current, []seat.ProposalChange{
{Op: OpPlace, Node: "g14", Module: "new"},
{Op: OpTakeOff, Node: "g14", Module: "zsh"},
{Op: OpUnpin, Node: "g14", Provision: "shell"},
{Op: OpDecide, Node: "novox", Field: "site", Value: "attic"},
{Op: client.OpPlace, Node: "g14", Module: "new"},
{Op: client.OpTakeOff, Node: "g14", Module: "zsh"},
{Op: client.OpUnpin, Node: "g14", Provision: "shell"},
{Op: client.OpDecide, Node: "novox", Field: "site", Value: "attic"},
})
if len(findings) != 0 || len(laid) != 2 || !reflect.DeepEqual(client.Assigned(laid["g14"]), []string{"app", "new"}) ||
len(laid["g14"].Holds) != 0 || laid["g14"].Pins != nil || laid["novox"].Decided.Site != "attic" || !laid["novox"].Decided.Hub {
@@ -109,21 +115,21 @@ func TestPlacement(t *testing.T) {
if !reflect.DeepEqual(client.Assigned(current["g14"]), []string{"zsh", "app"}) {
t.Error("laying changed the current placement")
}
if fresh, f := LayChanges(current, []seat.ProposalChange{{Op: OpPlace, Node: "pi", Module: "zsh"}}); len(f) != 0 || fresh["pi"].Node != "pi" {
if fresh, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "pi", Module: "zsh"}}); len(f) != 0 || fresh["pi"].Node != "pi" {
t.Errorf("a node with no key: %+v, %v", fresh, f)
}
for name, c := range map[string]struct {
change seat.ProposalChange
want string
}{
"not placed": {seat.ProposalChange{Op: OpTakeOff, Node: "g14", Module: "nope"}, "not placed"},
"no such field": {seat.ProposalChange{Op: OpDecide, Node: "g14", Field: "colour", Value: "x"}, "no decided field"},
"wrong type": {seat.ProposalChange{Op: OpDecide, Node: "g14", Field: "hub", Value: "x"}, "does not take"},
"not placed": {seat.ProposalChange{Op: client.OpTakeOff, Node: "g14", Module: "nope"}, "not placed"},
"no such field": {seat.ProposalChange{Op: client.OpDecide, Node: "g14", Field: "colour", Value: "x"}, "no decided field"},
"wrong type": {seat.ProposalChange{Op: client.OpDecide, Node: "g14", Field: "hub", Value: "x"}, "does not take"},
"unknown op": {seat.ProposalChange{Op: "move", Node: "g14"}, "no change of placement"},
"pin of nobody": {seat.ProposalChange{Op: OpPin, Node: "g14", Provision: "x"}, "provider"},
"not held": {seat.ProposalChange{Op: OpUnhold, Node: "g14", Module: "app", Claim: "x"}, "holds no"},
"bad port": {seat.ProposalChange{Op: OpPlace, Node: "g14", Module: "x", Ports: map[string]int64{"a": 1}}, "port"},
"seed of another": {seat.ProposalChange{Op: OpSeed, Node: "g14", Whole: &novox}, "seed"},
"pin of nobody": {seat.ProposalChange{Op: client.OpPin, Node: "g14", Provision: "x"}, "provider"},
"not held": {seat.ProposalChange{Op: client.OpUnhold, Node: "g14", Module: "app", Claim: "x"}, "holds no"},
"bad port": {seat.ProposalChange{Op: client.OpPlace, Node: "g14", Module: "x", Ports: map[string]int64{"a": 1}}, "port"},
"seed of another": {seat.ProposalChange{Op: client.OpSeed, Node: "g14", Whole: &novox}, "seed"},
} {
if _, f := LayChanges(current, []seat.ProposalChange{c.change}); len(f) != 1 || !strings.Contains(f[0], c.want) {
t.Errorf("%s: %v, want a finding naming %q", name, f, c.want)
@@ -136,8 +142,8 @@ func TestPlacement(t *testing.T) {
t.Errorf("NewProposalID = %q", got)
}
p := seat.Proposal{ID: id, By: "operator@anchor", Why: "move the app", At: time.Unix(1, 0).UTC(),
Changes: []seat.ProposalChange{{Op: OpTakeOff, Node: "g14", Module: "zsh"}}, Base: map[string]int64{"g14": 4},
State: ProposalAnswering}
Changes: []seat.ProposalChange{{Op: client.OpTakeOff, Node: "g14", Module: "zsh"}}, Base: map[string]int64{"g14": 4},
State: client.ProposalAnswering}
raw, err := EncodeProposal(p)
if err != nil {
t.Fatal(err)
@@ -150,7 +156,7 @@ func TestPlacement(t *testing.T) {
}
seed := p
seed.Seed, seed.ExportDigest = true, ExportDigest(current)
seed.Changes = []seat.ProposalChange{{Op: OpSeed, Node: "g14", Whole: &g14}}
seed.Changes = []seat.ProposalChange{{Op: client.OpSeed, Node: "g14", Whole: &g14}}
if _, err := EncodeProposal(seed); err != nil {
t.Errorf("the seed: %v", err)
}
@@ -161,17 +167,17 @@ func TestPlacement(t *testing.T) {
}
// The digests are what the earlier holder's code made of the same values (issue 1966: a proposal's ask binds its
// changes' digest, and a seed its export's), whatever order the generated types keep their fields in.
rich := seat.Proposal{ID: id, By: "operator@anchor", Why: "w", At: time.Unix(1, 0).UTC(), State: ProposalAnswering,
rich := seat.Proposal{ID: id, By: "operator@anchor", Why: "w", At: time.Unix(1, 0).UTC(), State: client.ProposalAnswering,
Base: map[string]int64{"g14": 4, "lab": 0, "novox": 2},
Changes: []seat.ProposalChange{
{Op: OpTakeOff, Node: "g14", Module: "zsh"},
{Op: OpPlace, Node: "g14", Module: "new", Ports: map[string]int64{"80": 8080}, Bindings: map[string]seat.Provider{"pg": {Node: "novox", Module: "postgres"}}},
{Op: OpPin, Node: "g14", Provision: "shell", Provider: &seat.Provider{Node: "g14", Module: "bash"}},
{Op: OpHold, Node: "g14", Module: "app", Claim: "node-shell", Scope: "node", Kind: "k", Site: "s"},
{Op: OpDecide, Node: "novox", Field: "site", Value: "attic"},
{Op: OpDecide, Node: "novox", Field: "peer-names", Value: map[string]any{"k": "v"}},
{Op: OpConfig, Node: "g14", Module: "app", MeshWide: map[string]any{"a": float64(1)}, Layer: map[string]any{"b": "c"}, MeshWideValues: map[string]any{"d": true}, Values: map[string]any{"e": []any{"f"}}},
{Op: OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7", Account: "x", AgentAccountHome: "/h"}, Keys: &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id", Sealing: "se"}},
{Op: client.OpTakeOff, Node: "g14", Module: "zsh"},
{Op: client.OpPlace, Node: "g14", Module: "new", Ports: map[string]int64{"80": 8080}, Bindings: map[string]seat.Provider{"pg": {Node: "novox", Module: "postgres"}}},
{Op: client.OpPin, Node: "g14", Provision: "shell", Provider: &seat.Provider{Node: "g14", Module: "bash"}},
{Op: client.OpHold, Node: "g14", Module: "app", Claim: "node-shell", Scope: "node", Kind: "k", Site: "s"},
{Op: client.OpDecide, Node: "novox", Field: "site", Value: "attic"},
{Op: client.OpDecide, Node: "novox", Field: "peer-names", Value: map[string]any{"k": "v"}},
{Op: client.OpConfig, Node: "g14", Module: "app", MeshWide: map[string]any{"a": float64(1)}, Layer: map[string]any{"b": "c"}, MeshWideValues: map[string]any{"d": true}, Values: map[string]any{"e": []any{"f"}}},
{Op: client.OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7", Account: "x", AgentAccountHome: "/h"}, Keys: &seat.ProposalChangeKeys{Overlay: "wg", Identity: "id", Sealing: "se"}},
}}
for name, c := range map[string]struct{ got, want string }{
"export": {ExportDigest(current), "sha256:742f03d8790d7efa6abff76636f4a69ba2704d646233960f52ce08604b96e128"},
@@ -190,23 +196,23 @@ func TestPlacement(t *testing.T) {
// 774/S42/AC1 (ADR 0587 §7): an add-node carries its enrolment's warrant naming its node and address, as its one
// change, from the enroller alone; a warrant naming another node or address is refused.
// At the operator's yes it carries the tunnel key alone (the operator, 2026-11-02 01:05).
joining := seat.ProposalChange{Op: OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7"},
joining := seat.ProposalChange{Op: client.OpAddNode, Node: "lab", Decided: &seat.NodeDecided{Address: "10.0.0.7"},
Keys: &seat.ProposalChangeKeys{Overlay: "wg"}}
add := seat.Proposal{ID: id, By: "ace.mesh-enrolment", Why: "lab joins", At: time.Unix(1, 0).UTC(),
Changes: []seat.ProposalChange{joining}, Base: map[string]int64{"lab": 0}, State: ProposalAnswering,
Changes: []seat.ProposalChange{joining}, Base: map[string]int64{"lab": 0}, State: client.ProposalAnswering,
Warrant: &seat.Warrant{Node: "lab", Address: "10.0.0.7", Ask: "a-1", Warrant: "w-1"}}
if raw, err := EncodeProposal(add); err != nil {
t.Errorf("an add-node: %v", err)
} else if back, err := DecodeProposal(raw); err != nil || back.Warrant == nil || back.Changes[0].Keys.Overlay != "wg" {
t.Errorf("an add-node round trip: %+v, %v", back, err)
}
if AddNodeRefusal(add, enroller, enroller) != "" || !strings.Contains(AddNodeRefusal(add, "mesh-delivery-assigner", enroller), "mesh-enrolment's alone") {
t.Errorf("add-node by caller: %q, %q", AddNodeRefusal(add, enroller, enroller), AddNodeRefusal(add, "mesh-delivery-assigner", enroller))
if AddNodeRefusal(add, enroller, enrollers) != "" || !strings.Contains(AddNodeRefusal(add, "mesh-delivery-assigner", enrollers), "enroll-lab's alone") {
t.Errorf("add-node by caller: %q, %q", AddNodeRefusal(add, enroller, enrollers), AddNodeRefusal(add, "mesh-delivery-assigner", enrollers))
}
if laid, f := LayChanges(current, add.Changes); len(f) != 0 || laid["lab"].Decided.Address != "10.0.0.7" || len(laid["lab"].Modules) != 0 {
t.Errorf("an add-node laid: %+v, %v", laid, f)
}
if _, f := LayChanges(current, []seat.ProposalChange{{Op: OpAddNode, Node: "g14", Decided: &seat.NodeDecided{Address: "10.0.0.9"},
if _, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpAddNode, Node: "g14", Decided: &seat.NodeDecided{Address: "10.0.0.9"},
Keys: &seat.ProposalChangeKeys{Overlay: "wg"}}}); len(f) != 1 || !strings.Contains(f[0], "placed already") {
t.Errorf("an add-node of a placed node: %v", f)
}
@@ -229,7 +235,7 @@ func TestPlacement(t *testing.T) {
p.Changes = []seat.ProposalChange{c}
},
"beside another change": func(p *seat.Proposal) {
p.Changes = append(p.Changes, seat.ProposalChange{Op: OpPlace, Node: "lab", Module: "zsh"})
p.Changes = append(p.Changes, seat.ProposalChange{Op: client.OpPlace, Node: "lab", Module: "zsh"})
},
"the hub": func(p *seat.Proposal) {
c := joining
@@ -237,7 +243,7 @@ func TestPlacement(t *testing.T) {
p.Changes = []seat.ProposalChange{c}
},
"a warrant on a move": func(p *seat.Proposal) {
p.Changes, p.Base = []seat.ProposalChange{{Op: OpTakeOff, Node: "g14", Module: "zsh"}}, map[string]int64{"g14": 4}
p.Changes, p.Base = []seat.ProposalChange{{Op: client.OpTakeOff, Node: "g14", Module: "zsh"}}, map[string]int64{"g14": 4}
},
} {
q := add
@@ -256,16 +262,16 @@ func TestPlacement(t *testing.T) {
}
// An enrolment's end takes its never-confirmed node off again: the enroller's alone, of a node running nothing.
remove := add
remove.Changes = []seat.ProposalChange{{Op: OpRemoveNode, Node: "lab"}}
if _, err := EncodeProposal(remove); err != nil || AddNodeRefusal(remove, "mesh-delivery-assigner", enroller) == "" ||
AddNodeRefusal(remove, enroller, enroller) != "" || !slices.Equal(RemovedNodes(remove), []string{"lab"}) {
t.Errorf("a remove-node: %v, %q", err, AddNodeRefusal(remove, "mesh-delivery-assigner", enroller))
remove.Changes = []seat.ProposalChange{{Op: client.OpRemoveNode, Node: "lab"}}
if _, err := EncodeProposal(remove); err != nil || AddNodeRefusal(remove, "mesh-delivery-assigner", enrollers) == "" ||
AddNodeRefusal(remove, enroller, enrollers) != "" || !slices.Equal(RemovedNodes(remove), []string{"lab"}) {
t.Errorf("a remove-node: %v, %q", err, AddNodeRefusal(remove, "mesh-delivery-assigner", enrollers))
}
joined := map[string]seat.Placement{"lab": {V: keys.Version, Node: "lab", Decided: seat.NodeDecided{Address: "10.0.0.7"}, Modules: []seat.PlacedModule{}}}
if laid, f := LayChanges(joined, remove.Changes); len(f) != 0 || len(laid) != 0 {
t.Errorf("a joining node removed: %v, %v", laid, f)
}
if _, f := LayChanges(current, []seat.ProposalChange{{Op: OpRemoveNode, Node: "g14"}}); len(f) != 1 ||
if _, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpRemoveNode, Node: "g14"}}); len(f) != 1 ||
!strings.Contains(f[0], "runs modules") {
t.Errorf("a running node removed: %v", f)
}
@@ -280,7 +286,7 @@ func TestPlacement(t *testing.T) {
"no change": func(p *seat.Proposal) { p.Changes = nil },
"no base": func(p *seat.Proposal) { p.Base = map[string]int64{"novox": 1} },
"no state": func(p *seat.Proposal) { p.State = "" },
"asked, no ask": func(p *seat.Proposal) { p.State = ProposalAsked },
"asked, no ask": func(p *seat.Proposal) { p.State = client.ProposalAsked },
"seed mixed": func(p *seat.Proposal) { p.Seed, p.ExportDigest = true, "d" },
"bad change": func(p *seat.Proposal) { p.Changes[0].Op = "swap" },
"seed, no digest": func(p *seat.Proposal) { *p = seed; p.ExportDigest = "" },
modules/mesh-delivery-assigner/internal/placement/proposal.go
+57 -61
View File
@@ -5,9 +5,11 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"strings"
"time"
enroller "git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshnodeenroller"
client "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient"
seat "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -17,37 +19,6 @@ import (
// lays them over the current keys, names the revision of each key it laid them over, and either refuses at once with every
// finding or writes the proposal under its id in its state `proposals` and a candidate for each node it must hear from.
// The operations a proposal's change names (ADR 0567 §2).
const (
OpPlace = "place" // place a module on a node, with its host ports and bindings
OpTakeOff = "take-off" // take a module off a node
OpPin = "pin" // pin the provider of a provision on a node
OpUnpin = "unpin" // remove a pin
OpHold = "hold" // a module placed on the node holds a seat
OpUnhold = "unhold" // it holds it no more
OpDecide = "decide" // set a decided field of the node (NodeDecided by its JSON name), its address among them
OpConfig = "config" // set a module's config placement or values on the node
OpSeed = "seed" // the node's whole placement: the take-over (adopt, §7), never given by a caller
// OpAddNode adds a joining node with its decided fields and the public halves of its keys, on the warrant of the
// enrolment that issued its token (novox/hq ADR 0587 §7): the enroller's alone, the proposal's one change.
OpAddNode = "add-node"
// OpRemoveNode takes a joining node off placement again when its enrolment ends unconfirmed, revoked or expired
// (the operator's decision of 2026-11-02 01:05): the enroller's alone, on the same warrant, the proposal's one
// change, of a node with no module and no holding. The holder deletes its key (RemovedNodes).
OpRemoveNode = "remove-node"
)
// The states of an open proposal.
const (
// ProposalAnswering: its candidates are written and the nodes it must hear from have not all answered.
ProposalAnswering = "answering"
// ProposalFailed: its verdict failed; it stays open, for its proposer to read and withdraw, or for a node's later
// answer to judge it again.
ProposalFailed = "failed"
// ProposalAsked: its verdict passed and the operator is asked to approve it.
ProposalAsked = "asked"
)
// VerdictWaits is a proposal's verdict while a node it must hear from has not answered within the bound; it passes or
// fails as a check's verdict does (ADR 0567 §4).
const VerdictWaits = "waits"
@@ -65,7 +36,7 @@ func Nodes(p seat.Proposal) []string { return sortedKeys(p.Base) }
func RemovedNodes(p seat.Proposal) []string {
var out []string
for _, c := range p.Changes {
if c.Op == OpRemoveNode {
if c.Op == client.OpRemoveNode {
out = append(out, c.Node)
}
}
@@ -84,22 +55,22 @@ func CheckChange(c seat.ProposalChange) error {
return nil
}
switch c.Op {
case OpPlace, OpTakeOff, OpConfig:
case client.OpPlace, client.OpTakeOff, client.OpConfig:
return needs("module", keys.CheckName("module", c.Module) == nil)
case OpPin:
case client.OpPin:
return needs("provision or provider", c.Provision != "" && c.Provider != nil && c.Provider.Node != "" && c.Provider.Module != "")
case OpUnpin:
case client.OpUnpin:
return needs("provision", c.Provision != "")
case OpHold, OpUnhold:
case client.OpHold, client.OpUnhold:
return needs("module or claim", keys.CheckName("module", c.Module) == nil && c.Claim != "")
case OpDecide:
case client.OpDecide:
return needs("field or value", c.Field != "" && c.Value != nil)
case OpSeed:
case client.OpSeed:
if c.Whole == nil || c.Whole.Node != c.Node {
return fmt.Errorf("placement: the seed of %s names no placement of that node", c.Node)
}
return client.CheckPlacement(*c.Whole)
case OpAddNode:
case client.OpAddNode:
if c.Decided == nil || c.Decided.Address == "" || c.Keys == nil || c.Keys.Overlay == "" {
return fmt.Errorf("placement: the add-node of %s names no address or no tunnel key", c.Node)
}
@@ -114,7 +85,7 @@ func CheckChange(c seat.ProposalChange) error {
return fmt.Errorf("placement: the add-node of %s places a module; a module is placed by a proposal of its own", c.Node)
}
return nil
case OpRemoveNode:
case client.OpRemoveNode:
if c.Module != "" || c.Decided != nil || c.Keys != nil || c.Whole != nil {
return fmt.Errorf("placement: the remove-node of %s names more than its node", c.Node)
}
@@ -157,16 +128,16 @@ func LayChanges(current map[string]seat.Placement, changes []seat.ProposalChange
findings = append(findings, fmt.Sprintf("change %d: %s is not placed on %s", i+1, c.Module, c.Node))
}
switch c.Op {
case OpSeed:
case client.OpSeed:
p = clonePlacement(*c.Whole)
case OpAddNode:
case client.OpAddNode:
// A joining node has no key yet (ADR 0587 §7): one that has is no joining node.
if _, has := current[c.Node]; has {
findings = append(findings, fmt.Sprintf("change %d: %s is placed already, so it does not join", i+1, c.Node))
continue
}
p = seat.Placement{V: keys.Version, Node: c.Node, Decided: *c.Decided, Modules: []seat.PlacedModule{}}
case OpRemoveNode:
case client.OpRemoveNode:
// Only a joining node never confirmed goes this way: one with no module and no holding (a node that runs
// anything is taken off module by module, by a proposal the operator approves).
cur, has := current[c.Node]
@@ -176,7 +147,7 @@ func LayChanges(current map[string]seat.Placement, changes []seat.ProposalChange
findings = append(findings, fmt.Sprintf("change %d: %s runs modules or holds seats, so it is no joining node to remove", i+1, c.Node))
}
continue
case OpPlace:
case client.OpPlace:
m := seat.PlacedModule{Module: c.Module, Ports: c.Ports, Bindings: c.Bindings}
if at >= 0 {
o := p.Modules[at]
@@ -186,7 +157,7 @@ func LayChanges(current map[string]seat.Placement, changes []seat.ProposalChange
} else {
p.Modules = append(p.Modules, m)
}
case OpTakeOff:
case client.OpTakeOff:
if at < 0 {
notPlaced()
continue
@@ -199,7 +170,7 @@ func LayChanges(current map[string]seat.Placement, changes []seat.ProposalChange
}
}
p.Holds = holds
case OpConfig:
case client.OpConfig:
if at < 0 {
notPlaced()
continue
@@ -217,23 +188,23 @@ func LayChanges(current map[string]seat.Placement, changes []seat.ProposalChange
if c.Values != nil {
m.Values = c.Values
}
case OpPin:
case client.OpPin:
if p.Pins == nil {
p.Pins = map[string]seat.Provider{}
}
p.Pins[c.Provision] = *c.Provider
case OpUnpin:
case client.OpUnpin:
delete(p.Pins, c.Provision)
if len(p.Pins) == 0 {
p.Pins = nil
}
case OpHold:
case client.OpHold:
if at < 0 {
notPlaced()
continue
}
p.Holds = append(p.Holds, seat.Holding{Claim: c.Claim, Scope: c.Scope, Module: c.Module, Site: c.Site, Kind: c.Kind})
case OpUnhold:
case client.OpUnhold:
kept := p.Holds[:0:0]
for _, h := range p.Holds {
if h.Module != c.Module || h.Claim != c.Claim {
@@ -245,7 +216,7 @@ func LayChanges(current map[string]seat.Placement, changes []seat.ProposalChange
continue
}
p.Holds = kept
case OpDecide:
case client.OpDecide:
d, err := decide(p.Decided, c.Field, c.Value)
if err != nil {
findings = append(findings, fmt.Sprintf("change %d: %v", i+1, err))
@@ -339,7 +310,7 @@ func CheckProposal(p seat.Proposal) error {
if err := CheckChange(c); err != nil {
return fmt.Errorf("placement: the proposal %s, change %d: %w", p.ID, i+1, err)
}
if (c.Op == OpSeed) != p.Seed {
if (c.Op == client.OpSeed) != p.Seed {
return fmt.Errorf("placement: the proposal %s mixes the seed with other changes", p.ID)
}
if _, ok := p.Base[c.Node]; !ok {
@@ -347,8 +318,8 @@ func CheckProposal(p seat.Proposal) error {
}
}
switch p.State {
case ProposalAnswering, ProposalFailed:
case ProposalAsked:
case client.ProposalAnswering, client.ProposalFailed:
case client.ProposalAsked:
if p.Ask == "" || p.Verdict == nil || p.Verdict.State != VerdictPass {
return fmt.Errorf("placement: the proposal %s is asked with no ask or no passing verdict", p.ID)
}
@@ -367,7 +338,7 @@ func CheckProposal(p seat.Proposal) error {
func checkAddNode(p seat.Proposal) error {
adds := 0
for _, c := range p.Changes {
if c.Op == OpAddNode || c.Op == OpRemoveNode {
if c.Op == client.OpAddNode || c.Op == client.OpRemoveNode {
adds++
}
}
@@ -384,7 +355,7 @@ func checkAddNode(p seat.Proposal) error {
if w == nil || w.Ask == "" || w.Warrant == "" {
return fmt.Errorf("placement: the add-node of %s carries no warrant of its enrolment", c.Node)
}
if c.Op == OpRemoveNode {
if c.Op == client.OpRemoveNode {
if w.Node != c.Node {
return fmt.Errorf("placement: the remove-node of %s carries the warrant of %s", c.Node, w.Node)
}
@@ -397,18 +368,43 @@ func checkAddNode(p seat.Proposal) error {
}
// AddNodeRefusal is why the holder of the seat refuses an add-node or a remove-node from caller, the module the runtime
// proved called (ADR 0587 §7), or "": only the enroller (the module holding the seat mesh-node-enroller, which the caller
// of this names) adds a node; what the proposal itself must hold, CheckProposal refuses.
func AddNodeRefusal(p seat.Proposal, caller, enroller string) string {
// proved called (ADR 0587 §7), or "": only a holder of the seat mesh-node-enroller (enrollers, EnrollersOf) adds a node;
// what the proposal itself must hold, CheckProposal refuses.
func AddNodeRefusal(p seat.Proposal, caller string, enrollers []string) string {
if err := checkAddNode(p); err != nil {
return err.Error()
}
if p.Warrant != nil && caller != enroller {
return fmt.Sprintf("an add-node or a remove-node is %s's alone, and %s proposed it", enroller, caller)
if p.Warrant != nil && !slices.Contains(enrollers, caller) {
if len(enrollers) == 0 {
return fmt.Sprintf("an add-node or a remove-node is the holder's of the seat %s alone, nobody holds it, and %s proposed it", enroller.Seat, caller)
}
return fmt.Sprintf("an add-node or a remove-node is %s's alone, and %s proposed it", strings.Join(enrollers, ", "), caller)
}
return ""
}
// EnrollersOf is the modules that hold the seat mesh-node-enroller on the placements given, sorted: the state seat-holders
// composed from them (SeatHoldersOf), read by the client's HoldersOf.
func EnrollersOf(placements map[string]seat.Placement) []string {
holders, _ := client.HoldersOf(rawHolders(SeatHoldersOf(placements)), enroller.Seat, "")
var out []string
for _, h := range holders {
if !slices.Contains(out, h.Module) {
out = append(out, h.Module)
}
}
slices.Sort(out)
return out
}
func rawHolders(held map[string]seat.SeatHolders) map[string][]byte {
out := map[string][]byte{}
for k, v := range held {
out[k], _ = json.Marshal(v)
}
return out
}
// EncodeProposal is p as JSON at the keys' Version; it refuses what DecodeProposal refuses.
func EncodeProposal(p seat.Proposal) ([]byte, error) {
if err := CheckProposal(p); err != nil {
modules/mesh-delivery-assigner/internal/puts/puts.go
+156
View File
@@ -0,0 +1,156 @@
// Package puts is the assigner's put: one version of a module put on one node by its tool `assigner_put` (novox/hq
// ADR 0546), kept under `<node>.<module>` in its state puts. It is the assigner's own value, which no other module reads
// (the earlier form lived in mesh-sdk's go/delivery; ADR 0634 §5), so the check and the codec live here.
package puts
import (
"encoding/json"
"fmt"
"regexp"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// Each put is the key `<node>.<module>` of the assigner's state puts, which holds the version asked, the hold, why, and each
// step the assigner took or waits on, so a reader watches the key to follow it. The want the put writes carries Put (and
// Hold), so the assigner's rules pass over that key while the put holds it.
// History is the revisions the puts state keeps per key: each step of a put is one.
const History = 20
// A put's steps, in the order they are taken (ADR 0546 clauses 3 to 9). A put stands at the last step written.
const (
// Asked is a put written by the tool, not taken up by a pass yet.
Asked = "asked"
// Build waits for the builder's build at the fingerprint.
Build = "build"
// Compose waits for the composition (and grant) at the fingerprint.
Compose = "compose"
// Person waits for the operator's answer to the ask about the move.
Person = "person"
// Snapshot waits for the bus's streams to be snapshotted after the operator's yes.
Snapshot = "snapshot"
// Wanted is the want written: the node's have has not reached it yet.
Wanted = "wanted"
// Holding is a held put whose have runs it healthy: the want stays until released.
Holding = "holding"
// Done is a put without hold whose have ran it healthy: the rules write the want again.
Done = "done"
// Released is a put released by the tool `assigner_release`.
Released = "released"
// Refused is a put that cannot go on: a build that failed, an answer no, a snapshot not taken. Its words say why.
Refused = "refused"
)
// Ended says a put at this step is over: nothing more is done for it and the rules own the want.
func Ended(step string) bool {
return step == Done || step == Released || step == Refused
}
// Step is one step a put took: when, which, and what was done or waits, in a sentence.
type Step struct {
At time.Time `json:"at"`
Step string `json:"step"`
Words string `json:"words"`
}
// StepsKept is how many steps a put keeps, newest last.
const StepsKept = 16
// Put is the value of a key of the state puts.
type Put struct {
V int `json:"v"`
// FP is the fingerprint put, of the new form.
FP string `json:"fp"`
// Hold keeps the want at FP until released.
Hold bool `json:"hold,omitempty"`
Why string `json:"why"`
// By is who asked, as the bus named the caller; "unproven" when it named none.
By string `json:"by,omitempty"`
At time.Time `json:"at"`
// Step is where the put stands now, one of the steps above; Words say it in a sentence.
Step string `json:"step"`
Words string `json:"words,omitempty"`
// Composed is the digest of the composition the want names, once known.
Composed string `json:"composed,omitempty"`
// Ask is the key, in the asks state, of the ask about the move, once asked.
Ask string `json:"ask,omitempty"`
// Snapshot says where the bus's streams were snapshotted, once they were.
Snapshot string `json:"snapshot,omitempty"`
// Steps are the steps taken, oldest first, at most StepsKept.
Steps []Step `json:"steps,omitempty"`
}
// Said is p at step with words, the step added to its steps when it differs from the last one.
func (p Put) Said(at time.Time, step, words string) Put {
if p.Step == step && p.Words == words {
return p
}
p.Step, p.Words = step, words
p.Steps = append(append([]Step(nil), p.Steps...), Step{At: at.UTC(), Step: step, Words: words})
if len(p.Steps) > StepsKept {
p.Steps = p.Steps[len(p.Steps)-StepsKept:]
}
return p
}
var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
func (p Put) check() error {
if _, err := meshcatalogclient.KeyPart(p.FP); err != nil {
return fmt.Errorf("puts: a put's fingerprint: %w", err)
}
if p.Why == "" {
return fmt.Errorf("puts: a put says why")
}
switch p.Step {
case Asked, Build, Compose, Person, Snapshot, Wanted, Holding, Done, Released, Refused:
default:
return fmt.Errorf("puts: %q is not a step of a put", p.Step)
}
if p.Composed != "" && !digestForm.MatchString(p.Composed) {
return fmt.Errorf("puts: a put's composition digest %q is not sha256:<64 lowercase hex digits>", p.Composed)
}
return nil
}
// Key is the key, in the state puts, of the put of module on node: `<node>.<module>`.
func Key(node, module string) (string, error) {
return nec.HaveKey(node, module)
}
// ParseKey reads a Key.
func ParseKey(key string) (node, module string, err error) {
node, module, isNode, err := nec.ParseHaveKey(key)
if err == nil && isNode {
err = fmt.Errorf("puts: %q is a node's own key, and no put", key)
}
return node, module, err
}
// Encode is p as JSON; it refuses a fingerprint not of the new form, a put that says no why and an unknown step.
func Encode(p Put) ([]byte, error) {
if err := p.check(); err != nil {
return nil, err
}
p.V = keys.Version
return json.Marshal(p)
}
// Decode reads a put; it refuses an unknown version (keys.ErrUnknownVersion) and what Encode refuses.
func Decode(data []byte) (Put, error) {
if err := keys.CheckVersion(data); err != nil {
return Put{}, err
}
var p Put
if err := json.Unmarshal(data, &p); err != nil {
return Put{}, fmt.Errorf("puts: value does not fit its shape: %w", err)
}
if err := p.check(); err != nil {
return Put{}, err
}
return p, nil
}
modules/mesh-delivery-assigner/internal/puts/puts_test.go
+66
View File
@@ -0,0 +1,66 @@
package puts
import (
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// TestPut holds a put's bytes equal to what go/delivery's codec wrote for the same value (golden), its steps kept and its
// key, and the refusals it made.
func TestPut(t *testing.T) {
a := "src2:" + strings.Repeat("a", 64)
at := time.Date(2026, 11, 5, 10, 0, 0, 0, time.UTC)
p := Put{FP: a, Hold: true, Why: "why", By: "me", At: at, Composed: "sha256:" + strings.Repeat("c", 64), Ask: "k", Snapshot: "s"}
p = p.Said(at, Wanted, "w1")
p = p.Said(at, Wanted, "w1")
p = p.Said(at.Add(time.Minute), Holding, "w2")
golden := `{"v":1,"fp":"` + a + `","hold":true,"why":"why","by":"me","at":"2026-11-05T10:00:00Z","step":"holding","words":"w2","composed":"sha256:` + strings.Repeat("c", 64) + `","ask":"k","snapshot":"s","steps":[{"at":"2026-11-05T10:00:00Z","step":"wanted","words":"w1"},{"at":"2026-11-05T10:01:00Z","step":"holding","words":"w2"}]}`
raw, err := Encode(p)
if err != nil || string(raw) != golden {
t.Fatalf("wrote %s, %v", raw, err)
}
back, err := Decode(raw)
if err != nil || back.Step != Holding || len(back.Steps) != 2 {
t.Fatalf("read back %+v, %v", back, err)
}
for i := 0; i < 2*StepsKept; i++ {
p = p.Said(at, Build, strings.Repeat("x", i+1))
}
if len(p.Steps) != StepsKept {
t.Fatalf("%d steps kept", len(p.Steps))
}
if k, err := Key("ace", "novox.be"); err != nil || k != "ace.novox=2ebe" {
t.Fatalf("Key = %q %v", k, err)
}
if n, m, err := ParseKey("ace.novox=2ebe"); err != nil || n != "ace" || m != "novox.be" {
t.Fatalf("ParseKey = %q %q %v", n, m, err)
}
if _, _, err := ParseKey("ace.node"); err == nil {
t.Error("a node's own key is a put's")
}
for name, bad := range map[string]Put{
"old fingerprint": {FP: "src1:x", Why: "w", Step: Asked},
"no why": {FP: a, Step: Asked},
"unknown step": {FP: a, Why: "w", Step: "later"},
"bad digest": {FP: a, Why: "w", Step: Asked, Composed: "x"},
} {
if _, err := Encode(bad); err == nil {
t.Errorf("%s: written", name)
}
}
if _, err := Decode([]byte(`{"fp":"` + a + `","why":"w","step":"asked"}`)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("no version: %v", err)
}
for _, s := range []string{Done, Released, Refused} {
if !Ended(s) {
t.Errorf("%s is not an end", s)
}
}
if Ended(Holding) {
t.Error("holding is an end")
}
}
modules/mesh-delivery-assigner/manifest_test.go
+11
View File
@@ -6,12 +6,14 @@ import (
"os"
"reflect"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/buckets"
plc "mesh-delivery-assigner/internal/placement"
)
@@ -48,6 +50,15 @@ func TestManifest(t *testing.T) {
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
// the buckets the program names are the manifest's reads, `<module>.<state>` as `<module>_<state>`
named := []string{buckets.Builds, buckets.Merges, buckets.Source, buckets.Answers, buckets.Presence, buckets.Grants,
buckets.ComposedRead, "mesh-provisioner_provisions", buckets.Have}
slices.Sort(named)
for i, r := range want {
if got := strings.Replace(named[i], "_", ".", 1); got != r {
t.Errorf("the buckets package names %q, the manifest reads %q", got, r)
}
}
states := map[string]int{}
for _, s := range m.State {
states[s.Name] = s.History