Compare commits

..
7 Commits
Author SHA1 Message Date
jschoubben 3ae3519714 The assigner pins mesh-sdk at c77d17f: a follower joining a watched state loses no change
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:52:33 +01:00
jschoubben 04aa1a1ace mesh-grants and the builder pin mesh-sdk at 8311ecb
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:36:58 +01:00
jschoubben b3a74e0bdc Merge builderA3's upgrade compare and builderA2's watched reader into group A 2026-11-01 03:35:05 +01:00
jschoubben b8d4a77e75 mesh-wants upgrade reads through mesh-sdk's watched reader, keeping no mirror of its own
One helper serves every part that reads whole states (ADR 0580 clause 3): the assigner's own watched mirror goes, its
compare reads the large states from stdio.WatchedStates and each manifest once per digest. mesh-sdk pinned at 8311ecb.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:34:21 +01:00
jschoubben 8112fe4b13 mesh-grants and the assigner read the grants part's key alone, root-free is each engine's word, and the hand-act log is the seat's alone (ADR 0571 clauses 9 and 11, ADR 0565 clause 11, issues 1953 and 1955)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:26:59 +01:00
jschoubben b74c371ed6 The builder asks the artifact sweep's seat for held and no longer the controller (ADR 0571 clause 2, issue 1955)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:26:59 +01:00
jschoubben f95d5b4d23 mesh-wants upgrade: read the large states by one watch each, started once, not a get per key, so compare answers in time (issue 1953 L5f)
Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
2026-11-01 03:22:12 +01:00
76 changed files with 2402 additions and 2896 deletions
+7
View File
@@ -0,0 +1,7 @@
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
@@ -14,6 +14,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
@@ -102,9 +103,9 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
t.Fatal(err)
}
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.place(t, "g14", "web")
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
running := strings.Repeat("3", 40)
b.place(t, "novox", "mesh-controller")
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
b.built(t, "mesh-controller", "novox/mesh-controller", running)
sdkHead := strings.Repeat("4", 40)
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
@@ -27,6 +27,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/compose/judge"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
)
@@ -39,8 +40,6 @@ type bed struct {
ctx context.Context
dir string
buckets map[string]jetstream.KeyValue
// placed is, by node, the modules the bed's placement names on it (place).
placed map[string][]string
}
func startBed(t *testing.T, leaveOut string) *bed {
@@ -236,7 +235,8 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
}
// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
// it from the manifest) it is read with no restart: a node it names gets its own want, the node composing its own.
// it from the manifest) it is read with no restart: a module only the placement names is
// said as placed by one source only.
pb := startBed(t, delivery.PlacementBucket)
pdone := pb.start(t)
select {
@@ -253,8 +253,13 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
t.Fatal(err)
}
if nw, err := delivery.DecodeNodeWant(pb.get(t, delivery.AssignerWantsBucket, "want.n1.node")); err != nil || !nw.Self {
t.Fatalf("the placement state made after the start was not read: n1's own want = %+v, %v", nw, err)
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised {
break
}
if time.Now().After(deadline) {
t.Fatal("the placement state made after the start was not read")
}
}
}
@@ -266,29 +271,6 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
}
}
// place adds module to the seat's placement of node, written as the seat writes it: one node key and a key per module
// (research 072 row 37). The placement alone names where modules run (ADR 0567 §8).
func (b *bed) place(t *testing.T, node, module string) {
t.Helper()
if b.placed == nil {
b.placed = map[string][]string{}
}
if !slices.Contains(b.placed[node], module) {
b.placed[node] = append(b.placed[node], module)
}
pl := delivery.Placement{Node: node}
for _, m := range b.placed[node] {
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
}
values, err := delivery.PlacementValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
b.putJSON(t, delivery.PlacementBucket, key, raw)
}
}
// built appends the end of a done trunk build of module at commit to the builder's log, as the builder does.
func (b *bed) built(t *testing.T, module, repository, commit string) {
t.Helper()
@@ -302,20 +284,24 @@ func (b *bed) built(t *testing.T, module, repository, commit string) {
}
}
// compose says the node's own composition is the digest d (a letter repeated), as its have names it (`<node>.node`): a
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
func (b *bed) compose(t *testing.T, node string, d byte) string {
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
// move to it, in mesh-controller_composed.
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
t.Helper()
key, err := delivery.HaveNodeKey(node)
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
t.Fatal(err)
}
digest := "sha256:" + strings.Repeat(string(d), 64)
v, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
if judgement != delivery.JudgedNeither {
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
}
v, err := delivery.EncodeComposedDigest(cd)
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, key, v)
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
return digest
}
@@ -343,8 +329,8 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
t.Fatal(err)
}
b.putJSON(t, delivery.ModulesBucket, "web", mod)
b.place(t, "g14", "web")
b.place(t, "ace", "other")
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
b.putJSON(t, provider.AssignmentsBucket, "ace.other", []byte(`{"node":"ace","module":"other","build":"build-2"}`))
heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
// 1927/AC1: the candidate carries the fingerprint set the SDK computes, as the indexer writes it.
@@ -425,14 +411,14 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
for _, node := range []string{"ace", "g14", "shanks"} {
b.compose(t, node, 'c')
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, node, on)
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, node+".web", have)
b.place(t, node, "web")
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
}
b.compose(t, "shanks", 'a')
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
b.start(t)
wkey, _ := delivery.WantKey("ace", "web")
@@ -440,8 +426,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if err != nil || want.FP != fNew {
t.Fatalf("the canary ace's want = %+v, %v", want, err)
}
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w)
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
}
var neither asks.Condition
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
@@ -465,8 +451,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
// there by itself: the condition is cleared.
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
t.Errorf("g14's want = %+v, want F, the node composing it", w)
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
t.Errorf("g14's want = %+v, want F with its composition", w)
}
cleared := false
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
@@ -517,7 +503,7 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
b.putJSON(t, delivery.PresenceBucket, "ace", on)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.place(t, "ace", "web")
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
b.start(t)
hkey, _ := delivery.HeldKey("web", fNew)
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
@@ -580,7 +566,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
wantKey, _ := delivery.WantKey("ace", "mail")
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
b.place(t, "ace", "mail")
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
b.start(t)
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
@@ -632,8 +618,8 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
gkey, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
b.compose(t, "ace", 'c')
b.place(t, "ace", name)
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
}
count := func() int {
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
@@ -663,6 +649,58 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
}
}
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
func TestAWantWaitsForItsComposition(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "web", mod)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey("web", fNew)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
b.start(t)
wkey, _ := delivery.WantKey("ace", "web")
time.Sleep(time.Second)
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
t.Fatalf("a want was written before the composition existed: %s", e.Value())
}
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
}
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
break
}
}
}
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
}
// the node's own want
nkey, _ := delivery.WantNodeKey("ace")
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
}
ownKey, _ := delivery.ComposedNodeKey("ace")
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
t.Fatalf("the node's own want = %+v, %v", w, err)
}
}
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
t.Helper()
@@ -677,7 +715,7 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.place(t, "ace", name)
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
}
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
@@ -740,8 +778,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.compose(t, "ace", 'a')
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
@@ -756,8 +794,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
b.get(t, delivery.AssignerAsksBucket, hold)
b.answer(t, hold, "release")
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
if w.Composed != "" || !w.Self || w.Rule != 0 {
t.Errorf("the want = %+v, want F by the rules, the node composing it", w)
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
cleared := false
@@ -787,19 +825,19 @@ func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
// value, so the node retries.
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Self: true}) // a placed node's want names the build alone
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
b.start(t)
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
a := b.askNow(t, akey, 0)
if !strings.Contains(a.Ask.Explanation, "no space left") {
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
t.Fatalf("the backup ask = %+v", a)
}
// an answer to another ask is no answer to this one
@@ -810,7 +848,7 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
}
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
if w1.FP != fNew || w1.Composed != "" || !w1.Self {
if w1.FP != fNew || w1.Composed != first {
t.Errorf("the want = %+v", w1)
}
// the node retries and the backup fails again: a new have value, asked again under the next number
@@ -825,10 +863,91 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
if w2.Words == w1.Words {
t.Errorf("the second answer left the want as it was: %+v", w2)
}
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
a3 := b.askNow(t, akey, 3)
if !strings.Contains(a3.Ask.Explanation, second) {
t.Errorf("the ask for the new revision = %+v", a3)
}
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
t.Errorf("the old answer was taken for the new revision: %+v", w)
}
}
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
// neither is written at once.
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, recorded)
old := "sha256:" + strings.Repeat("a", 64)
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
b.start(t)
akey, _ := delivery.RecordAskKey("ace", "web", nw)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
t.Fatalf("the record ask = %+v, %v", a, err)
}
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
t.Fatalf("the move was written before the yes: %+v", w)
}
b.answer(t, akey, assign.OptionYes)
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
// a new revision at the same fingerprint, judged to move data, waits as well
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
b.get(t, delivery.AssignerAsksBucket, rkey)
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
t.Fatalf("the revision was written before the yes: %+v", w)
}
// and a revision judged neither is written at once
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
}
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
b.start(t)
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
}
// the controller composes the bus at the fingerprint it runs with a new user list
key, _ := delivery.ComposedKey("ace", "bus", fOld)
free := "sha256:" + strings.Repeat("e", 64)
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
if w.FP != fOld {
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
}
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
t.Errorf("the user-list revision was asked")
}
}
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
// cleared saying why, and the want stays at the fingerprint.
@@ -836,7 +955,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
@@ -869,7 +988,7 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", 'b')
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
@@ -30,6 +30,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provisions"
"mesh-delivery-assigner/internal/assign"
@@ -57,7 +58,10 @@ func main() {
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
buscompose.ResolvedBucket,
provider.AssignmentsBucket,
delivery.PartGrantsBucket,
buscompose.LeaseBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
@@ -123,14 +127,40 @@ type buckets map[string]jetstream.KeyValue
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
out := buckets{}
// The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names
// where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the
// controller holds it, as before.
away, why := false, ""
if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil {
logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err)
} else {
out[buscompose.LeaseBucket] = kv
e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey)
var raw []byte
if gerr == nil {
raw = e.Value()
}
if errors.Is(gerr, jetstream.ErrKeyNotFound) {
gerr = nil
}
away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr)
}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
if name == buscompose.LeaseBucket {
continue
}
kv, err := js.KeyValue(ctx, name)
if name == provider.AssignmentsBucket && err != nil && away {
logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why)
continue
}
if name == delivery.AssignerPutsBucket && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if slices.Contains(seatBuckets, name) && err != nil {
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement no module is placed.
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement the controller's
// assignments alone name where modules run, as before any placement was written (ADR 0549 §5).
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
@@ -163,9 +193,10 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if name == delivery.PartGrantsBucket && err != nil {
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
if sourceOfGrants(name) && err != nil {
// The grants part's grants and the controller's lease choose where grants are read from (issue 774): without
// them the controller's grants are read, as before the lease is handed away.
logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err)
continue
}
if err != nil {
@@ -184,7 +215,13 @@ func startsWithout(name string) bool {
delivery.AssignerPutsBucket:
return true
}
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
return slices.Contains(seatBuckets, name) || sourceOfGrants(name)
}
// sourceOfGrants says the bucket is the grants part's grants or the controller's lease: neither stops the assigner when it
// is not there.
func sourceOfGrants(name string) bool {
return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket
}
// run is the program: it returns when ctx ends or the bus cannot be used.
@@ -294,7 +331,10 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
buscompose.ResolvedBucket,
provider.AssignmentsBucket,
delivery.PartGrantsBucket,
buscompose.LeaseBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
@@ -377,7 +417,12 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if e.Operation() == jetstream.KeyValuePut {
value = e.Value()
}
before := w.handedAway()
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
if name == buscompose.LeaseBucket && w.handedAway() == before {
// The controller renews its lease every few seconds: only the lease's being handed away makes a pass.
continue
}
wake()
}
}
@@ -413,13 +458,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
started := 0
for _, name := range watched {
if kvs[name] == nil && startsWithout(name) {
if kvs[name] == nil && (startsWithout(name) || name == provider.AssignmentsBucket) {
continue // not on the bus at this start (open said so)
}
started++
@@ -435,8 +480,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
return err
}
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
// module is placed.
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then the
// controller's assignments alone name where modules run (ADR 0549 §5).
type appeared struct {
name string
kv jetstream.KeyValue
@@ -449,7 +494,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
@@ -476,10 +521,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
}
if bus != nil {
p.seat = newSeatActs(bus, wake, logger.Printf)
p.seat = newSeatActs(bus, wake, logger.Printf, w.handedAway)
}
if js != nil {
p.publish = streamPublisher(ctx, js, thisNode)
@@ -19,8 +19,10 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
@@ -72,21 +74,28 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
for _, c := range []struct {
missing string
err error
starts bool
lease []byte
said string
}{
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
// Issue 774: the grants part's grants not there do not stop the start.
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
@@ -103,14 +112,12 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
}
placed, err := delivery.PlacementValues(delivery.Placement{Node: "ace"})
ckey, _ := delivery.ComposedNodeKey("ace")
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
if err != nil {
t.Fatal(err)
}
for key, raw := range placed {
kv := kvs[delivery.PlacementBucket].(*watchKV)
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
}
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
@@ -129,13 +136,18 @@ type refusingJS struct {
jetstream.JetStream
missing string
err error
lease []byte // the controller's lease, when it holds one
}
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
if name == r.missing {
return nil, r.err
}
return &memKV{values: map[string][]byte{}}, nil
kv := &memKV{values: map[string][]byte{}}
if name == buscompose.LeaseBucket && r.lease != nil {
kv.values[buscompose.ControllerLeaseKey] = r.lease
}
return kv, nil
}
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
@@ -59,10 +59,12 @@ type passer struct {
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
// §1, issue 1496): raised once per words, cleared once when it ends.
nodeFailed map[string]string
// callsCleared says the first pass cleared the conditions of refused controller calls a previous process raised
// (clearStaleCalls): the assigner makes no such call now (issue 774).
callsCleared bool
// The seat mesh-placement's holder (seat.go, ADR 0567): its acts (nil: none is made), how it says
// callSaid is, by the key of a refused controller call's condition, what is raised (issue 774): by this process, or by
// a previous one and taken over from the findings.
callSaid map[string]*callSaid
// callsSwept says the first pass took over the refused calls' conditions a previous process raised (sweepCalls).
callsSwept bool
// The seat mesh-placement's holder (seat.go, ADR 0567): its calls of the controller (nil: none is made), how it says
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
seat *seatActs
publish publisher
@@ -90,7 +92,8 @@ func (p *passer) pass(ctx context.Context) {
p.backups(ctx)
}
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
func (p *passer) nodeWants(ctx context.Context) {
written := p.w.read(delivery.AssignerWantsBucket)
placed := p.w.placedNodes()
@@ -106,6 +109,20 @@ func (p *passer) nodeWants(ctx context.Context) {
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
}
for node, d := range p.w.nodeComposed() {
if placed[node] {
continue
}
key, err := delivery.WantNodeKey(node)
if err != nil {
continue
}
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
continue
}
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
}
}
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
@@ -113,7 +130,7 @@ func (p *passer) nodeWants(ctx context.Context) {
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
func (p *passer) wants(ctx context.Context) {
get := kvGetter(p.kvs[delivery.ManifestsBucket])
assigned := p.w.assigned()
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
manifests := map[string]catalogue.Manifest{}
for name, mod := range trunk {
@@ -139,7 +156,9 @@ func (p *passer) wants(ctx context.Context) {
defer p.unplacedFindings()
defer p.seatHolders(ctx)
defer p.policySplits(manifests, chosen)
p.clearStaleCalls()
differ := p.withCalls(p.w.placementDiffers())
p.sayCalls(differ)
frozen := p.sayPlacement(differ)
placed := p.w.placedNodes()
puts := p.putsByModule()
p.staleAsks(ctx, trunk)
@@ -167,7 +186,15 @@ func (p *passer) wants(ctx context.Context) {
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
if placed[node] && p.w.handedAway() {
return p.w.ownComposed(node) // from the lease write: the digest its own have names (ADR 0571 §6)
}
if placed[node] {
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
// names the build alone, and the node judges the move itself.
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
}
return p.w.composed(node, name, fp)
},
GrantedAt: func(fp string) bool {
key, err := delivery.ControllerGrantKey(name, fp)
@@ -240,6 +267,9 @@ func (p *passer) wants(ctx context.Context) {
p.recordAsk(ctx, name, fp, need)
}
for node, want := range out.Wants {
if frozen[node+"."+name] {
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
}
key, err := delivery.WantKey(node, name)
if err != nil {
p.logf("no want of %s on %s is written: %v", name, node, err)
@@ -873,6 +903,42 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
p.said[c.Key] = true
}
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
// for them and keeps the one written before. A say that fails is made again at the next pass.
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
frozen := map[string]bool{}
now := map[string]bool{}
for _, d := range differ {
frozen[d.Node+"."+d.Module] = true
c := assign.PlacementCondition(d, true)
now[c.Key] = true
if p.said[c.Key+".raised"] {
continue
}
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
if err := p.say(c); err != nil {
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
continue
}
p.said[c.Key+".raised"] = true
}
prefix := delivery.AssignerModule + ".placement."
for key := range p.said {
c, raised := strings.CutSuffix(key, ".raised")
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
continue
}
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
continue
}
delete(p.said, key)
}
return frozen
}
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
// person must allow (ADR 0547 §3): for a have of phase
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
@@ -937,8 +1003,11 @@ func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen ma
now = p.now
}
split := map[string]bool{}
// The controller writes no policy any more (ADR 0571 §6): nothing is compared with.
for _, s := range assign.Splits(manifests, chosen, nil) {
theirs := map[string][]string{}
if !p.w.handedAway() { // the controller writes no policy from the lease write (ADR 0571 §6)
theirs = assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket))
}
for _, s := range assign.Splits(manifests, chosen, theirs) {
split[s.Module] = true
if p.splits[s.Module] {
continue
@@ -8,14 +8,17 @@ import (
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
)
@@ -49,34 +52,6 @@ func newPasser(kvs buckets, at time.Time) *passer {
now: func() time.Time { return at }}
}
// placeOn puts the seat's placement of node, naming the modules, into the world: one node key and a key per module
// (research 072 row 37).
func placeOn(t *testing.T, w *world, node string, modules ...string) {
t.Helper()
pl := delivery.Placement{Node: node}
for _, m := range modules {
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
}
values, err := delivery.PlacementValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
w.put(delivery.PlacementBucket, key, raw)
}
}
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
func composes(t *testing.T, w *world, node, digest string) {
t.Helper()
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
w.put(delivery.HaveBucket, key, raw)
}
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
@@ -221,7 +196,7 @@ func putOfTheBus(t *testing.T) {
p := newPasser(kvs, now)
bus := &fakeBus{calls: make(chan string, 8)}
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, nil)
p.acts.snapshot = func(context.Context, string, string) (string, error) {
return "novox's restore point of nats (streams) taken 22:01", nil
}
@@ -236,7 +211,7 @@ func putOfTheBus(t *testing.T) {
}
p.w.put(bucket, key, data)
}
placeOn(t, p.w, "novox", "nats")
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
mdigest := strings.Repeat("e", 64)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
world(delivery.ModulesBucket, "nats", mod)
@@ -292,7 +267,9 @@ func putOfTheBus(t *testing.T) {
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
default:
}
composes(t, p.w, "novox", digest)
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
world(delivery.ControllerComposedBucket, ckey, cd)
gkey, _ := delivery.ControllerGrantKey("nats", fp)
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
world(delivery.PartGrantsBucket, gkey, g)
@@ -379,7 +356,7 @@ func putOfTheBus(t *testing.T) {
// the journal, so it is loud with no operator channel.
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
aPlacementAloneNamesWhereModulesRun(t)
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []asks.Condition
@@ -553,7 +530,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
placeOn(t, p.w, "g14", "web")
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
@@ -591,7 +568,7 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
placeOn(t, p.w, "shanks", "systemd-resolved")
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
p.w.put(delivery.PresenceBucket, "shanks", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
@@ -655,13 +632,15 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
name string
trunk map[string]delivery.Module
gone []string // the keys taken away
noDigit bool // the node's own composition is not read
noDigit bool // the controller's composition is not read
heldAt string // the node's want is at this fingerprint, whose composition is digOld
}{
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true},
{"the trunk is not read at all", nil, nil, false},
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
{"the trunk is not read at all", nil, nil, false, ""},
} {
t.Run(c.name, func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
@@ -688,9 +667,21 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
asksKV.values[key] = v
p.w.put(delivery.AssignerAsksBucket, key, v)
}
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
if !c.noDigit {
composes(t, p.w, "novox", digNew)
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
p.w.put(delivery.ControllerComposedBucket, ckey, v)
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
if c.heldAt != "" {
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
}
p.w.put(delivery.ControllerComposedBucket, okey, old)
}
if c.heldAt != "" {
wkey, _ := delivery.WantKey("novox", "web")
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
p.w.put(delivery.AssignerWantsBucket, wkey, w)
}
heldOld, _ := delivery.HeldKey("web", fpOld)
wantsKV.values[heldOld] = []byte("x")
@@ -788,38 +779,134 @@ func mustKey(key string, err error) string {
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
if cerr != nil {
t.Fatal(cerr)
}
put(delivery.ControllerComposedBucket, ckey, cd)
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
put(delivery.ControllerComposedBucket, tkey, tcd)
akey, _ := delivery.RecordAskKey("g14", "web", revised)
for i := 0; i < 2; i++ {
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
}
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil {
t.Fatal(err)
}
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
p.wants(context.Background())
got, err := delivery.DecodeWant(wantsKV.values[wkey])
if err != nil || got.FP != fpOld || got.Composed != revised {
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
}
if _, there := asksKV.values[akey]; !there {
t.Error("the answered record ask was taken away")
}
}
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
// module in an assignment only is no module of its node; before any placement is read nothing is placed and the check composes
// no input; placed, the check composes over the input the nodes compose from (frombus, 1126, ADR 0547 §2).
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
// assignments are the one source and nothing is raised.
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
if got := p.w.assigned(); len(got) != 0 {
t.Fatalf("before any placement: assigned %v", got)
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
assigned := func(node, module string) {
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
}
place := func(node string, modules ...string) {
pl := delivery.Placement{Node: node}
for _, m := range modules {
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
}
// One node key and a key per module (research 072 row 37); a module taken off leaves its key, which the node
// key no longer names.
values, err := delivery.PlacementValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(delivery.PlacementBucket, key, raw)
}
}
assigned("g14", "a")
assigned("g14", "b")
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
t.Fatalf("before any placement: assigned %v, said %v", got, said)
}
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
}
placeOn(t, p.w, "g14", "c", "a")
placeOn(t, p.w, "n2", "d")
if got := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "c"}) || !slices.Equal(got["n2"], []string{"d"}) || len(got) != 2 {
t.Errorf("the placement alone: assigned %v", got)
place("g14", "a", "c")
got, _ := p.w.assigned()
frozen := p.sayPlacement(p.w.placementDiffers())
p.sayPlacement(p.w.placementDiffers())
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
!strings.Contains(said[1].Explanation, "the seat mesh-placement's placement names c on g14") {
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
}
place("g14", "a", "b", "c")
assigned("g14", "c")
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
}
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
// wrote its record; until then it says why and composes the assignments alone.
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
}
placeOn(t, p.w, "g14", "a", "b", "c")
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
p.w.put(delivery.HaveBucket, "g14.node", rec)
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n2", TunnelKey: "k2"}})
p.w.put(delivery.HaveBucket, "n2.node", rec2)
m := p.w.mesh(context.Background(), none)
if m.Input == nil || len(m.Input.Nodes) != 2 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
}
// Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules
// run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of
// the node. A lease that goes back to a controller brings the comparison back.
place("n2", "d")
assigned("g14", "e")
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`))
got, all := p.w.assigned()
if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 {
t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all)
}
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 {
t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:])
}
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`))
if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] {
t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen)
}
}
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
@@ -834,7 +921,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
placeOn(t, p.w, "g14", "web")
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
put(delivery.PlacementBucket, "placement.g14", pl)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
@@ -871,10 +958,15 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
if _, there := asksKV.values[akey]; !there {
t.Fatal("a have waiting for a yes raised no record ask")
}
// Issue 774, 2026-10-31: the node's own composition is not the digest its have waits on; the ask stays, and a pass that
// changes nothing puts nothing: on novox the asks of postgres, keycloak and mongodb were taken away and made again every
// 8 s, and a want put unchanged each pass made the node compose again each time.
composes(t, p.w, "g14", "sha256:"+strings.Repeat("c", 64))
// Issue 774, 2026-10-31: while the controller still composes the placed node's module (its lease not handed away), its
// digests are not the one the node's have waits on; the ask stays, and a pass that changes nothing puts nothing: on novox
// the asks of postgres, keycloak and mongodb were taken away and made again every 8 s, and a want put unchanged each
// pass made the node compose again each time.
for _, fp := range []string{fpTest, fpOld} {
ck, _ := delivery.ComposedKey("g14", "web", fp)
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedUnchanged})
put(delivery.ControllerComposedBucket, ck, cv)
}
asked := asksKV.values[akey]
p.wants(context.Background())
if !bytes.Equal(asksKV.values[akey], asked) {
@@ -911,37 +1003,178 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
}
}
// 774/S25/AC1 (ADR 0567 §8): an approved placement change is written and followed with its warrant recorded and
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
// cleared once, and said again at a later pass while the findings cannot be read.
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
bus := &fakeBus{calls: make(chan string, 8)}
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
var said []string
p.publish = func(event string, _ any, _ string) error { said = append(said, event); return nil }
prop := delivery.PlacementProposal{ID: "p-one", By: "operator@laptop", Why: "move web"}
change := delivery.ChangeOf(delivery.Placement{Node: "anchor"}, delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}})
p.followWrite(prop, delivery.Answer{}, []delivery.PlacementChange{change})
if len(bus.calls) != 0 || !slices.Equal(said, []string{delivery.PlacementChangedEvent}) {
t.Fatalf("calls = %d, said = %v", len(bus.calls), said)
}
// refusingBus is the assigner's process account against a controller that refuses the calls whose verb is in refuse.
type refusingBus struct {
fakeBus
refuse map[string]string
// pins are the controller's pins its placement-export answers, by node.
mu sync.Mutex
pins map[string]map[string]delivery.Provider
}
func (b *refusingBus) setPins(node string, pins map[string]delivery.Provider) {
b.mu.Lock()
defer b.mu.Unlock()
if b.pins == nil {
b.pins = map[string]map[string]delivery.Provider{}
}
b.pins[node] = pins
}
func (b *refusingBus) Call(key string, body any) (json.RawMessage, error) {
if key == exportCall {
b.mu.Lock()
defer b.mu.Unlock()
node, _ := body.(map[string]any)["node"].(string)
answer, _ := json.Marshal(map[string]delivery.Placement{node: {V: delivery.Version, Node: node, Pins: b.pins[node]}})
return json.Marshal(map[string]any{"ok": true, "answer": json.RawMessage(answer)})
}
for verb, words := range b.refuse {
if strings.HasSuffix(key, "."+verb) {
return nil, errors.New(words)
}
}
return json.RawMessage(`{"ok":true}`), nil
}
// Issue 774: a call of the controller it refuses is a condition, never only a journal line: raised once, urgent, naming the verb,
// the node, the module and the refusal; cleared when a later call for the pair is answered, or, for a module's place, once the
// pair differed and the two sources agree.
func TestARefusedControllerCallIsACondition(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
var conds []asks.Condition
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
fails := 1
p.findings.Keys = func() ([]string, error) {
if fails > 0 {
fails--
return nil, errors.New("the bucket is not there yet")
}
return []string{"module.anchor.web.controller-call", "module.laptop.db.controller-pin", "module.anchor.web.neither"}, nil
bus := &refusingBus{refuse: map[string]string{"assign": `assign takes no argument "why" — it takes "build", "module", "node"; nothing was done`}}
woke := make(chan struct{}, 4)
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
assignWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
call := func(c assign.ControllerCall) {
p.seat.callAll([]assign.ControllerCall{c})
<-woke
}
p.clearStaleCalls()
p.clearStaleCalls()
p.clearStaleCalls()
if len(conds) != 2 || conds[0].State != asks.StateCleared || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
conds[1].Key != "mesh-delivery-assigner.controller-pin.laptop.db" {
t.Fatalf("the old conditions cleared = %+v", conds)
differ := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
call(assignWeb)
p.sayCalls(differ)
p.sayCalls(differ)
if len(conds) != 1 || conds[0].State != asks.StateRaised || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
!strings.Contains(conds[0].Explanation, `takes no argument "why"`) || !strings.Contains(conds[0].Headline, "assign web on anchor") {
t.Fatalf("after a refused assign, said = %+v", conds)
}
bus.refuse = nil
call(assignWeb)
p.sayCalls(nil)
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != conds[0].Key {
t.Fatalf("after the assign is answered, said = %+v", conds)
}
// Refused again, then assigned by hand: cleared once the pair, seen differing, agrees; not before it was seen differing.
bus.refuse = map[string]string{"assign": "refused"}
call(assignWeb)
p.sayCalls(nil)
if len(conds) != 3 || conds[2].State != asks.StateRaised {
t.Fatalf("a second refusal is not raised: %+v", conds)
}
p.sayCalls(nil)
if len(conds) != 3 {
t.Fatalf("cleared before the pair was seen differing: %+v", conds)
}
p.sayCalls(differ)
p.sayCalls(nil)
if len(conds) != 4 || conds[3].State != asks.StateCleared || !strings.Contains(conds[3].Resolved, "agree") {
t.Fatalf("the hand assign does not clear it: %+v", conds)
}
p.sayCalls(nil)
if len(conds) != 4 {
t.Fatalf("said again after the clear: %+v", conds)
}
}
// Issue 774: a refused pin's condition, and one a previous process raised, clear once the pair agrees after it was seen differing:
// the controller's pins (its placement-export, read again when its compositions change) against the placement's, the controller's
// assignments against the placement for a module's place. A start whose findings cannot be read takes them over at a later pass.
func TestARefusedCallClearsWhenTheSourcesAgree(t *testing.T) {
store := delivery.Provider{Node: "anchor", Module: "store"}
pinDB := assign.ControllerCall{Verb: "pin", Node: "laptop", Module: "db",
Args: map[string]any{"node": "laptop", "provision": "db", "from": "anchor", "module": "store"}}
placeWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
webDiffers := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
for name, c := range map[string]struct {
call *assign.ControllerCall // refused in this process; nil: raised by a previous one
findings []string // the findings a previous process left
keysFail int // how many first reads of the findings fail
differ []assign.Pair // the pairs the two sources name apart, while they differ
agree func(*refusingBus) // the person's hand call
key string
}{
"a pin refused, pinned by hand": {call: &pinDB, agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
"a place refused, assigned by hand": {call: &placeWeb, differ: webDiffers, agree: func(*refusingBus) {},
key: "mesh-delivery-assigner.controller-call.anchor.web"},
"a pin raised before a start": {findings: []string{"module.laptop.db.controller-pin"}, keysFail: 1,
agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
"a place raised before a start": {findings: []string{"module.anchor.web.controller-call"}, differ: webDiffers, agree: func(*refusingBus) {},
key: "mesh-delivery-assigner.controller-call.anchor.web"},
} {
t.Run(name, func(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
var conds []asks.Condition
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
fails := c.keysFail
p.findings.Keys = func() ([]string, error) {
if fails > 0 {
fails--
return nil, errors.New("the bucket is not there yet")
}
return c.findings, nil
}
bus := &refusingBus{refuse: map[string]string{"pin": "refused", "assign": "refused"}}
woke := make(chan struct{}, 16)
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
// The placement pins db on laptop; the controller does not.
raw, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "laptop", Pins: map[string]delivery.Provider{"db": store}})
key, _ := delivery.LegacyPlacementKey("laptop")
p.w.put(delivery.PlacementBucket, key, raw)
// pass is a pass, then one more for each background read it started.
pass := func(differ []assign.Pair) {
p.sayCalls(differ)
for {
select {
case <-woke:
p.sayCalls(differ)
case <-time.After(20 * time.Millisecond):
return
}
}
}
if c.call != nil {
p.seat.callAll([]assign.ControllerCall{*c.call})
<-woke
}
pass(c.differ)
pass(c.differ)
for _, cond := range conds {
if cond.State == asks.StateCleared {
t.Fatalf("cleared while the sources differ: %+v", conds)
}
}
c.agree(bus)
p.w.put(delivery.ControllerComposedBucket, "laptop.x", []byte("moved")) // a pin changes a composition
pass(nil)
if len(conds) == 0 {
t.Fatal("nothing was raised")
}
last := conds[len(conds)-1]
if last.Key != c.key || last.State != asks.StateCleared || !strings.Contains(last.Resolved, "agree") {
t.Fatalf("the hand call does not clear it: %+v", conds)
}
n := len(conds)
pass(nil)
if len(conds) != n {
t.Errorf("said again after the clear: %+v", conds[n:])
}
})
}
}
@@ -55,8 +55,8 @@ type putActs struct {
snapshot func(ctx context.Context, node, module string) (string, error)
}
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
a := &putActs{rec: handacts.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs {
a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
said: map[string]string{}}
a.snapshot = a.snapshotTheBus
return a
@@ -97,7 +97,7 @@ func (p *passer) puts(ctx context.Context) {
if len(all) == 0 {
return
}
assigned := p.w.assigned()
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
get := kvGetter(p.kvs[delivery.ManifestsBucket])
wanted := p.w.wantsOf()
@@ -10,12 +10,14 @@ import (
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/compose/judge"
@@ -57,23 +59,80 @@ func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) p
}
}
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
// the controller: placement is the one source (ADR 0567 §8, issue 774).
// seatActs keeps what the seat's calls of the controller came to while this process runs: the export read again at a
// seed's warrant, and each call that keeps the controller's assignments in step (§8).
type seatActs struct {
bus caller
wake func()
logf func(string, ...any)
mu sync.Mutex
bus caller
wake func()
logf func(string, ...any)
export map[string]exportCheck
// calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to.
calls map[string]string
// outcomes is, by the key of a call's condition, the last call made for its pair once answered or refused: refused
// "" when the controller answered it (issue 774: a refused call is a condition, never only a journal line).
outcomes map[string]callOutcome
// pins is, by node, the controller's pins as its placement-export last answered them (pinsOf).
pins map[string]pinRead
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
rec handacts.Recorder
}
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
return &seatActs{bus: bus, wake: wake, logf: logf, rec: handacts.Recorder{Ask: bus.Call}}
// callOutcome is what one call of the controller came to: the call, and the controller's refusal ("" when it answered).
type callOutcome struct {
call assign.ControllerCall
refused string
}
// exportCheck is the export read again for one proposal: running, its answer, or why it was not read.
type exportCheck struct {
running bool
export map[string]delivery.Placement
err string
}
func newSeatActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *seatActs {
return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{},
outcomes: map[string]callOutcome{}, pins: map[string]pinRead{}, rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}}
}
// exportOf is the export read again for a proposal, asked once in the background.
func (a *seatActs) exportOf(id string) exportCheck {
a.mu.Lock()
defer a.mu.Unlock()
if c, ok := a.export[id]; ok {
return c
}
a.export[id] = exportCheck{running: true}
go func() {
defer a.wake()
raw, err := a.bus.Call(exportCall, map[string]any{})
c := exportCheck{}
if err != nil {
c.err = "the controller's placement-export did not answer: " + err.Error()
} else if c.export, err = assign.ExportOf(raw); err != nil {
c.err = err.Error()
}
a.mu.Lock()
a.export[id] = c
a.mu.Unlock()
}()
return exportCheck{running: true}
}
// forget drops what is kept of a proposal's export, so a later warrant reads it again.
func (a *seatActs) forget(id string) {
if a == nil {
return
}
a.mu.Lock()
delete(a.export, id)
a.mu.Unlock()
}
// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
// the controller's while that seat has no holder (ADR 0565 §11, handacts.Recorder).
// the controller's while that seat has no holder and the controller holds its lease (ADR 0565 §11, handacts.Recorder).
func (a *seatActs) record(warrant string) {
if a == nil || warrant == "" {
return
@@ -88,35 +147,257 @@ func (a *seatActs) record(warrant string) {
}()
}
// clearStaleCalls clears, once, the conditions of the refused controller calls a previous process raised (findings of the
// kinds controller-call and controller-pin): the assigner calls no verb of the controller any more (ADR 0567 §8, issue 774),
// so nothing would clear them. It is tried again at the next pass while the findings cannot be read.
func (p *passer) clearStaleCalls() {
if p.callsCleared || p.say == nil || p.findings.Keys == nil {
// handAct records a hand-back in the hand-act log (§7), as record does.
func (a *seatActs) handAct(prop delivery.PlacementProposal) {
go func() {
if taken, err := a.rec.Record(handacts.Request{What: "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "),
Why: prop.Why, Cause: "asked by " + prop.By + ", approved by the operator"}); err != nil {
a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err)
} else {
a.logf("the hand-back %s is recorded in the log of %s", prop.ID, taken.Log)
}
}()
}
// callAll makes the controller's calls in order in the background, each said in the journal and kept by its pair.
func (a *seatActs) callAll(calls []assign.ControllerCall) {
if len(calls) == 0 {
return
}
a.mu.Lock()
for _, c := range calls {
a.calls[c.Node+"."+c.Module] = c.Key() + " is asked"
}
a.mu.Unlock()
go func() {
defer a.wake()
for _, c := range calls {
words, refused := c.Key()+" answered", ""
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+"."+c.Verb, c.Args); err != nil {
refused = err.Error()
words = c.Key() + " refused: " + refused
}
a.logf("keeping the controller in step with placement: %s", words)
a.mu.Lock()
a.calls[c.Node+"."+c.Module] = words
a.outcomes[assign.CallConditionKey(c)] = callOutcome{call: c, refused: refused}
a.mu.Unlock()
}
}()
}
// outcomesNow is a copy of every call's last outcome, by the key of its condition.
func (a *seatActs) outcomesNow() map[string]callOutcome {
if a == nil {
return nil
}
a.mu.Lock()
defer a.mu.Unlock()
return maps.Clone(a.outcomes)
}
// pinRead is the controller's pins of one node as its placement-export answered them, for the generation of the
// controller's compositions it was asked at: a pin made at the controller changes a composition, which the assigner
// watches, so a read is asked again only after such a change, never on a timer.
type pinRead struct {
gen uint64
running bool
known bool
pins map[string]delivery.Provider
}
// pinsOf is the controller's pins of node as last read, and whether a read answered; a read is asked in the background
// when none was made at gen, and wakes a pass when it answers.
func (a *seatActs) pinsOf(node string, gen uint64) (map[string]delivery.Provider, bool) {
if a == nil {
return nil, false
}
a.mu.Lock()
defer a.mu.Unlock()
r, ok := a.pins[node]
if !ok || (r.gen != gen && !r.running) {
r.gen, r.running = gen, true
a.pins[node] = r
go func() {
defer a.wake()
raw, err := a.bus.Call(exportCall, map[string]any{"node": node})
var pins map[string]delivery.Provider
var known bool
if err == nil {
var export map[string]delivery.Placement
if export, err = assign.ExportOf(raw); err == nil {
pins, known = export[node].Pins, true
}
}
if err != nil {
a.logf("the controller's pins of %s were not read, so a refused pin's condition there waits: %v", node, err)
}
a.mu.Lock()
now := a.pins[node]
now.running = false
if known {
now.pins, now.known = pins, true
}
a.pins[node] = now
a.mu.Unlock()
}()
}
return r.pins, r.known
}
// sayCalls makes a refused call of the controller loud (issue 774): the condition of each pair whose last call was refused
// is raised once per refusal, naming the verb, the node, the module and the controller's words. It is cleared when a later
// call for the pair is answered, or once the pair, seen differing since the refusal, agrees between the two sources (a
// person made the call by hand): for a module's place the controller's assignments and the placement, for a pin the
// controller's pins (its placement-export, read again after its compositions change) and the placement's. The first pass
// after a start takes over what a previous process raised (its findings) and clears each whose pair agrees; until the
// findings are read it is tried again at every pass. A say that fails is made again at the next pass.
func (p *passer) sayCalls(differ []assign.Pair) {
if p.say == nil {
return
}
if p.callSaid == nil {
p.callSaid = map[string]*callSaid{}
}
differs := map[string]bool{}
for _, d := range differ {
differs[d.Node+"."+d.Module] = true
}
outcomes := p.seat.outcomesNow()
if !p.callsSwept {
p.callsSwept = p.sweepCalls(outcomes)
}
for key, o := range outcomes {
said := p.callSaid[key]
switch {
case o.refused != "" && (said == nil || said.adopted || said.refused != o.refused):
if err := p.say(assign.CallRefusedCondition(o.call, o.refused, true, "")); err != nil {
p.logf("that %s was refused could not be said, and is said at the next pass: %v", o.call.Key(), err)
continue
}
p.callSaid[key] = &callSaid{call: o.call, refused: o.refused}
case o.refused == "" && said != nil:
if err := p.say(assign.CallRefusedCondition(o.call, "", false, o.call.Key()+" was answered.")); err != nil {
p.logf("the condition of %s could not be cleared, and is cleared at the next pass: %v", o.call.Key(), err)
continue
}
delete(p.callSaid, key)
}
}
var placements map[string]delivery.Placement
for key, said := range p.callSaid {
c := said.call
var apart bool
if c.CallKind() == assign.KindControllerCall {
apart = differs[c.Node+"."+c.Module]
} else {
theirs, known := p.seat.pinsOf(c.Node, p.w.bucketGen(delivery.ControllerComposedBucket))
if !known {
continue
}
if placements == nil {
placements, _ = p.seatPlacements()
}
ours, has := placements[c.Node].Pins[c.Module]
to, had := theirs[c.Module]
apart = has != had || ours != to
}
if apart {
said.differed = true
continue
}
if !said.differed {
continue // not seen differing since the refusal: the world may not hold the write yet
}
if err := p.say(assign.CallRefusedCondition(c, "", false, agreeWords(c))); err != nil {
p.logf("the condition of %s could not be cleared, and is cleared at the next pass: %v", c.Key(), err)
continue
}
if o, ok := outcomes[key]; ok {
p.seat.forgetOutcome(key, o)
}
delete(p.callSaid, key)
}
}
// sweepCalls takes over the refused calls' conditions a previous process raised, from the findings: each is followed as
// one seen differing, so it clears at the first pass its pair agrees. It answers whether the findings were read.
func (p *passer) sweepCalls(outcomes map[string]callOutcome) bool {
if p.findings.Keys == nil {
return true
}
keys, err := p.findings.Keys()
if err != nil {
p.logf("the findings were not read, so the refused controller calls a previous process raised are cleared at the next pass: %v", err)
return
p.logf("the findings were not read, so the refused calls a previous process raised are taken over at the next pass: %v", err)
return false
}
p.callsCleared = true
for _, k := range keys {
rest, ok := strings.CutPrefix(k, "module.")
if !ok {
continue
}
for _, kind := range []string{assign.KindControllerCall, assign.KindControllerPin} {
if pair, ok := strings.CutSuffix(rest, "."+kind); ok {
if node, module, two := strings.Cut(pair, "."); two {
if err := p.say(assign.CallClearedCondition(kind, node, module)); err != nil {
p.callsCleared = false
p.logf("the condition of the old %s call for %s on %s could not be cleared, and is cleared at the next pass: %v", kind, module, node, err)
}
}
pair, ok := strings.CutSuffix(rest, "."+kind)
node, module, two := strings.Cut(pair, ".")
if !ok || !two {
continue
}
verb := "assign"
if kind == assign.KindControllerPin {
verb = "pin"
}
c := assign.ControllerCall{Verb: verb, Node: node, Module: module}
key := assign.CallConditionKey(c)
if _, made := outcomes[key]; made || p.callSaid[key] != nil {
continue
}
p.callSaid[key] = &callSaid{call: c, differed: true, adopted: true}
}
}
return true
}
func agreeWords(c assign.ControllerCall) string {
if c.CallKind() == assign.KindControllerPin {
return "the controller's pins and the placement agree on " + c.Module + " for " + c.Node + "."
}
return "the controller's assignments and the placement agree on " + c.Module + " on " + c.Node + "."
}
// callSaid is a raised call's condition: its call, the refusal it said, whether its pair was seen differing since, and
// whether it was taken over from a previous process.
type callSaid struct {
call assign.ControllerCall
refused string
differed bool
adopted bool
}
// forgetOutcome drops a refused call's outcome once its condition is cleared, unless a later call replaced it.
func (a *seatActs) forgetOutcome(key string, o callOutcome) {
a.mu.Lock()
defer a.mu.Unlock()
if now, ok := a.outcomes[key]; ok && now.refused == o.refused && now.call.Verb == o.call.Verb {
delete(a.outcomes, key)
}
}
// callOf is what the last call for a pair came to, "" when none was made.
func (a *seatActs) callOf(node, module string) string {
if a == nil {
return ""
}
a.mu.Lock()
defer a.mu.Unlock()
return a.calls[node+"."+module]
}
// withCalls names, on each pair the two sources do not both name, the call the assigner made for it (§8).
func (p *passer) withCalls(pairs []assign.Pair) []assign.Pair {
for i := range pairs {
pairs[i].Call = p.seat.callOf(pairs[i].Node, pairs[i].Module)
}
return pairs
}
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
@@ -263,18 +544,17 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos
p.endProposal(ctx, prop, prop.Withdrawn)
return
}
if prop.Seed || prop.HandBack {
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone")
return
}
akey, _ := delivery.PlacementAskKey(prop.ID)
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
if prop.State == delivery.ProposalAsked {
askPulse, _ := delivery.PulseAskKey(prop.ID)
switch {
case p.answeredWith(akey, assign.OptionYes):
p.warranted(ctx, prop, rev, akey, current, revs)
if prop.HandBack {
p.handBack(ctx, prop, akey, revs)
} else {
p.warranted(ctx, prop, rev, akey, current, revs)
}
return
case p.answeredWith(akey, assign.OptionNo):
p.endProposal(ctx, prop, "declined by the operator")
@@ -284,6 +564,19 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos
return
}
}
if prop.HandBack {
// The operator's alone (§7): no node answers it; its ask is raised at once, and again on again.
if prop.State != delivery.ProposalAsked || again {
brief := fmt.Sprintf("%s (asked by %s): every node's placement key is deleted, %s, and each node returns to the "+
"controller's compositions; the history is kept", prop.Why, prop.By, strings.Join(prop.Nodes(), ", "))
now := p.clock().UTC()
prop.Verdict = &delivery.PlacementVerdict{State: delivery.VerdictPass, At: now}
if prop, ok := p.ask(ctx, prop, akey, brief); ok {
p.writeProposal(ctx, prop, rev)
}
}
return
}
if moved(prop, revs) {
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
return
@@ -454,7 +747,8 @@ func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
held, recorded := p.lays(ctx)
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
Current: current, Revisions: revs, Held: held, Recorded: recorded})
Current: current, Revisions: revs, Held: held, Recorded: recorded,
Seed: prop.Seed, ExportDigest: prop.ExportDigest, Leased: p.w.handedAway()})
if len(findings) > 0 {
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
return
@@ -467,6 +761,7 @@ func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal,
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
}
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
p.seat.forget(prop.ID)
p.writeProposal(ctx, again, rev)
}
@@ -483,14 +778,35 @@ func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
}
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
// proposal deleted, and placement-changed said.
// warranted writes an approved proposal (§5): for the seed, after the export is read again and found unchanged (§7);
// each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
// proposal deleted, placement-changed said, and the controller kept in step (§8).
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
return
}
answer, _ := p.w.answer(akey)
if prop.Seed {
if p.seat == nil {
return
}
c := p.seat.exportOf(prop.ID)
switch {
case c.running:
return // its answer wakes the next pass
case c.err != "":
if words := "the export cannot be read again at the warrant: " + c.err; prop.Words != words {
prop.Words = words
p.writeProposal(ctx, prop, rev)
}
return
case delivery.ExportDigest(c.export) != prop.ExportDigest:
p.endProposal(ctx, prop, "the controller's export changed since the seed was proposed: the new export is proposed as the seed")
p.proposeSeed(ctx, prop, c.export)
return
}
}
if moved(prop, revs) {
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
return
@@ -515,7 +831,7 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal,
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
// again over the keys as this write left them.
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
p.followWrite(prop, answer, changes)
p.followWrite(prop, answer, changes, current, writtenLaid)
if rest, ok := assign.Rest(prop, writtenLaid); ok {
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
} else {
@@ -541,17 +857,22 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal,
p.deletePulses(ctx, prop.ID)
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.followWrite(prop, answer, changes)
p.followWrite(prop, answer, changes, current, writtenLaid)
}
// followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the
// controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774).
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
// followWrite follows the keys a warrant wrote: the warrant recorded, placement-changed said, and, until the lease
// write, the controller's assignments kept in step for them (§8). The seed calls no controller verb: it is what the
// controller holds already (§9).
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange,
before map[string]delivery.Placement, written map[string]*delivery.Placement) {
if len(changes) == 0 {
return
}
p.seat.record(answer.Warrant)
p.sayChanged(prop, answer, changes)
if p.seat != nil && !p.w.handedAway() && !prop.Seed {
p.seat.callAll(assign.ControllerCalls(before, written))
}
}
func sortedNodes(m map[string]*delivery.Placement) []string {
@@ -563,6 +884,44 @@ func sortedNodes(m map[string]*delivery.Placement) []string {
return out
}
// handBack deletes every key on the operator's warrant (§7) at the revision the hand-back read, records it in the
// controller's hand-act log and says placement-changed; every node returns to the controller's compositions. A key moved
// since ends it, and the operator hands back again.
func (p *passer) handBack(ctx context.Context, prop delivery.PlacementProposal, akey string, revs map[string]uint64) {
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
p.endProposal(ctx, prop, "its warrant does not bind this hand-back, so nothing is deleted: "+err.Error())
return
}
answer, _ := p.w.answer(akey)
current, held := p.seatPlacements()
var changes []delivery.PlacementChange
for _, node := range prop.Nodes() {
key, _ := delivery.PlacementNodeKey(node)
err := error(nil)
if held[node] != prop.Base[node] {
err = fmt.Errorf("its placement is at revision %d, the hand-back read %d", held[node], prop.Base[node])
} else {
_, err = p.writeNode(ctx, node, nil)
}
if err != nil {
p.sayChanged(prop, answer, changes)
p.endProposal(ctx, prop, fmt.Sprintf("%s moved since the hand-back was asked (%v): hand back again", key, err))
return
}
c := delivery.ChangeOf(current[node], delivery.Placement{Node: node})
c.Before = prop.Base[node]
changes = append(changes, c)
}
if p.seat != nil {
p.seat.handAct(prop)
}
p.seat.record(answer.Warrant)
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.deletePulses(ctx, prop.ID)
p.sayChanged(prop, answer, changes)
}
// sayChanged says placement-changed for the keys written.
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
if len(changes) == 0 {
@@ -579,6 +938,26 @@ func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Ans
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
}
// proposeSeed proposes an export as the seed again, by the same proposer and why.
func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal, export map[string]delivery.Placement) {
held, recorded := p.lays(ctx)
now := p.clock()
seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now,
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: p.w.handedAway()})
if len(findings) > 0 {
p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; "))
return
}
for _, node := range seed.Nodes() {
key, _ := delivery.PlacementCandidateKey(seed.ID, node)
c := delivery.PlacementCandidate{Proposal: seed.ID, Placement: laid[node]}
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
}
seed.Written = now.UTC()
p.writeProposal(ctx, seed, 0)
}
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
// whole with why it ended.
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
@@ -594,6 +973,7 @@ func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposa
return
}
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.seat.forget(prop.ID)
p.logf("the proposal %s ends: %s", prop.ID, why)
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
}
@@ -10,6 +10,7 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -206,7 +207,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
}
for _, c := range changes {
if c.Op == delivery.OpSeed {
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
}
}
current, revs, err := s.placements()
@@ -219,7 +220,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
}
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded})
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
if len(findings) > 0 {
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
}
@@ -269,15 +270,92 @@ func (s *seatTools) doAgain(args map[string]any) (any, error) {
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
}
// doAdopt is the verb adopt (§7): gone since the lease write. The seed was the controller's export proposed whole, and the
// mesh has no controller export to take over (ADR 0547 §11, ADR 0567 §11); nothing is written.
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
return nil, errors.New("adopt is gone: placement is the mesh's alone since the lease write, and there is no controller to take it over from (ADR 0547 §11, ADR 0567 §11)")
current, _, err := s.placements()
if err != nil {
return nil, err
}
if len(current) > 0 {
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
}
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
for _, id := range ids {
if p, err := s.proposal(id); err == nil && p.Seed {
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
}
}
}
raw, err := s.ask(exportCall, map[string]any{})
if err != nil {
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
}
export, err := assign.ExportOf(raw)
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
if err != nil {
return nil, err
}
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
if len(findings) > 0 {
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p, laid); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// doHandBack is the verb hand-back (§7): gone since the lease write, as adopt is; nothing is deleted.
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
// unreadable, is not handed away.
func (s *seatTools) leased() bool {
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
return away
}
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
// records the hand act and says placement-changed; every node returns to the controller's compositions.
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
return nil, errors.New("hand-back is gone: placement is the mesh's alone since the lease write, and there is no controller to hand it back to (ADR 0547 §11, ADR 0567 §11)")
if s.leased() {
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
}
why := arg(args, "why")
if why == "" {
return nil, errors.New("hand-back says why: it is recorded with the hand act")
}
if by == "" {
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
}
_, revs, err := s.placements()
if err != nil {
return nil, err
}
if len(revs) == 0 {
return nil, errors.New("there is no placement key to hand back")
}
now := s.now()
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the hand-back is not written: %w", err)
}
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
"node returns to the controller's compositions"}, nil
}
// placementAnswer is what the verb placement answers.
@@ -13,6 +13,7 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
@@ -36,8 +37,8 @@ func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
}
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt and hand-back are
// answered as gone and change nothing (ADR 0547 §11, ADR 0567 §11); placement and
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt proposes the
// controller's export whole as the seed, once; hand-back records the hand act and deletes every key; placement and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
@@ -52,35 +53,32 @@ func TestTheSeatsVerbs(t *testing.T) {
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
st[haveState].values[n+".node"] = raw
}
export, _ := json.Marshal(map[string]any{"anchor": anchor})
var asked []string
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if key == exportCall {
out, _ := json.Marshal(map[string]any{"ok": true, "output": string(export)})
return out, nil
}
return json.RawMessage(`{"ok":true}`), nil
})
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
// 774/S28/AC1: adopt and hand-back are answered as gone, naming their records, and change nothing and ask nobody.
raw, _ := delivery.EncodePlacement(anchor)
st[delivery.PlacementState].values["placement.anchor"] = raw
for verb, call := range map[string]func() (any, error){
"adopt": func() (any, error) { return s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop") },
"hand-back": func() (any, error) { return s.doHandBack(map[string]any{"why": "back"}, "op") },
} {
if _, err := call(); err == nil || !strings.Contains(err.Error(), verb+" is gone") || !strings.Contains(err.Error(), "ADR 0567 §11") {
t.Errorf("%s: %v", verb, err)
}
}
if len(asked) != 0 || len(st[delivery.PlacementProposalsState].values) != 0 || len(st[delivery.PlacementCandidatesState].values) != 0 ||
len(st[delivery.PlacementState].values) != 1 {
t.Fatalf("adopt or hand-back changed something: asked %v", asked)
}
// withdraw and again mark the proposal; the holder ends or judges it.
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
// adopt: the export proposed whole as the seed, once.
got, err := s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
seed := got.(proposed).Proposal
if !seed.Seed || seed.ExportDigest != delivery.ExportDigest(map[string]delivery.Placement{"anchor": anchor}) || !slices.Equal(asked, []string{exportCall}) ||
len(st[delivery.PlacementCandidatesState].values) != 1 || st[delivery.PlacementProposalsState].values[seed.ID] == nil {
t.Fatalf("adopt = %+v, asked %v", seed, asked)
}
if _, err := s.doAdopt(map[string]any{"why": "again"}, "op"); err == nil || !strings.Contains(err.Error(), "is open") {
t.Errorf("a second adopt with a seed open: %v", err)
}
// withdraw and again mark the proposal; the holder ends or judges it.
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
t.Fatal(err)
}
@@ -97,8 +95,10 @@ func TestTheSeatsVerbs(t *testing.T) {
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
raw, _ := delivery.EncodePlacement(anchor)
st[delivery.PlacementState].values["placement.anchor"] = raw
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
!strings.Contains(err.Error(), "ghost") || len(st[delivery.PlacementProposalsState].values) != 0 {
!strings.Contains(err.Error(), "ghost") || !strings.Contains(err.Error(), "lease write") || len(st[delivery.PlacementProposalsState].values) != 0 {
t.Errorf("a refused proposal: %v", err)
}
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
@@ -146,20 +146,32 @@ func TestTheSeatsVerbs(t *testing.T) {
}
}
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand placement back to.
// hand-back: the operator's alone, written as a hand-back proposal naming every key at its revision, for the assigner
// to ask; nothing is deleted by the verb.
got, err = s.doHandBack(map[string]any{"why": "back"}, "op")
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
t.Errorf("hand-back = %+v, %v", got, err)
}
// Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and
// there is no controller to hand placement back to.
st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}}
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
t.Errorf("a hold: %v", err)
t.Errorf("a hold after the lease write: %v", err)
}
if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") {
t.Errorf("a hand-back: %v", err)
t.Errorf("a hand-back after the lease write: %v", err)
}
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the controller's composition until the lease write, the node's own after; an unplaced node is said", func(t *testing.T) {
w := newWorld()
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
ckey, _ := delivery.ComposedKey("ace", "web", fp)
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedNeither,
Policy: catalogue.PolicyRoll, PolicyFrom: catalogue.FromDefault})
w.putEntry(delivery.ControllerComposedBucket, ckey, cv, 1, time.Now())
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
pkey, _ := delivery.LegacyPlacementKey("ace")
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
@@ -168,11 +180,15 @@ func TestTheSeatsVerbs(t *testing.T) {
hk, _ := delivery.HaveNodeKey(node)
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
}
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("c", 64) || len(w.unplacedNodes()) != 0 {
t.Fatalf("before the lease write: %+v %v, unplaced %v", d, ok, w.unplacedNodes())
}
w.putEntry(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator"}`), 2, time.Now())
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
t.Errorf("ace: %+v %v", d, ok)
t.Errorf("after the lease write, ace: %+v %v", d, ok)
}
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
t.Errorf("g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
t.Errorf("after the lease write, g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
}
findings := map[string][]byte{}
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
@@ -340,7 +356,7 @@ func TestTheSeatsVerbs(t *testing.T) {
func wantsVerbs(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings"} {
for _, state := range []string{modulesState, manifestsState, composedState, policiesState, haveState, putsState, "findings"} {
st[state] = &memKV{values: map[string][]byte{}}
}
manifest := func(name, extra string) {
@@ -355,7 +371,13 @@ func wantsVerbs(t *testing.T) {
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
}
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
composed := func(module, policy string) {
key, _ := delivery.ComposedKey("anchor", module, fpNew)
st[composedState].values[key], _ = json.Marshal(delivery.ComposedDigest{V: delivery.Version, Digest: "sha256:" + strings.Repeat("c", 64),
Judgement: delivery.JudgedNeither, Policy: policy, PolicyFrom: catalogue.FromPerson, PolicyWhy: "chosen at the controller"})
}
composed("web", catalogue.PolicyRecord)
composed("photos", catalogue.PolicyRecord)
var asked []string
var said []string
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
@@ -375,11 +397,10 @@ func wantsVerbs(t *testing.T) {
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
t.Errorf("nats = %+v", p)
}
// compare: the controller writes no policy, so no module splits; each node's own composition is the digest its have names.
// compare: web rolls at the assigner and records at the controller.
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
if got := out.(map[string]any); len(got["splits"].([]assign.Split)) != 0 || got["empty"] != true || got["findings"] == nil ||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
t.Errorf("compare = %+v", got)
if splits := out.(map[string]any)["splits"].([]assign.Split); len(splits) != 1 || splits[0].Module != "web" || out.(map[string]any)["findings"] == nil {
t.Errorf("splits = %+v", splits)
}
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
// large states (the modules, their manifests, the composed record).
@@ -401,14 +422,14 @@ func wantsVerbs(t *testing.T) {
if werr != nil {
t.Fatal(werr)
}
if got := wout.(map[string]any); got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) {
t.Errorf("compare by watch = %+v", got)
if ws := wout.(map[string]any)["splits"].([]assign.Split); len(ws) != 1 || ws[0].Module != "web" {
t.Errorf("splits by watch = %+v", ws)
}
}
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
t.Errorf("the manifests were read by key %d times, want each once", n)
}
for _, state := range []string{modulesState, haveState, health.FindingsState} {
for _, state := range []string{modulesState, composedState, health.FindingsState} {
if watches[state] != 1 {
t.Errorf("%s was watched %d times, want once", state, watches[state])
}
@@ -428,6 +449,9 @@ func wantsVerbs(t *testing.T) {
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil },
Keys: func() ([]string, error) { return []string{"module.gone.policy-split"}, nil }}}
for k, v := range st[composedState].values {
p.w.putEntry(delivery.ControllerComposedBucket, k, v, 1, time.Now())
}
man := map[string]catalogue.Manifest{}
for _, name := range []string{"web", "photos"} {
raw, _, _, _ := st.get(modulesState, name)
@@ -437,10 +461,9 @@ func wantsVerbs(t *testing.T) {
man[name], _ = catalogue.Decode([]byte(kept.JSON))
}
findings["module.gone.policy-split"] = []byte("{}")
// The controller writes no policy now (ADR 0571 §6): the process compares with nothing, and deletes a finding left.
p.policySplits(man, nil)
if len(findings) != 0 {
t.Errorf("findings = %v, want a stale one deleted", findings)
if _, ok := findings[assign.PolicySplitKey("web")]; !ok || len(findings) != 1 {
t.Errorf("findings = %v, want web's split alone (a stale one deleted)", findings)
}
// a person's record for web: written, recorded, said; the split is gone.
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
@@ -9,6 +9,7 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
@@ -19,14 +20,15 @@ import (
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
// own have names (the controller writes no policy any more, so no module's policy differs); `bus` says what a bus upgrade would do. The assigner's process reads
// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from
// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads
// `policies` when it judges a move and writes the finding policy-split while a split stands.
// The states the verbs read, as the runtime names them.
const (
policiesState = delivery.AssignerPoliciesState
manifestsState = delivery.IndexerModule + ".manifests"
composedState = "mesh-controller.composed"
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
PolicyChangedEvent = "policy-changed"
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
@@ -54,6 +56,13 @@ func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error),
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
}
// leased says the controller's lease is handed away.
func (s *wantsTools) leased() bool {
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
return away
}
// values reads every key of a state: from the watched copy when there is one, else key by key.
func (s *wantsTools) values(state string) (map[string][]byte, error) {
if s.watched != nil {
@@ -145,23 +154,15 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
chosen, unreadable := assign.DecodePolicies(raw)
if arg(args, "compare") == "true" {
haves, err := s.values(haveState)
composed, err := s.values(composedState)
if err != nil {
return nil, err
}
composed := map[string]string{}
for key, raw := range haves {
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
composed[node] = h.Composed
}
}
}
splits := assign.Splits(manifests, chosen, nil) // the controller writes no policy to differ from (ADR 0571 §6)
splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed))
if splits == nil {
splits = []assign.Split{}
}
out := map[string]any{"splits": splits, "empty": len(splits) == 0, "composed": composed}
out := map[string]any{"splits": splits, "empty": len(splits) == 0}
// The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15).
if found, err := s.values(health.FindingsState); err != nil {
out["findings-unread"] = err.Error()
@@ -228,7 +229,7 @@ func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[str
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
}
}
rec := handacts.Recorder{Ask: s.ask}
rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased}
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
recorded := taken.Log
if rerr != nil {
@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
@@ -18,6 +19,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
)
@@ -26,6 +28,18 @@ import (
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
const noModuleWords = delivery.CandidateNoneNoModule
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
// internal/view/view.go): an assignment stands while it has not ended and has no state.
type assignment struct {
Node string `json:"node"`
Module string `json:"module"`
Build string `json:"build"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
// digest. The watches write it; a pass reads it.
type world struct {
@@ -196,15 +210,38 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
return p
}
// assigned is each node's modules the seat mesh-placement's placement names, in name order (ADR 0549 §5, ADR 0567 §8): the
// placement is the one source of where modules run. The controller's assignments are read no more (issue 774).
func (w *world) assigned() map[string][]string {
placed, _ := w.placed()
out := make(map[string][]string, len(placed))
for node, modules := range placed {
out[node] = slices.Sorted(slices.Values(modules))
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
func (w *world) assigned() (map[string][]string, map[string]assignment) {
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
// the lease write's own check refuses that order (ADR 0547 §11).
placed, seeded := w.placed()
byAssignment, all := w.assignedByController()
if away := w.handedAway(); away && seeded {
both, _ := assign.Placed(byAssignment, placed, seeded, away)
return both, map[string]assignment{}
}
return out
both, _ := assign.Placed(byAssignment, placed, seeded, false)
return both, all
}
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
// the controller's lease is handed away, since the placement is then the one source.
func (w *world) placementDiffers() []assign.Pair {
placed, seeded := w.placed()
byAssignment, _ := w.assignedByController()
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
return differ
}
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
func (w *world) handedAway() bool {
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
return away
}
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
@@ -234,6 +271,24 @@ func (w *world) placedNodes() map[string]bool {
return out
}
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
// name order.
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
out, all := map[string][]string{}, map[string]assignment{}
for _, value := range w.read(provider.AssignmentsBucket) {
var a assignment
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
continue
}
out[a.Node] = append(out[a.Node], a.Module)
all[a.Node+"."+a.Module] = a
}
for _, mods := range out {
sort.Strings(mods)
}
return out, all
}
// trunk is every module the indexer holds at the trunk, with its value.
func (w *world) trunk() map[string]delivery.Module {
out := map[string]delivery.Module{}
@@ -252,7 +307,7 @@ func (w *world) trunk() map[string]delivery.Module {
// mesh is the mesh as the trunk holds it, for the composition check.
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
m.Assigned = w.assigned()
m.Assigned, _ = w.assigned()
for name, mod := range w.trunk() {
if mod.Manifest == "" {
continue
@@ -557,14 +612,49 @@ func (w *world) built(module, fp string) bool {
return err == nil && b.State == delivery.BuildDone
}
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
// controller's copy (mesh-controller_composed) is read no more (issue 774).
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
//
// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a
// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets
// no want (and the pass says so, unplacedFindings).
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
if !w.placedNodes()[node] {
if w.handedAway() {
if !w.placedNodes()[node] {
return delivery.ComposedDigest{}, false
}
return w.ownComposed(node)
}
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
return delivery.ComposedDigest{}, false
}
return w.ownComposed(node)
w.mu.Lock()
value, ok := w.values[delivery.ControllerComposedBucket][key]
w.mu.Unlock()
if !ok {
return delivery.ComposedDigest{}, false
}
d, err := delivery.DecodeComposedDigest(value)
return d, err == nil
}
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
out := map[string]delivery.ComposedDigest{}
if w.handedAway() {
return out // the controller composes no node from the lease write (ADR 0571 §6)
}
for key, value := range w.read(delivery.ControllerComposedBucket) {
node, _, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || !isNode {
continue
}
if d, err := delivery.DecodeComposedDigest(value); err == nil {
out[node] = d
}
}
return out
}
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
@@ -629,7 +719,7 @@ func (w *world) runningCommits(module string) []string {
var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads. A node whose have names none is not composed yet.
// what rule 7 reads from the lease write. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
key, err := delivery.HaveNodeKey(node)
if err != nil {
@@ -647,9 +737,12 @@ func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
Words: "the node composes its own"}, true
}
// unplacedNodes are the nodes whose engine has a have and that no placement names: they get no
// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no
// want, since nothing composes them.
func (w *world) unplacedNodes() []string {
if !w.handedAway() {
return nil
}
placed := w.placedNodes()
var out []string
for key := range w.read(delivery.HaveBucket) {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+20 -4
View File
@@ -1,7 +1,23 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364 h1:UUDPWfUjyerptar+wPwI891jTUAtbbuXori0j3phTE4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3 h1:lb6hz+SBsoCAzsiCCvdtPKyE66Y5Ng/DBAkonAexPzA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -117,6 +117,7 @@ func saidConditionsAreFindings(t *testing.T) {
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
{PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"},
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
} {
@@ -1,12 +1,122 @@
package assign
import (
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
// the seat mesh-placement's placement (ADR 0549 §5, ADR 0567 §8). The assigner writes wants only for a pair both name:
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease
// handed away) the placement is the one source.
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
// OnlyPlacement.
type Pair struct {
Node, Module, Only string
// Call is what became of the assigner's call of the controller's verb for the pair, in words, when it made one.
Call string
}
// The source that names a pair alone.
const (
OnlyAssignments = "the controller's assignments"
OnlyPlacement = "the seat mesh-placement's placement"
)
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's
// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the
// assignments are then a source of nothing.
func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) {
if handedAway && seeded {
out := make(map[string][]string, len(placed))
for node, modules := range placed {
out[node] = slices.Sorted(slices.Values(modules))
}
return out, nil
}
if !seeded {
return assigned, nil
}
set := func(m map[string][]string) map[string]bool {
out := map[string]bool{}
for node, modules := range m {
for _, module := range modules {
out[node+"\x00"+module] = true
}
}
return out
}
inAssigned, inPlaced := set(assigned), set(placed)
both := map[string][]string{}
var differ []Pair
for node, modules := range assigned {
for _, module := range modules {
if inPlaced[node+"\x00"+module] {
both[node] = append(both[node], module)
} else {
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
}
}
}
for node, modules := range placed {
for _, module := range modules {
if !inAssigned[node+"\x00"+module] {
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
}
}
}
for _, modules := range both {
sort.Strings(modules)
}
sort.Slice(differ, func(i, j int) bool {
if differ[i].Node != differ[j].Node {
return differ[i].Node < differ[j].Node
}
return differ[i].Module < differ[j].Module
})
return both, differ
}
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
// both name it or neither does.
func PlacementCondition(p Pair, raised bool) asks.Condition {
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("the controller's assignments and the seat's placement agree on %s on %s.", p.Module, p.Node)}
}
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
var needs string
if p.Only == OnlyAssignments {
needs = fmt.Sprintf("propose placing %s on %s (mesh-placement.propose), or unassign it at the controller", p.Module, p.Node)
} else {
needs = fmt.Sprintf("assign %s to %s at the controller, as the approved placement says, or propose taking it off", p.Module, p.Node)
}
if p.Call != "" {
needs += "; the assigner's call: " + p.Call
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
}
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
// §1): raised naming each, cleared when none is left.
func SeatWaitsCondition(unbound []string) asks.Condition {
@@ -48,6 +48,26 @@ func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (p
return catalogue.Upgrade(m, c)
}
// ControllerPolicies is, by module, every policy the controller writes beside a composition's digest of it.
func ControllerPolicies(composed map[string][]byte) map[string][]string {
out := map[string][]string{}
for key, raw := range composed {
_, module, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || isNode {
continue
}
d, err := delivery.DecodeComposedDigest(raw)
if err != nil || d.Policy == "" {
continue
}
if !contains(out[module], d.Policy) {
out[module] = append(out[module], d.Policy)
sort.Strings(out[module])
}
}
return out
}
func contains(list []string, s string) bool {
for _, v := range list {
if v == s {
@@ -68,7 +68,7 @@ func StepPut(w PutWorld) PutNext {
}
fp := short(p.FP)
if !w.Assigned {
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
}
switch w.Build {
case delivery.BuildDone:
@@ -1,7 +1,9 @@
package assign
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
@@ -17,23 +19,30 @@ import (
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
// digest.
type ProposeIn struct {
ID, By, Why string
At time.Time
Changes []delivery.PlacementProposalChange
Current map[string]delivery.Placement
Revisions map[string]uint64
Held func(module string) bool
Recorded func(node string) bool
ID, By, Why string
At time.Time
Changes []delivery.PlacementProposalChange
Current map[string]delivery.Placement
Revisions map[string]uint64
Held func(module string) bool
Recorded func(node string) bool
Seed bool
ExportDigest string
// Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk
// buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then
// the holder keeps the controller in step and refuses what its verbs cannot take.
Leased bool
}
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change the
// controller's verbs cannot take before the lease write (§8), a change that does not fit its node, and what
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
Changes: in.Changes, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
Changes: in.Changes, Seed: in.Seed, ExportDigest: in.ExportDigest, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
var findings []string
if p.By == "" {
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
@@ -45,8 +54,14 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla
findings = append(findings, "a proposal changes something: it names no change")
}
for i, c := range in.Changes {
if c.Op == delivery.OpSeed {
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
if (c.Op == delivery.OpSeed) != in.Seed {
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
continue
}
if !in.Leased {
if why := c.BeforeLease(); why != "" {
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
}
}
}
laid, lay := delivery.LayChanges(in.Current, in.Changes)
@@ -103,19 +118,112 @@ func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) (
return out, nil
}
// The kinds of the condition (and finding) of a refused controller call, which the assigner raised until it stopped calling
// the controller's verbs (ADR 0567 §8, issue 774).
// ControllerCall is one call of the controller's seat verbs the holder makes after writing an approved proposal, until
// the lease write (§8): assign, unassign, pin or unpin, with its arguments.
type ControllerCall struct {
Verb string
Args map[string]any
// Node and Module name the pair the call keeps in step (a pin names no module).
Node, Module string
}
// Key names the call, as the condition of its pair says it.
func (c ControllerCall) Key() string {
return fmt.Sprintf("mesh-controller.%s %s %s", c.Verb, c.Node, c.Module)
}
// ControllerCalls is every call that brings the controller's assignments and pins to the placement after: assign for a
// module placed on a node before not, unassign for one taken off, pin for a pin set or changed, unpin for one removed. In
// node order, unassigns before assigns, so a module moved between nodes is first taken off.
//
// Each call carries only the arguments the controller's seat contract declares for its verb (catalogue.ControllerVerbs):
// the controller refuses any other argument and does nothing (issue 774: a "why" added to every call had every approved
// placement refused since 2026-10-28). The proposal and its warrant are in the assigner's own record instead: the
// placement-changed event and the hand-act log.
func ControllerCalls(before map[string]delivery.Placement, after map[string]*delivery.Placement) []ControllerCall {
nodes := make([]string, 0, len(after))
for node := range after {
nodes = append(nodes, node)
}
sort.Strings(nodes)
var off, on, pins []ControllerCall
for _, node := range nodes {
was, now := before[node], delivery.Placement{}
if a := after[node]; a != nil {
now = *a
}
had, has := map[string]bool{}, map[string]bool{}
for _, m := range was.Modules {
had[m.Module] = true
}
for _, m := range now.Modules {
has[m.Module] = true
if !had[m.Module] {
on = append(on, ControllerCall{Verb: "assign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
}
}
for _, m := range was.Modules {
if !has[m.Module] {
off = append(off, ControllerCall{Verb: "unassign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
}
}
for _, provision := range sortedKeys(now.Pins) {
to := now.Pins[provision]
if old, ok := was.Pins[provision]; !ok || old != to {
pins = append(pins, ControllerCall{Verb: "pin", Node: node, Module: provision,
Args: map[string]any{"node": node, "provision": provision, "from": to.Node, "module": to.Module}})
}
}
for _, provision := range sortedKeys(was.Pins) {
if _, ok := now.Pins[provision]; !ok {
pins = append(pins, ControllerCall{Verb: "unpin", Node: node, Module: provision, Args: map[string]any{"node": node, "provision": provision}})
}
}
}
return append(append(off, on...), pins...)
}
// CallKind is the kind of the condition of a refused call: one for a module's place (assign, unassign), one for a pin.
func (c ControllerCall) CallKind() string {
if c.Verb == "pin" || c.Verb == "unpin" {
return KindControllerPin
}
return KindControllerCall
}
// The kinds of a refused controller call's condition (and finding).
const (
KindControllerCall = "controller-call"
KindControllerPin = "controller-pin"
)
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
// module, or node and provision for a pin).
func CallClearedCondition(kind, node, module string) asks.Condition {
return asks.Condition{Key: delivery.AssignerModule + "." + kind + "." + node + "." + module,
Subject: asks.Scope{Scope: "module", ID: module, Machine: node}, State: asks.StateCleared,
Resolved: "the assigner calls no verb of the controller any more: placement is the one source."}
// CallConditionKey is the key of the condition of a call refused for its pair (node and module, or node and provision).
func CallConditionKey(c ControllerCall) string {
return delivery.AssignerModule + "." + c.CallKind() + "." + c.Node + "." + c.Module
}
// CallRefusedCondition is the condition the assigner raises when the controller refused (or did not answer) a call that
// keeps its assignments in step with an approved placement (ADR 0567 §8, issue 774): the placement is written, the
// controller's assignments are not, so nothing new reaches the node (no bus account, no composition). It names the verb,
// the node, the module (or provision) and the controller's words, keyed by the pair, and is cleared (raised false) when a
// later call for the pair is answered, or both sources agree on it, with resolved saying which.
func CallRefusedCondition(c ControllerCall, refusal string, raised bool, resolved string) asks.Condition {
key := CallConditionKey(c)
subject := asks.Scope{Scope: "module", ID: c.Module, Machine: c.Node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: resolved}
}
headline := fmt.Sprintf("the controller refused %s %s on %s", c.Verb, c.Module, c.Node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
needs := fmt.Sprintf("call mesh-controller.%s for %s on %s by hand once the refusal's cause is fixed", c.Verb, c.Module, c.Node)
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. An approved placement was written, and the assigner's call %s to keep the controller's "+
"assignments in step was refused: %s. Until the controller holds it, the placement and the assignments differ, so "+
"the assigner writes no want for the pair and nothing new reaches %s. This clears when a call for it is answered "+
"or both sources agree.", asks.NeedsYou, needs, c.Key(), refusal, c.Node)}
}
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
@@ -138,6 +246,9 @@ func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking
}
nodes := p.Nodes()
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
if p.Seed {
headline = "Take placement over from the controller?"
}
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
}
@@ -157,6 +268,57 @@ func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking
}, Asking: asking}, nil
}
// ExportOf reads the controller's answer to placement-export (§7): its verb answers what the command printed, the
// JSON of every node's placement, in `output` (the controller's verbAnswer; `answer` when it carries it as data). A
// refused or unreadable answer is an error naming it.
func ExportOf(raw []byte) (map[string]delivery.Placement, error) {
var a struct {
Output string `json:"output"`
OK *bool `json:"ok"`
Answer json.RawMessage `json:"answer"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the controller's placement-export answered no verb answer: %w", err)
}
if a.OK != nil && !*a.OK {
return nil, fmt.Errorf("the controller's placement-export refused: %s", strings.TrimSpace(a.Output))
}
var byNode map[string]json.RawMessage
if len(a.Answer) > 0 && string(a.Answer) != "null" {
if err := json.Unmarshal(a.Answer, &byNode); err != nil {
return nil, fmt.Errorf("the controller's placement-export is no placement by node: %w", err)
}
} else if err := json.NewDecoder(strings.NewReader(a.Output)).Decode(&byNode); err != nil {
// The command prints its JSON first; what its standard error said after it is left.
return nil, fmt.Errorf("the controller's placement-export printed no placement by node: %w", err)
}
out := map[string]delivery.Placement{}
for node, rawP := range byNode {
p, err := delivery.DecodePlacement(rawP)
if err != nil {
return nil, fmt.Errorf("the controller's placement-export of %s cannot be read: %w", node, err)
}
if p.Node != node {
return nil, fmt.Errorf("the controller's placement-export names %s under %s", p.Node, node)
}
out[node] = p
}
if len(out) == 0 {
return nil, fmt.Errorf("the controller's placement-export names no node")
}
return out, nil
}
// SeedChanges is the seed's changes: every node's placement of the export, whole, in node order.
func SeedChanges(export map[string]delivery.Placement) []delivery.PlacementProposalChange {
var out []delivery.PlacementProposalChange
for _, node := range sortedKeys(export) {
p := export[node]
out = append(out, delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: node, Whole: &p})
}
return out
}
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
@@ -8,6 +8,7 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -37,30 +38,39 @@ func TestThePlacementSeatDecides(t *testing.T) {
in ProposeIn
want string
}{
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
"a port before the lease": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}}), "lease write"},
"a holding": {in(delivery.PlacementProposalChange{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "mesh-store", Scope: "mesh"}), "lease write"},
"a binding": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}), "lease write"},
"a config place": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}), "lease write"},
"a config access": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}}), "lease write"},
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "adopt's alone"},
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
} {
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
}
}
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
// A module's config values may be proposed before the lease write (§13); once the lease is written every change is
// the state's alone, and none waits.
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
t.Errorf("config values before the lease write: %v", f)
}
for _, c := range []delivery.PlacementProposalChange{
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
} {
if _, _, f := Propose(in(c)); len(f) != 0 {
t.Errorf("%s is refused: %v", c.Op, f)
leased := in(c)
leased.Leased = true
if _, _, f := Propose(leased); len(f) != 0 {
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
}
}
@@ -82,10 +92,32 @@ func TestThePlacementSeatDecides(t *testing.T) {
}
}
// Issue 1126: the seed places the private network, which no repository holds; it is the mesh's own, and passes.
wired := laptop
wired.Modules = append(slices.Clone(laptop.Modules), delivery.PlacedModule{Module: "mesh-wireguard"})
seed := in(SeedChanges(map[string]delivery.Placement{"laptop": wired})...)
seed.Seed, seed.ExportDigest, seed.Current, seed.Revisions = true, "sha256:x", map[string]delivery.Placement{}, map[string]uint64{}
seed.Held = func(m string) bool { return m != "mesh-wireguard" } // the indexer holds every module but it
if _, _, f := Propose(seed); len(f) != 0 {
t.Errorf("a seed placing the private network is refused: %v", f)
}
writes, err := Writes(p, laid)
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
t.Fatalf("writes = %+v, %v", writes, err)
}
pinned := *laid["laptop"]
pinned.Pins = map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
calls := ControllerCalls(map[string]delivery.Placement{"anchor": anchor, "laptop": laptop},
map[string]*delivery.Placement{"anchor": laid["anchor"], "laptop": &pinned})
var said []string
for _, c := range calls {
said = append(said, c.Key())
}
if !slices.Equal(said, []string{"mesh-controller.unassign anchor web", "mesh-controller.assign laptop web", "mesh-controller.pin laptop db"}) {
t.Errorf("the controller's calls = %v", said)
}
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
t.Fatalf("the ask = %+v, %v", a, err)
@@ -95,4 +127,112 @@ func TestThePlacementSeatDecides(t *testing.T) {
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
t.Error("the ask's yes binds changes it was not asked about")
}
export, _ := json.Marshal(map[string]any{"anchor": anchor, "laptop": laptop})
for name, c := range map[string]struct {
raw string
want int
}{
"in output": {`{"ok":true,"output":` + quote(string(export)+"\n") + `}`, 2},
"as answer": {`{"ok":true,"output":"","answer":` + string(export) + `}`, 2},
"refused": {`{"ok":false,"output":"no store"}`, 0},
"no node": {`{"ok":true,"output":"{}"}`, 0},
"no version": {`{"ok":true,"output":"{\"anchor\":{\"node\":\"anchor\",\"modules\":[]}}"}`, 0},
} {
got, err := ExportOf([]byte(c.raw))
if len(got) != c.want || (c.want == 0) != (err != nil) {
t.Errorf("%s: %d nodes, %v", name, len(got), err)
}
}
if seed := SeedChanges(map[string]delivery.Placement{"laptop": laptop, "anchor": anchor}); len(seed) != 2 || seed[0].Node != "anchor" || seed[0].Op != delivery.OpSeed {
t.Errorf("the seed's changes = %+v", seed)
}
}
func quote(s string) string {
raw, _ := json.Marshal(s)
return string(raw)
}
// Issue 774: every call the assigner makes of the controller carries only the arguments the controller's seat contract
// declares for its verb, and every one it requires; the controller refuses any other argument and does nothing. A "why"
// added to every call (2026-10-28) had each approved placement refused.
func TestControllerCallsFollowTheContract(t *testing.T) {
declared := map[string]map[string]any{}
required := map[string][]string{}
for _, v := range catalogue.ControllerVerbs {
props, _ := v.Input["properties"].(map[string]any)
declared[v.Name] = props
switch r := v.Input["required"].(type) {
case []string:
required[v.Name] = r
}
}
db := map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
moved := map[string]delivery.Provider{"db": {Node: "laptop", Module: "store"}}
placed := func(node string, pins map[string]delivery.Provider, modules ...string) delivery.Placement {
p := delivery.Placement{Node: node, Pins: pins}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
}
return p
}
ptr := func(p delivery.Placement) *delivery.Placement { return &p }
for name, c := range map[string]struct {
before map[string]delivery.Placement
after map[string]*delivery.Placement
verbs []string
}{
"assign": {map[string]delivery.Placement{}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil, "web"))}, []string{"assign"}},
"unassign": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil))}, []string{"unassign"}},
"key deleted": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": nil}, []string{"unassign", "unpin"}},
"pin set": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", db, "web"))}, []string{"pin"}},
"pin changed": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", moved, "web"))}, []string{"pin"}},
"moved and pinned": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil)), "laptop": ptr(placed("laptop", db, "web"))}, []string{"unassign", "assign", "pin"}},
} {
calls := ControllerCalls(c.before, c.after)
var verbs []string
for _, call := range calls {
verbs = append(verbs, call.Verb)
props, ok := declared[call.Verb]
if !ok {
t.Errorf("%s: the controller's seat has no verb %q", name, call.Verb)
continue
}
for arg := range call.Args {
if _, ok := props[arg]; !ok {
t.Errorf("%s: %s gives %q, which mesh-controller.%s does not take, so the controller refuses it", name, call.Key(), arg, call.Verb)
}
}
for _, arg := range required[call.Verb] {
if _, ok := call.Args[arg]; !ok {
t.Errorf("%s: %s lacks %q, which mesh-controller.%s requires", name, call.Key(), arg, call.Verb)
}
}
}
if !slices.Equal(verbs, c.verbs) {
t.Errorf("%s: the calls = %v, want %v", name, verbs, c.verbs)
}
}
}
// Issue 774: a refused call is a condition keyed by its pair, a module's place apart from a pin, naming the verb, the node, the
// module and the refusal; its clear says why.
func TestCallRefusedCondition(t *testing.T) {
assign := ControllerCall{Verb: "assign", Node: "anchor", Module: "web"}
pin := ControllerCall{Verb: "pin", Node: "anchor", Module: "web"}
c := CallRefusedCondition(assign, `assign takes no argument "why"`, true, "")
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
!strings.Contains(c.Headline, "assign web on anchor") || !strings.Contains(c.Explanation, `takes no argument "why"`) {
t.Errorf("the raised condition = %+v", c)
}
if CallConditionKey(pin) == c.Key {
t.Error("a pin's condition shares the key of a module's place")
}
if cl := CallRefusedCondition(assign, "", false, "answered"); cl.State != asks.StateCleared || cl.Key != c.Key || cl.Resolved != "answered" {
t.Errorf("the clear = %+v", cl)
}
if _, _, raised, err := FindingOf(c, time.Unix(0, 0)); err != nil || !raised {
t.Errorf("the condition is no finding: %v", err)
}
}
@@ -39,8 +39,8 @@ type Module struct {
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
Chosen *catalogue.PersonPolicy
Nodes []string
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
@@ -131,7 +131,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
short(m.FP))
}
if _, ok := m.composition(node, m.FP); !ok {
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
}
if words, waiting := m.provisionWaiting(node); waiting {
return false, delivery.RuleProvisions, words
@@ -412,8 +412,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
next.Newest, next.Rule, next.Words = m.FP, rule, words
at.FP = target
}
// A placed node's want names the build alone: it is the same target while the build is.
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
@@ -535,7 +534,7 @@ func (m Module) selfDerived(node string, before delivery.Want, had bool, next de
d, there := m.composition(node, next.FP)
if !there || d.Digest == "" {
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
"and the controller has not composed it at %s, so its want stays", m.Name, node, short(next.FP))}
return next, false
}
at := m
@@ -815,13 +815,6 @@ func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
t.Errorf("the answer was erased: %+v", w)
}
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
m.Self = func(string) bool { return true }
placed := map[string]delivery.Want{"ace": {FP: fNew, Self: true, Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
t.Errorf("the answer in a placed node's want was erased: %+v", w)
}
}
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
@@ -33,7 +33,7 @@ func TestManifest(t *testing.T) {
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved",
want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.composed", "mesh-controller.lease",
"mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
@@ -82,7 +82,8 @@ func TestManifest(t *testing.T) {
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
}
if want := []string{
if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.pin",
"seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin",
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
"seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
@@ -30,6 +30,9 @@
"mesh-conditions.takeover",
"mesh-config.kept",
"mesh-config.resolved",
"mesh-controller.assignments",
"mesh-controller.composed",
"mesh-controller.lease",
"mesh-delivery-builder.build-log",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.candidates",
@@ -46,6 +49,11 @@
"operator-channel"
],
"invokes": [
"seat:mesh-controller.assign",
"seat:mesh-controller.pin",
"seat:mesh-controller.placement-export",
"seat:mesh-controller.unassign",
"seat:mesh-controller.unpin",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.hand-acts",
"seat:mesh-hand-acts.warranted",
@@ -12,7 +12,6 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -20,7 +19,7 @@ import (
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
// controller had never taken in, and the builder kept them done.
// heldVerb answers the seat mesh-artifact-sweep's verb held on the bus, as its holder does: {result: {output, ok,
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
// answer}}; a reference is gone while gone says so.
type heldVerb struct {
mu sync.Mutex
@@ -30,7 +29,7 @@ type heldVerb struct {
func (h *heldVerb) serve(t *testing.T, b *bed) {
t.Helper()
sub, err := b.js.Conn().Subscribe("mesh.seat."+catalogue.ArtifactSweepSeat+".call.held.>", func(m *nats.Msg) {
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
var args struct {
References []string `json:"references"`
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -6,7 +6,6 @@ import (
"fmt"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"time"
@@ -178,42 +177,6 @@ func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
}
}
// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the
// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined
// ask again, so a lone node off the agreed toolchain cannot spin on it.
const (
DefaultDeclineBound = 30 * time.Minute
DefaultDeclinePause = 5 * time.Second
)
func (b *Builder) declineBound() time.Duration {
if b.DeclineBound > 0 {
return b.DeclineBound
}
return DefaultDeclineBound
}
func (b *Builder) declinePause() time.Duration {
if b.DeclinePause != 0 {
return max(b.DeclinePause, 0)
}
return DefaultDeclinePause
}
// declinedWords names the nodes that declined an ask and why, in the order of their names.
func declinedWords(by map[string]string) string {
nodes := make([]string, 0, len(by))
for n := range by {
nodes = append(nodes, n)
}
sort.Strings(nodes)
var parts []string
for _, n := range nodes {
parts = append(parts, n+": "+by[n])
}
return strings.Join(parts, "; ")
}
func (b *Builder) askings() int {
if b.Askings > 0 {
return b.Askings
@@ -292,8 +255,6 @@ func (b *Builder) unlisten(key, language string, c chan string) {
var (
errCheck = errors.New("the check bound passed")
errAgain = errors.New("asked again")
// errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2).
errDeclined = errors.New("declined")
)
type byHand struct{ why string }
@@ -307,42 +268,11 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask
again := b.listen(key, ask.Language)
defer b.unlisten(key, ask.Language, again)
asking := askingOf(asked.step, l.module)
var firstDecline time.Time
declines := map[string]string{} // node -> why it declined, the last
for {
step, why, err := b.waitOnce(ctx, ask, again)
switch {
case errors.Is(err, errCheck):
continue
case errors.Is(err, errDeclined):
// Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once.
// Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the
// checker's decline loop is, issue 890).
now := time.Now()
if firstDecline.IsZero() {
firstDecline = now
}
declines[step.Node] = why
if bound := b.declineBound(); now.Sub(firstDecline) > bound {
return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines))
}
// The pause is a context's bound, not a timer (ADR 0437 §8).
pause, stop := context.WithTimeout(ctx, b.declinePause())
<-pause.Done()
stop()
if ctx.Err() != nil {
return step, ask, ctx.Err()
}
old := ask.ID
ask.ID, ask.At = buildstep.NewID(), time.Now()
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil {
return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err)
}
if err := b.Asks.Put(ctx, ask); err != nil {
return step, ask, err
}
b.drop(ctx, ask, old)
b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why)
case errors.Is(err, errAgain):
if asking >= b.askings() {
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
@@ -387,11 +317,6 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
defer stop()
}
step, err := b.Asks.Wait(wait, ask)
if _, declined := delivery.DeclinedOf(step); err == nil && declined {
// A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again
// after a pause, within DeclineBound.
return step, step.Reason, errDeclined
}
if err == nil && buildrun.IsLost(step) {
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
// words of why, not at the check bound.
@@ -67,9 +67,6 @@ type Builder struct {
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
// with why; DefaultAskings when 0.
Askings int
// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
DeclineBound, DeclinePause time.Duration
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
Said func(AskedAgain)
// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
@@ -418,7 +415,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
if planErr == nil && !nothing {
if why := b.stale(m, plan); why != "" {
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
b.logf("%s at %s is not built: %s", module, m.FP2, why)
return
}
@@ -509,11 +506,10 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
continue // an archive is packed with no toolchain
}
language := buildrun.LanguageOf[bm]
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
if ag.Wait != "" {
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
if none != "" {
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
}
tc := ag.Toolchain
want, ok := m.Toolchains[language]
if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
@@ -547,8 +543,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
repository := delivery.RepositoryName(m.Repository)
var lanes []*lane
used := map[string]delivery.ToolchainUsed{}
var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
var kept []string // build modules whose artifacts stay as the prior done key has them
var kept []string // build modules whose artifacts stay as the prior done key has them
for _, bm := range plan.Modules() {
if prior != nil && len(only) > 0 && !only[bm] {
kept = append(kept, bm)
@@ -558,12 +553,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
language := buildrun.LanguageOf[bm]
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
if ag.Wait != "" {
return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
if none != "" {
return failed("the %s toolchain is not agreed: %s", language, none)
}
tc := ag.Toolchain
sitting = append(sitting, SittingOutWords(language, ag)...)
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
}
@@ -660,9 +653,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
}
}
step := "built " + strings.Join(steps, "; ")
if len(sitting) > 0 {
step += " (" + strings.Join(sitting, "; ") + ")"
}
if len(used) == 0 {
used = nil
}
@@ -952,13 +942,3 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
return e
}
// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
func SittingOutWords(language string, a delivery.Agreement) []string {
var out []string
for _, s := range a.SittingOut {
out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
}
return out
}
@@ -223,14 +223,6 @@ var (
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
r := newRig(t, goBundle, archiveA, tsBundle)
// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false)
defer func() {
if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") {
t.Errorf("the key does not say ace sits out: %s", st)
}
}()
var logged []delivery.BuildLogEntry
var lmu sync.Mutex
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
@@ -738,25 +730,6 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
}
})
t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.DeclinePause, r.b.DeclineBound = -1, 150*time.Millisecond
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}})
waitForAsks(t, r, 1)
for ended, deadline := 0, time.Now().Add(5*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn
r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined})
ended++
}
}
r.b.Wait()
v := valueOf(t, r.st, r.key)
if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") {
t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings)
}
})
for _, c := range []struct {
name string
askings int
@@ -36,7 +36,6 @@ func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerd
// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not
// "not yet run": a pass names its machine, a fail fails the check with the assigner's words.
func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) {
t.Run("the read awaits the assigner's composition", checkerAwaitsTheAssignersComposition)
for _, c := range []struct {
state, owed, words string
}{
@@ -21,8 +21,7 @@ import (
// 1297/AC3 (its bus half): on the real bus, the checker's read of the assigner's composition waits while the key is
// absent and while the assigner says it waits, and answers the verdict the assigner then writes under the check's key.
// A case of TestTheCheckersLineSaysTheAssignersComposition (merged into it for the test budget, issue 774).
func checkerAwaitsTheAssignersComposition(t *testing.T) {
func TestTheCheckerAwaitsTheAssignersComposition(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "comp")
if err != nil {
t.Fatal(err)
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+14 -2
View File
@@ -1,5 +1,17 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -342,11 +342,6 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
}
for _, a := range asks { // ADR 0584: every check ask names the agreed toolchain, so a node off it declines
if a.Check.Toolchain == nil || a.Check.Toolchain.Version != "go1.27.1" || a.Check.Toolchain.Recipe != "7" {
t.Errorf("a check ask names toolchain %+v, want go1.27.1 recipe 7", a.Check.Toolchain)
}
}
for _, a := range asks[1:] {
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
@@ -166,9 +166,7 @@ type target struct {
module string
node string
toolchain string
// agreed is the toolchain the ask names, so a node off it declines the check (ADR 0584); nil when none is agreed.
agreed *delivery.ToolchainUsed
fetched Fetched
fetched Fetched
// fetchArgs are the arguments of the fetch its check ask runs.
fetchArgs map[string]any
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
@@ -376,7 +374,6 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
t := &target{pull: p, module: bm}
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
t.toolchain = tc.Version
t.agreed = &delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
}
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
@@ -431,7 +428,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
p := t.pull
placed := len(needs) > 0
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages, Toolchain: t.agreed}
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
if st.j.Nodes != nil {
names, err := st.j.Nodes.Names(ctx)
if err == nil && len(names) == 0 {
@@ -498,12 +495,8 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
bound = DefaultDeclineBound
}
if now.Sub(firstDecline) > bound {
lacks := strings.Join(needs, ", ")
if lacks == "" {
lacks = strings.Join(d.Lacks, ", ") // a node off the agreed toolchain declined it (ADR 0584)
}
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
p.Repository, p.Number, lacks, bound, step.Node))
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
}
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
a.ID, a.At = buildstep.NewID(), now.UTC()
@@ -423,9 +423,7 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}},
// The mesh packages its go.sum names (ADR 0573 §2, issue 1585): this module's own go.sum names mesh-sdk.
Packages: map[string][]string{"modules/a": {"git.novox.be/novox/mesh-sdk/go"}}}
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+10 -2
View File
@@ -1,5 +1,13 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0 h1:q6ywchpTM9fwaJ1UNwDSuOjb5FYDFcsaeiTrKoB4scc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+14 -4
View File
@@ -1,7 +1,17 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -50,37 +50,23 @@ const Env = "MESH_CHECK_BESIDE"
// CapturedSuffix is what a captured file's name carries that the repository's own does not.
const CapturedSuffix = ".captured"
// notPlaced are the repositories the checker places beside no mesh-delivery check (besideRules in
// mesh-delivery-checker/internal/check/beside.go places mesh-sdk and the catalogue): a test reads them from the captured
// copy even in a merge check, and says so (issue 1585). Every other sibling missing from a merge check fails the test.
var notPlaced = map[string]bool{"mesh-host": true}
// Dir is the named repository a test reads — the clone beside a merge check, or the captured copy — and
// says which in the test's log.
func Dir(t testing.TB, repository string) string {
t.Helper()
root := os.Getenv(Env)
if root != "" {
if root := os.Getenv(Env); root != "" {
dir := filepath.Join(root, repository)
_, err := os.Stat(dir)
switch {
case err == nil:
t.Logf("judged against %s as cloned beside this check", dir)
return dir
case !notPlaced[repository]:
// A sibling the checker places that is missing is the checker's fault, never judged against a stale copy.
if _, err := os.Stat(dir); err != nil {
t.Fatalf("%s is not beside this check in %s=%s, so its agreement with this repository cannot be "+
"judged here: %v", repository, Env, root, err)
}
t.Logf("judged against %s as cloned beside this check", dir)
return dir
}
from := filepath.Join(Captured(), repository)
if _, err := os.Stat(from); err != nil {
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
}
if root != "" {
t.Logf("%s is not beside this check in %s=%s (the checker places it beside no mesh-delivery check): read from "+
"its captured copy", repository, Env, root)
}
dir, err := layOut(repository, from)
if err != nil {
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
@@ -5013,10 +5013,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
// mesh-build-machine row goes, when no registered manifest claims it.
// Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1);
// fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts
// (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2);
// fifty-seven with mesh-grants (ADR 0582 clause 1).
if len(catalogue.Seats()) != 57 {
t.Errorf("the mesh defines %d seats rather than 57; the set is closed, so a change here is "+
// (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2).
if len(catalogue.Seats()) != 56 {
t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+
"a decision (novox/hq ADR 0110): %s", len(catalogue.Seats()), seatNames())
}
}
+6 -8
View File
@@ -7,22 +7,20 @@ The copy carries the catalogue's private details: domains, first names in module
private addresses. That is acceptable while mesh-sdk is private; if it ever goes public, this copy is
a second place to clean besides the catalogue itself.
Only what go/catalogue's tests read is captured (R4b, issue 1379), copied here from mesh-sdk with the judge's tests
(issue 1585). mesh-host refreshed 2026-11-01 from its main, the catalogue 2026-10-21 from its main; before that it
was mesh-controller's copy, which still held two modules that had left the catalogue.
Only what go/catalogue's tests read is captured (R4b, issue 1379). Refreshed 2026-10-21 from the mains of both
repositories; before that it was mesh-controller's copy, which still held two modules that had left the catalogue.
mesh-catalog b5dbfbb784507e211372a7c9640d91f7d6dadd84 modules/*/module.json, modules/*/files/**, modules/*/templates/**, predefined.json
mesh-host 233bb23bbcb1a829ce2a6164a31a5c5cc4f3d950 internal/apply/placement_guard.go
mesh-host cdcba9ca73755854d21e4e657d7e46c32b9b5265 internal/apply/placement_guard.go
To move them, from mesh-delivery's modules/mesh-delivery-indexer directory, with the two repositories checked out
beside mesh-delivery:
To move them, from this repository's go/ directory, with the two repositories checked out beside mesh-sdk:
rm -rf testdata/beside/mesh-catalog testdata/beside/mesh-host
mkdir -p testdata/beside/mesh-catalog testdata/beside/mesh-host/internal/apply
git -C ../../../mesh-catalog archive <commit> -- predefined.json ':(glob)modules/*/module.json' \
git -C ../../mesh-catalog archive <commit> -- predefined.json ':(glob)modules/*/module.json' \
':(glob)modules/*/files/**' ':(glob)modules/*/templates/**' | tar -x -C testdata/beside/mesh-catalog
find testdata/beside -name module.json -exec mv {} {}.captured \;
git -C ../../../mesh-host show <commit>:internal/apply/placement_guard.go \
git -C ../../mesh-host show <commit>:internal/apply/placement_guard.go \
> testdata/beside/mesh-host/internal/apply/placement_guard.go.captured
and write the commits here. The placement guard is kept as .captured so no Go tool reads it as this
@@ -56,7 +56,7 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-host/internal/declaration"
)
// protectedRoots are paths no directory, access or mount source may be, nor hold.
@@ -384,9 +384,6 @@ func ownedByAccount(owner string, a homeAccount) bool {
if owner == a.Name {
return true
}
if a.UID < 0 {
return false // an account the declaration makes has no uid yet: by name alone, never by a number
}
user, _, _ := strings.Cut(owner, ":")
if uid, err := strconv.Atoi(user); err == nil {
return uid == a.UID
@@ -416,91 +413,12 @@ const (
// refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module,
// owner a directory's declared owner.
func refusePath(path string, kind placing, module, owner string) error {
return refusePathWith(path, kind, module, owner, nil)
}
// declaredHomes is the home of each account the declaration itself makes (a user resource with a home), by the
// account's name; the account is not in the user database yet when the declaration is judged, and the engine makes
// it where it is declared, before the directory that is owned by it is reached (novox/hq issue 1542). Its uid is
// not known, so a directory is owned by it by name alone. Only for the directory placement guard; no other
// reader of accountsOfHomes sees these.
func declaredHomes(resources []declaration.Resource) map[string]homeAccount {
var accounts []passwdEntry
var out map[string]homeAccount
for _, r := range resources {
u, ok := r.(*declaration.User)
if !ok || u.Name == "" {
continue
}
home := filepath.Clean(u.Home)
if u.Home == "" || !filepath.IsAbs(home) || home == "/" {
continue
}
if accounts == nil {
accounts = passwdEntries()
}
// Only ever adds a home. An account the machine has keeps the home it has (applyUser never moves it), so
// a declared home of one is not its home, unless it is the very home it has: the account an earlier apply
// made, as declared, which must be judged the same at every later apply. And a home at or below another
// account's, or at or above a protected root, is not a new one.
taken := false
for _, e := range accounts {
if e.name == u.Name && e.home != home {
taken = true
}
if e.name != u.Name && e.home != "" && atOrBelow(home, e.home) {
taken = true
}
}
for _, root := range protectedRoots {
taken = taken || home == root || below(root, home)
}
if taken {
continue
}
if out == nil {
out = map[string]homeAccount{}
}
out[home] = homeAccount{Name: u.Name, UID: -1}
}
return out
}
// passwdEntry is an account of the user database and its home, whatever its uid.
type passwdEntry struct{ name, home string }
// passwdEntries is every account of the user database with its home (empty and root homes as "").
func passwdEntries() []passwdEntry {
out := []passwdEntry{}
f, err := os.Open(passwdFile)
if err != nil {
return out
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
fields := strings.Split(sc.Text(), ":")
if len(fields) < 6 {
continue
}
home := filepath.Clean(fields[5])
if home == "." || home == "/" || home == "/nonexistent" {
home = ""
}
out = append(out, passwdEntry{fields[0], home})
}
return out
}
// refusePathWith is refusePath with the homes of the accounts the declaration makes (declaredHomes) counted
// among the machine's.
func refusePathWith(path string, kind placing, module, owner string, declared map[string]homeAccount) error {
clean := filepath.Clean(path)
if !filepath.IsAbs(clean) {
return nil // the declaration refuses a relative path already; a named volume is not a path
}
for _, p := range []string{clean, resolved(clean)} {
if err := refuseOne(p, kind, module, owner, declared); err != nil {
if err := refuseOne(p, kind, module, owner); err != nil {
if p != clean {
err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why)
err.Path = clean
@@ -511,7 +429,7 @@ func refusePathWith(path string, kind placing, module, owner string, declared ma
return nil
}
func refuseOne(path string, kind placing, module, owner string, declared map[string]homeAccount) *PlacementRefusedError {
func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError {
for _, root := range protectedRoots {
if path == root || below(root, path) {
return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " +
@@ -564,18 +482,6 @@ func refuseOne(path string, kind placing, module, owner string, declared map[str
return nil
}
homes := accountsOfHomes()
if ssh && len(declared) > 0 {
merged := map[string]homeAccount{}
for home, a := range homes {
merged[home] = a
}
for home, a := range declared {
if _, held := merged[home]; !held {
merged[home] = a
}
}
homes = merged
}
var deepest string
for home := range homes {
if below(path, home) && len(home) > len(deepest) {
@@ -612,12 +518,11 @@ func moduleOfID(id string) string {
// a file into it, because nothing judged the file's path.
func refusedPlaces(resources []declaration.Resource) map[string]error {
out := map[string]error{}
declared := declaredHomes(resources)
for _, r := range resources {
var err error
switch res := r.(type) {
case *declaration.Directory:
err = refusePathWith(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner, declared)
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
// And outside every home, the walk's judgement before anything is applied (novox/hq issue 358): a
// link another account could make in its path refuses it, and with it what is below it, whatever
// order they are declared in. The apply walks again at the moment of use. A directory handed over
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-teller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+8 -2
View File
@@ -1,5 +1,11 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+8 -2
View File
@@ -1,5 +1,11 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-watchman/internal/deadletters"
@@ -39,13 +38,6 @@ const (
routeDead = "dead-letter"
routeBusFault = "bus-fault"
routeProv = "provisioned"
routeLease = "lease"
)
// The lease write's subjects (ADR 0582 clauses 4, 6): the seat mesh-grants' events, and ASSIGNMENTS deleted.
var (
leaseEventPrefix = "mesh.seat." + catalogue.GrantsSeat + ".event."
assignmentsGone = "$JS.EVENT.ADVISORY.STREAM.DELETED." + buscompose.AssignmentsStream
)
// The bus module's events of the connections the bus closed (S9).
@@ -69,8 +61,6 @@ func routeOf(subject string) string {
return routeTools
case subject == CheckedSubject:
return routeChecked
case strings.HasPrefix(subject, leaseEventPrefix) || subject == assignmentsGone:
return routeLease
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."):
return routeAdvisory
case strings.HasPrefix(subject, "mesh.events.dead."):
@@ -106,7 +96,6 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
}
if dead.isTheControllersNotices(a.ID) {
dead.check() // the lease write's step removed it: the watchman may take the notices now
w.RemovalGone(watch.RemovalConsumer)
return
}
w.Advised(a)
@@ -121,22 +110,6 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
kind = watch.KindConnectionClosed
}
w.Closed(kind, c)
case routeLease:
var body struct {
Altered string `json:"altered"`
Failed string `json:"failed"`
}
_ = json.Unmarshal(m.Data, &body)
switch m.Subject {
case assignmentsGone:
w.RemovalGone(watch.RemovalStream)
case leaseEventPrefix + catalogue.GrantsEventLeaseAltered:
w.LeaseAltered(body.Altered)
case leaseEventPrefix + catalogue.GrantsEventLeaseHanded:
w.LeaseHanded()
case leaseEventPrefix + catalogue.GrantsEventRemovalFailed:
w.RemovalFailed(body.Failed)
}
case routeProv:
var r watch.ProvReport
if err := json.Unmarshal(m.Data, &r); err != nil {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+14 -4
View File
@@ -1,7 +1,17 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -59,23 +59,6 @@ func TestTheWatchmansFindings(t *testing.T) {
}
w.ModuleVerdict("node-tools", value(""), Put)
}, map[string]string{}},
{"774/S22/AC1, AC2: lease-altered and removal-failed are findings; lease-handed and the removal read gone clear them", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.LeaseAltered("the key holder names controller@a pid 9, not mesh-grants by lease-write-1")
if f := o.findings[FindingKey(ScopeBus, "mesh-controller_lease", KindLeaseAltered)]; !strings.Contains(f.Summary, "controller@a pid 9") {
t.Fatalf("findings = %v", o.findings)
}
w.LeaseHanded()
w.RemovalFailed("the stream ASSIGNMENTS: ASSIGNMENTS has 1 consumers")
k := FindingKey(ScopeBus, "lease-removal", KindRemovalFailed)
if f := o.findings[k]; !strings.Contains(f.Summary, "ASSIGNMENTS has 1 consumers") {
t.Fatalf("findings = %v", o.findings)
}
w.RemovalGone(RemovalConsumer) // not what it waits on
if _, ok := o.findings[k]; !ok {
t.Fatal("cleared by the consumer, while the stream it names stands")
}
w.RemovalGone(RemovalStream)
}, map[string]string{}},
{"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Step("go", "n1.ask1", runningStep("ask1", t0), Put)
expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second)
@@ -1,80 +0,0 @@
package watch
import (
"strings"
)
// **The lease write's findings** (novox/hq ADR 0582 clauses 4 and 6, issue 774): mesh-grants says lease-altered when the
// lease key or its bucket's expiry differs from what it wrote, and removal-failed when the day-later removal of the
// controller's consumer of DEAD_LETTER_NOTICES or the stream ASSIGNMENTS failed; the watchman raises each as a finding.
// lease-altered clears when lease-handed is said again (the operator asked the write again); removal-failed clears when
// what it names reads gone: the consumer's deletion advisory and ASSIGNMENTS' deletion advisory, each heard.
// The two kinds.
const (
KindLeaseAltered = "lease-altered"
KindRemovalFailed = "removal-failed"
)
// The finding's IDs: the lease's bucket, and the removal.
const (
leaseID = "mesh-controller_lease"
removalID = "lease-removal"
)
// What a failed removal still waits on.
const (
RemovalConsumer = "DEAD_LETTER_NOTICES.controller"
RemovalStream = "ASSIGNMENTS"
)
// LeaseAltered raises the finding that the lease differs from what mesh-grants wrote, naming what changed.
func (w *Watchman) LeaseAltered(what string) {
w.mu.Lock()
defer w.mu.Unlock()
w.find(Finding{Scope: ScopeBus, ID: leaseID, Kind: KindLeaseAltered, Severity: Warning, Source: "mesh-grants",
Summary: "the controller's lease differs from what mesh-grants wrote: " + what,
Said: what,
Needs: "a look at who changed the lease; mesh-grants writes nothing until lease-write is asked again.",
Resolved: "lease-write is asked and warranted again, and the lease reads as written"})
}
// LeaseHanded clears the lease-altered finding: the lease reads as written again.
func (w *Watchman) LeaseHanded() {
w.mu.Lock()
defer w.mu.Unlock()
w.unfind(FindingKey(ScopeBus, leaseID, KindLeaseAltered))
}
// RemovalFailed raises the finding that the day-later removal failed, naming what failed; it waits on each of the
// consumer and the stream the failure names.
func (w *Watchman) RemovalFailed(failed string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.removalWaits == nil {
w.removalWaits = map[string]bool{}
}
for _, what := range []string{RemovalConsumer, RemovalStream} {
if strings.Contains(failed, strings.SplitN(what, ".", 2)[0]) {
w.removalWaits[what] = true
}
}
if len(w.removalWaits) == 0 { // names neither: waits on both
w.removalWaits[RemovalConsumer], w.removalWaits[RemovalStream] = true, true
}
w.find(Finding{Scope: ScopeBus, ID: removalID, Kind: KindRemovalFailed, Severity: Warning, Source: "mesh-grants",
Summary: "the day-later removal of the controller's consumer and stream failed: " + failed,
Said: failed,
Needs: "a look at why; mesh-grants tries again in a day, and grants_removal_due starts it sooner.",
Resolved: "the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS read gone"})
}
// RemovalGone hears one of the two removed (RemovalConsumer or RemovalStream), and clears the finding once neither waits.
func (w *Watchman) RemovalGone(what string) {
w.mu.Lock()
defer w.mu.Unlock()
delete(w.removalWaits, what)
if len(w.removalWaits) == 0 {
w.unfind(FindingKey(ScopeBus, removalID, KindRemovalFailed))
}
}
@@ -86,8 +86,6 @@ type Watchman struct {
recount func()
// waiting is, by `<consumer>.<node>`, each of its provisions in waits, with its words.
waiting map[string]map[string]string
// removalWaits is what a failed removal of the lease write still waits on to read gone (lease.go).
removalWaits map[string]bool
// health is each node's own health report, and tunnels each node's tunnel key from its record.
health map[string]delivery.NodeHealth
tunnels map[string]string
@@ -12,7 +12,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The three bus adapters for mesh-sdk go/asker on the part's own credential, as mesh-delivery-teller has them (copied: the
@@ -82,16 +81,3 @@ func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c a
return err
}
}
// seatSaying says the seat mesh-grants' events on its seat subject into EVENTS, as the module that holds it (novox/hq
// ADR 0582 clauses 3, 4, 6): lease-handed, lease-altered and removal-failed.
func seatSaying(ctx context.Context, js jetstream.JetStream) func(event string, body any) error {
return func(event string, body any) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
_, err = js.Publish(ctx, "mesh.seat."+catalogue.GrantsSeat+".event."+event, data)
return err
}
}
@@ -1,297 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/seal"
"mesh-grants/internal/grants"
)
// standIns are the seats and tools the lease write asks, standing in on the real bus's side: node-service-manager on
// the controller's node (a unit's ActiveState, as the systemd module answers it), the bus module's expiry tool acting on
// the real lease bucket's stream (as nats-tools' SetBucketExpiry does), and the checks answering green.
type standIns struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
active string
calls []string
}
func (s *standIns) Call(key string, body any) (json.RawMessage, error) {
s.mu.Lock()
s.calls = append(s.calls, key)
s.mu.Unlock()
base, _, _ := strings.Cut(key, "@")
args, _ := body.(map[string]any)
var out any
switch base {
case grants.CallServiceStop:
s.active, out = "inactive", map[string]any{"ok": true}
case grants.CallServiceStart:
s.active, out = "active", map[string]any{"ok": true}
case grants.CallServiceStatus:
out = map[string]any{"ActiveState": s.active}
case grants.CallBucketExpiry:
now := fmt.Sprint(args["max_age"])
d, err := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
if err != nil {
return nil, err
}
st, err := s.js.Stream(s.ctx, "KV_"+fmt.Sprint(args["bucket"]))
if err != nil {
return nil, err
}
cfg := st.CachedInfo().Config
cfg.MaxAge = d
if d > 0 && (cfg.Duplicates == 0 || cfg.Duplicates > d) {
cfg.Duplicates = d
}
if _, err := s.js.UpdateStream(s.ctx, cfg); err != nil {
return nil, err
}
out = map[string]any{"bucket": args["bucket"], "now": now, "changed": true}
case grants.CallIssue:
out = map[string]any{"status": "resolved"}
case grants.CallNodeTrusts:
out = map[string]any{"every": true, "not_yet": []string{}}
case grants.CallConfigCompare:
out = map[string]any{"source": "indexer", "count": 0}
case grants.CallUpgrade:
out = map[string]any{"empty": true, "splits": []any{}}
case grants.CallVaultStatus:
out = map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}
case grants.CallStreams:
out = map[string]any{"detail": []any{}}
case grants.CallUsers:
out = map[string]any{"users": []any{}}
case grants.CallRecordWarrant, grants.CallRecordHandAct:
out = map[string]any{}
default:
return nil, fmt.Errorf("no stand-in for %s", key)
}
return json.Marshal(out)
}
// memStore is the part's own state, in memory, with its time to live per key recorded.
type memStore struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
}
func (m *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
v, ok := m.vals[state+"/"+key]
return v, m.revs[state+"/"+key], ok, nil
}
func (m *memStore) Put(state, key string, value []byte) (uint64, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.vals[state+"/"+key] = append([]byte(nil), value...)
m.revs[state+"/"+key]++
return m.revs[state+"/"+key], nil
}
func (m *memStore) PutTTL(state, key string, value []byte, _ time.Duration) (uint64, error) {
return m.Put(state, key, value)
}
func (m *memStore) Keys(state string) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []string
for k := range m.vals {
if s, key, _ := strings.Cut(k, "/"); s == state {
out = append(out, key)
}
}
return out, nil
}
func (m *memStore) Delete(state, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.vals, state+"/"+key)
return nil
}
type memObjects struct{}
func (memObjects) PutObject(string, []byte) error { return nil }
func (memObjects) DeleteObject(string) error { return nil }
// yesAsker answers every ask the part raises with the operator's yes, as the router's warrant hands it.
type yesAsker struct{ asked []string }
func (a *yesAsker) Ask(q asks.Ask) error { a.asked = append(a.asked, q.ID); return nil }
func (a *yesAsker) Where(string) (string, *asks.Warrant, error) { return "open", nil, nil }
func (a *yesAsker) Cancel(string) error { return nil }
func (a *yesAsker) Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) {
return asker.Acted, act(asks.Option{ID: w.Option}, w)
}
// Issue 774, 774/S23 (novox/hq ADR 0582 clauses 3, 5, "How it is checked"): mesh-grants' lease write and hand-back
// against the real bus, with a stand-in controller holding the lease in a bucket with an expiry (2 s standing in for 15
// s, so the test waits past it in seconds), and stand-in node-service-manager and bus tool seats.
func TestTheLeaseWriteOnTheBus(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "lease")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, _ := jetstream.New(nc)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
const ttl = 2 * time.Second
was := grants.LeaseExpiry
grants.LeaseExpiry = ttl
t.Cleanup(func() { grants.LeaseExpiry = was })
lease, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buscompose.LeaseBucket, History: 1, TTL: ttl})
if err != nil {
t.Fatal(err)
}
controller, _ := json.Marshal(buscompose.LeaseHolder{Instance: "controller@a pid 1", Host: "a", Epoch: 60})
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
t.Fatal(err)
}
bucket := func(state string) jetstream.KeyValue {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state), History: 5})
if err != nil {
t.Fatal(err)
}
return kv
}
put := func(kv jetstream.KeyValue, key string, v any) {
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(ctx, key, raw); err != nil {
t.Fatal(err)
}
}
nats := `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},` +
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`
digest := delivery.ManifestDigest([]byte(nats))
put(bucket(grants.ReadManifests), digest, delivery.Manifest{V: delivery.Version, Module: "nats", JSON: nats})
put(bucket(grants.ReadModules), "nats", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32), Manifest: digest,
Repository: "novox/mesh-catalog", Commit: "c"})
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}}})
if _, err := bucket(grants.ReadPlacement).Put(ctx, "placement.a", pl); err != nil {
t.Fatal(err)
}
put(bucket(grants.ReadTakeover), health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "taken"})
put(bucket(grants.ReadAccounts), buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1, "at": "2026-11-01T00:00:00Z"})
busKey, _ := seal.GenerateKeyPair()
have := bucket(grants.ReadHave)
loads := func(source, digest string) {
put(have, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: source, Revision: 1, Digest: digest, Users: 1, Key: busKey.PublicKey}})
}
loads("a.nats.src2", "sha256:aa")
seats := &standIns{ctx: ctx, js: js, active: "active"}
store := &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
p := &grants.Part{Read: newBusReader(ctx, js), Call: seats, Store: store, Lease: &leaseHand{ctx: ctx, js: js},
Asker: &yesAsker{}, Objects: memObjects{}, Nonce: func() string { return "0001" }}
act := func(ask func(string, string) (any, error)) {
t.Helper()
if _, err := ask("operator@shanks", "issue 774's integration test"); err != nil {
t.Fatal(err)
}
p.Compose() // the process raises the ask
r, _ := p.LeaseState()
if r.Open == nil {
t.Fatalf("no act open: %+v", r)
}
if err := p.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: r.Open.Ask, Option: "yes", Outcome: asks.OutcomeChosen}); err != nil {
t.Fatal(err)
}
r, _ = p.LeaseState()
if r.Open == nil || r.Open.List == "" {
t.Fatalf("the act is not waiting on its list: %+v", r)
}
loads(delivery.GrantsUserListSource, r.Open.List) // the bus node loads it
p.Compose()
if r, _ := p.LeaseState(); r.Open != nil {
t.Fatalf("the act did not finish: %+v", r.Open)
}
}
expiry := func() time.Duration {
st, err := lease.Status(ctx)
if err != nil {
t.Fatal(err)
}
return st.TTL()
}
// 774/S23/AC1.
act(p.LeaseWrite)
e, err := lease.Get(ctx, buscompose.ControllerLeaseKey)
if err != nil {
t.Fatal(err)
}
if away, says := buscompose.LeaseHandedAway(e.Value(), true, nil); !away || !strings.Contains(string(e.Value()), `"handed-to":"mesh-grants"`) {
t.Fatalf("the lease after the write: %s (%s)", e.Value(), says)
}
if expiry() != 0 || seats.active != "inactive" {
t.Fatalf("the lease's expiry is %s, the controller %s", expiry(), seats.active)
}
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); !errors.Is(err, jetstream.ErrKeyExists) {
t.Fatalf("the controller's retake: %v, want refused", err)
}
time.Sleep(ttl + time.Second)
if e, err := lease.Get(ctx, buscompose.ControllerLeaseKey); err != nil {
t.Fatalf("the lease is gone past the old expiry: %v", err)
} else if away, _ := buscompose.LeaseHandedAway(e.Value(), true, nil); !away {
t.Fatalf("the lease past the old expiry: %s", e.Value())
}
// 774/S23/AC2.
act(p.LeaseHandBack)
if expiry() != ttl || seats.active != "active" {
t.Fatalf("after the hand-back the expiry is %s, the controller %s", expiry(), seats.active)
}
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
t.Fatalf("the controller cannot take the lease back: %v", err)
}
}
@@ -1,79 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// leaseHand is the process's hand on the controller's lease (novox/hq ADR 0582 clause 3): its key holder, which the
// lease write's cut grants the part alone, and the bucket's expiry, read from the bucket's own stream.
type leaseHand struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
kv jetstream.KeyValue
}
func (l *leaseHand) bucket() (jetstream.KeyValue, error) {
l.mu.Lock()
defer l.mu.Unlock()
if l.kv != nil {
return l.kv, nil
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
kv, err := l.js.KeyValue(ctx, buscompose.LeaseBucket)
if err != nil {
return nil, fmt.Errorf("the lease's bucket cannot be opened: %w", err)
}
l.kv = kv
return kv, nil
}
func (l *leaseHand) PutHolder(value []byte) error {
kv, err := l.bucket()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
_, err = kv.Put(ctx, buscompose.ControllerLeaseKey, value)
return err
}
func (l *leaseHand) DeleteHolder() error {
kv, err := l.bucket()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
return kv.Delete(ctx, buscompose.ControllerLeaseKey)
}
func (l *leaseHand) Expiry() (time.Duration, error) {
kv, err := l.bucket()
if err != nil {
return 0, err
}
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
defer cancel()
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
return st.TTL(), nil
}
// PutTTL writes a key of the part's own state with its own time to live (the state lease).
func (s ownStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
return s.c.State(state).RenewTTL(key, json.RawMessage(value), ttl)
}
+4 -4
View File
@@ -16,8 +16,10 @@
// It holds no timer, and never exits for a refused read, write or publish: a refusal is said and tried again at the next
// event.
//
// **The seat mesh-grants** (novox/hq ADR 0582 clause 1, issue 774): the part holds it for the lease write, serving
// lease-write and lease-hand-back beside its own grants_* tools, and says lease-handed, lease-altered and removal-failed.
// **No seat in this milestone.** ADR 0547 clause 5 gives the part the seat `mesh-grants`; a `mesh-*` seat must be in
// mesh-sdk's own seat list, which the running controller's composer does not know, so the part serves module tools
// (grants_*) and claims no seat until take-over. The composer grants the membership publish by the module's name,
// mesh-grants.
//
// Started by the tool runner, with no credential of its own, it serves those tools (a process never serves, ADR 0436).
package main
@@ -119,8 +121,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
client := &asker.Client{Module: account.Module, State: conn.State(grants.StateTakeover), Publish: publishVia(ctx, js),
Record: recordVia(ctx, nc, account.Module), Subscribe: decidedVia(nc, account.Module, logger.Printf)}
f.Part.Asker, f.Part.Say = client, sayVia(ctx, js, account.Module)
// The lease write's hand on the lease, and its seat's events (ADR 0582 clauses 3, 4, 6).
f.Part.Lease, f.Part.SaySeat = &leaseHand{ctx: ctx, js: js}, seatSaying(ctx, js)
f.Part.Forget = func(id string) error { return conn.State(grants.StateTakeover).Delete("ask." + id) }
if err := client.Listen(f.Part.Warranted, f.Part.Said); err != nil {
logger.Printf("the operator's answers cannot be heard yet, so take-over and hand-back wait: %v", err)
@@ -3,10 +3,8 @@ package main
import (
"encoding/json"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-grants/internal/grants"
)
@@ -46,10 +44,6 @@ func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(st
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
return stdio.State(state).PutTTL(key, json.RawMessage(value), ttl)
}
// runtimeBus calls tools and says events through the runtime.
type runtimeBus struct{}
@@ -177,40 +171,9 @@ func toolsOf(p *grants.Part) []stdio.Tool {
"with the tool that answers each. Asks the controller nothing; writes nothing. Read-only.",
Input: schema(nil),
Run: func(map[string]any) (any, error) { return p.LeaseCheck() }},
// The seat's two verbs (ADR 0582 clauses 1, 5): asked of the operator by the part's process.
{Name: grants.SeatTool(grants.ActLeaseWrite), Description: seatVerb(grants.ActLeaseWrite).Description,
Input: seatVerb(grants.ActLeaseWrite).Input,
Run: func(a map[string]any) (any, error) {
return p.LeaseWrite(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.SeatTool(grants.ActLeaseHandBack), Description: seatVerb(grants.ActLeaseHandBack).Description,
Input: seatVerb(grants.ActLeaseHandBack).Input,
Run: func(a map[string]any) (any, error) {
return p.LeaseHandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
}},
{Name: grants.ToolRemovalDue, Description: "Write the lease write's removal-due again with a short time to live, so " +
"its expiry marker starts the day-later removal of the controller's consumer of DEAD_LETTER_NOTICES and the " +
"stream ASSIGNMENTS (novox/hq ADR 0582 clause 6): the recovery for a marker missed while the part was down, the " +
"removal still started by the marker and nothing else. Refused unless the lease is handed to mesh-grants and the " +
"removal has not run. Recorded as a hand act.",
Input: schema(map[string]string{"why": "why, kept with the hand act", "ttl": "the time to live, e.g. 10s (1s to 24h)"}, "why", "ttl"),
Run: func(a map[string]any) (any, error) {
return p.RemovalDue(stdio.CallerOf(a), grants.Answer(a, "why"), grants.Answer(a, "ttl"))
}},
}
}
// seatVerb is the seat mesh-grants' verb as mesh-sdk declares it.
func seatVerb(name string) catalogue.Verb {
seat, _ := catalogue.SeatNamed(catalogue.GrantsSeat)
for _, v := range seat.Serves {
if v.Name == name {
return v
}
}
return catalogue.Verb{Name: name}
}
// serveTools serves the tools to the runtime until it stops.
func serveTools() error {
p := &grants.Part{Read: runtimeReader{}, Call: runtimeBus{}, Store: runtimeStore{}, Emit: runtimeBus{}}
+1 -8
View File
@@ -3,21 +3,14 @@ module mesh-grants
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
github.com/nats-io/nats-server/v2 v2.11.17
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
+2 -7
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -20,11 +18,8 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+76 -99
View File
@@ -319,38 +319,8 @@ func (b *fakeBus) refuse(module, raw, problem string) {
Problems: []string{problem}})
}
// assign places module on node in the seat's placement, the one place the part reads it (ADR 0549 §5; issue 774).
func (b *fakeBus) assign(node, module string) {
var pl delivery.Placement
if raw, ok := b.states[grants.ReadPlacement]["placement."+node]; ok {
pl, _ = delivery.DecodePlacement(raw)
}
pl.Node = node
for _, m := range pl.Modules {
if m.Module == module {
return
}
}
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: module})
raw, err := delivery.EncodePlacement(pl)
if err != nil {
panic(err)
}
if b.states[grants.ReadPlacement] == nil {
b.states[grants.ReadPlacement] = map[string][]byte{}
}
b.states[grants.ReadPlacement]["placement."+node] = raw
}
// unassign takes module off node in the seat's placement.
func (b *fakeBus) unassign(node, module string) {
pl, err := delivery.DecodePlacement(b.states[grants.ReadPlacement]["placement."+node])
if err != nil {
return
}
pl.Modules = slices.DeleteFunc(pl.Modules, func(m delivery.PlacedModule) bool { return m.Module == module })
raw, _ := delivery.EncodePlacement(pl)
b.states[grants.ReadPlacement]["placement."+node] = raw
b.put(grants.ReadAssignments, node+"."+module, map[string]any{"node": node, "module": module, "assigned": "2026-10-25T20:00:00Z"})
}
const (
@@ -380,6 +350,7 @@ func fixture() *fakeBus {
b.assign("a", "nats")
b.assign("a", "mod-x")
b.assign("b", "mod-y")
b.put(grants.ReadAssignments, "b.gone", map[string]any{"node": "b", "module": "gone", "assigned": "2026-10-01T00:00:00Z", "ended": "2026-10-02T00:00:00Z"})
b.put(grants.ReadHave, "a.node", map[string]any{"v": 1})
b.put(grants.ReadHave, "b.node", map[string]any{"v": 1})
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
@@ -427,18 +398,8 @@ func TestGrants(t *testing.T) {
if err != nil {
t.Fatalf("mesh-sdk cannot decode the manifest: %v", err)
}
// The part holds the seat mesh-grants for the lease write (ADR 0582 clause 1, issue 774, 774/S14), as mesh-sdk's seat
// list declares it.
if seat, ok := catalogue.SeatNamed(catalogue.GrantsSeat); m.Module != "mesh-grants" || len(m.Claims) != 1 || !ok ||
catalogue.CanHold(m, seat) != nil {
t.Errorf("module %q claims %v: the part holds mesh-grants alone (%v)", m.Module, m.Claims, catalogue.CanHold(m, seat))
}
var keyTTL bool
for _, s := range m.State {
keyTTL = keyTTL || (s.Name == grants.StateLease && s.KeyTTL)
}
if !keyTTL {
t.Errorf("the state %s gives no time to live per key, which removal-due needs", grants.StateLease)
if m.Module != "mesh-grants" || len(m.Claims) != 0 {
t.Errorf("module %q claims %v: the part claims no seat until mesh-sdk's seat list names mesh-grants", m.Module, m.Claims)
}
for _, row := range []struct {
what string
@@ -488,9 +449,8 @@ func TestGrants(t *testing.T) {
if path != publishOnly && path != asksOnly && (strings.Contains(string(src), ".Publish(") || strings.Contains(string(src), "mesh.assignment")) {
t.Errorf("%s publishes: only publish.go publishes memberships, and asking.go the operator's asks", path)
}
// And the seat mesh-grants' events (ADR 0582 clauses 3, 4, 6; issue 774).
if path == asksOnly && (strings.Count(string(src), ".Publish(") != 3 || strings.Contains(string(src), "mesh.assignment")) {
t.Errorf("asking.go publishes other than an ask, a cancel and a condition, and the seat's events, through js.Publish thrice")
if path == asksOnly && (strings.Count(string(src), ".Publish(") != 2 || strings.Contains(string(src), "mesh.assignment")) {
t.Errorf("asking.go publishes other than an ask, a cancel and a condition, through js.Publish twice")
}
if path == publishOnly && strings.Count(string(src), ".Publish(") != 1 {
t.Errorf("publish.go publishes other than one membership")
@@ -541,8 +501,8 @@ func TestGrants(t *testing.T) {
key, _ := delivery.BuilderBuildKey("mod-x", fp)
b.put(grants.ReadBuilds, key, map[string]any{"v": delivery.Version, "state": "done", "manifest": digest})
}, []string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"},
{"placement refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadPlacement] = true },
[]string{"controller", "node.a", "node.b"}, grants.ReadPlacement},
{"placement refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
[]string{"controller", "node.a", "node.b"}, grants.ReadAssignments},
} {
t.Run(row.name, func(t *testing.T) {
b := fixture()
@@ -1130,10 +1090,8 @@ func TestGrants(t *testing.T) {
t.Errorf("read of the keys per module: placed %v, gaps %v", got.Placed, got.Gaps)
}
b.refused[grants.ReadPlacement] = true
// The placement alone places (issue 774, 774/S29): not read, nothing is placed, and it is a failed read.
if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadPlacement && g.Failed }) ||
len(read.Assigned) != 0 {
t.Errorf("placement not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
if read := grants.ReadInputs(b, &grants.Manifests{}); slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadPlacement && g.Failed }) {
t.Errorf("placement not on the bus is a failed read: %v", read.Gaps)
}
// Issue 1945, the user list of 2026-10-27 09:21: a placement's holds are the holders on record, not every holder.
// A mesh seat no placement holds keeps the module claiming it as its holder, wherever placement puts it.
@@ -1405,7 +1363,7 @@ func TestGrants(t *testing.T) {
// settled is the placement of the lists agreeing, every hash held, and the loaded list as edit leaves it.
settled := func(edit func(modX *grants.User)) func() {
return func() {
b.states[grants.ReadPlacement] = nil
b.states[grants.ReadAssignments] = nil
b.assign("a", "nats")
b.assign("a", "mod-x")
b.assign("b", "mod-y")
@@ -1616,7 +1574,7 @@ func TestGrants(t *testing.T) {
b.loaded[i].Subscribe = append(slices.Clone(b.loaded[i].Subscribe), seatGrant)
}
}
b.assign("a", "mod-y")
b.put(grants.ReadAssignments, "a.mod-y", map[string]any{"node": "a", "module": "mod-y", "assigned": "2026-10-27T08:20:00Z"})
b.put(grants.ReadAccounts, "a.mod-y", map[string]any{"user": "a.mod-y", "hash": "$2a$10$x", "generation": 1})
said = nil
res := p.Compose()
@@ -1638,7 +1596,7 @@ func TestGrants(t *testing.T) {
b.refuseUsers = true
b.mu.Unlock()
b.mu.Lock()
b.unassign("a", "mod-y")
delete(b.states[grants.ReadAssignments], "a.mod-y")
b.mu.Unlock()
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "the grants of the list in force cannot be read") {
t.Errorf("a pass whose loaded list cannot be read = %+v", res.Sealed)
@@ -2076,11 +2034,11 @@ func TestGrants(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { f.Run(ctx); close(done) }()
waitOpened(t, w, grants.ReadPlacement)
waitOpened(t, w, grants.ReadAssignments)
if r := f.Refused(); r[grants.ReadModules] == "" {
waitFor(t, func() bool { return f.Refused()[grants.ReadModules] != "" })
}
w.change(grants.ReadPlacement, "placement.a")
w.change(grants.ReadAssignments, "a.mod-x")
waitOpened(t, w, grants.ReadModules)
waitFor(t, func() bool { return len(f.Refused()) == 0 })
select {
@@ -2273,64 +2231,83 @@ func TestGrants(t *testing.T) {
}
})
// ADR 0571 §13's test for this repository (issue 774, 774/S24/AC1), replacing the guard of ADR 0549 §6 and its list
// ../assignments-readers: at phase write no module.json here depends on mesh-controller beyond the lease and the three
// kept to the write (mesh-sdk catalogue.ManifestRefusals, the judge grants_lease_check runs).
t.Run("no module of this repository depends on the controller beyond what the lease write keeps", func(t *testing.T) {
// The guard of ADR 0549 §6 for this repository, as mesh-catalog's checks/placement is for the catalogue: no module.json
// here reads the controller's assignments unless ../assignments-readers names it with why, and that list only shrinks.
t.Run("assignments readers: only the named modules read the controller's assignments, and the list only shrinks", func(t *testing.T) {
raw, err := os.ReadFile("../assignments-readers")
if err != nil {
t.Fatal(err)
}
allowed := map[string]bool{}
for _, line := range strings.Split(string(raw), "\n") {
if f := strings.Fields(line); len(f) > 1 && !strings.HasPrefix(f[0], "#") {
allowed[f[0]] = true
} else if len(f) == 1 {
t.Errorf("../assignments-readers names %s with no why", f[0])
}
}
paths, _ := filepath.Glob("../*/module.json")
manifests := map[string]catalogue.Manifest{}
reads := map[string]bool{}
for _, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
m, err := catalogue.Decode(raw)
if err != nil {
var m struct {
Module string `json:"module"`
Reads []string `json:"reads"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatalf("%s: %v", path, err)
}
manifests[m.Module] = m
if slices.Contains(m.Reads, grants.ReadAssignments) {
reads[m.Module] = true
if !allowed[m.Module] {
t.Errorf("%s reads %s: read where modules run from the seat mesh-placement's placement (ADR 0549 §3); the lease write "+
"is refused while a module reads the assignments (ADR 0549 §6)", m.Module, grants.ReadAssignments)
}
}
}
if len(manifests) < 8 {
t.Fatalf("read %d manifests, want every module of the repository", len(manifests))
}
for _, why := range catalogue.ManifestRefusals(manifests, catalogue.PhaseWrite) {
t.Error(why)
for m := range allowed {
if !reads[m] {
t.Errorf("../assignments-readers names %s, which no longer reads %s: take it off the list", m, grants.ReadAssignments)
}
}
})
// 774/S29/AC1 (ADR 0549 §5, ADR 0571 §10): the memberships composed from the placement alone, with no read of the
// controller, are the ones the union of placement and assignments gave on the same input: the fixture's (two nodes,
// the bus and mod-x on a, mod-y on b, as placed on the live mesh's own shape of nats beside its modules), whose
// assignments named the same modules. Neither the controller's assignments nor its condition events are read.
t.Run("placement alone places, with the memberships the union gave", func(t *testing.T) {
// 774 (the L3 review, ADR 0549 §3, §6): once the controller's lease is handed away the assignments are frozen and place
// nothing; the seat's placement alone does. A module left only in the stale assignments loses its account and its
// grant names no account of it; before the lease write the two are still unioned.
t.Run("placement alone once the lease is handed away: a module only in the stale assignments has no account or grant of it", func(t *testing.T) {
b := fixture()
b.loaded = shadowOf(t, b)
p := partOn(b)
p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"}
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: "mod-x"}}})
b.states[grants.ReadPlacement] = map[string][]byte{"placement.a": pl}
for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} {
b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1})
}
fp := "src2:" + strings.Repeat("ab", 32)
key, _ := delivery.PartGrantKey("mod-y", fp)
grantOf := func() []string {
g, _ := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key])
return g.Accounts
}
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-28", "epoch": 60})
p.Compose()
want := map[string]string{"a.nats": "", "a.mod-x": "", "b.mod-y": ""}
for key := range want {
raw, _, found, _ := b.ownGet(grants.StateMemberships, key)
if !found {
t.Fatalf("no membership of %s from the placement", key)
}
want[key] = string(raw)
if b.own[grants.StateAccounts]["b.mod-y"] == nil || !slices.Equal(grantOf(), []string{"b.mod-y"}) {
t.Fatalf("under the controller's lease the assignments no longer count: account %v, grant %v",
b.own[grants.StateAccounts]["b.mod-y"] != nil, grantOf())
}
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", "b.mod-y"}) {
t.Errorf("memberships %v", got)
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
in := grants.ReadInputs(b, &grants.Manifests{})
if len(in.Assigned["b"]) != 0 || !slices.Equal(in.Assigned["a"], []string{"mod-x", "nats"}) || in.BusNode != "a" {
t.Errorf("after the lease write the stale assignments still place: %v, bus %q", in.Assigned, in.BusNode)
}
for _, r := range grants.Reads {
if strings.HasPrefix(r, "mesh-controller.") && r != grants.ReadLease {
t.Errorf("the part reads %s", r)
}
}
for _, c := range grants.Consumes {
if strings.HasPrefix(c, "mesh-controller.") {
t.Errorf("the part consumes %s", c)
}
}
for _, c := range b.calls {
if strings.Contains(c, "mesh-controller") {
t.Errorf("the part called %s", c)
}
res := p.Compose()
if res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) || slices.Contains(grantOf(), "b.mod-y") {
t.Fatalf("a module only in the stale assignments kept its account or grant: %+v, grant %v", res.Accounts, grantOf())
}
})
@@ -2372,7 +2349,7 @@ func TestGrants(t *testing.T) {
t.Errorf("the wanted accounts are read by %v", readers)
}
// A read that fails retires nothing; once read, the user placement dropped is retired.
b.unassign("b", "mod-y")
delete(b.states[grants.ReadAssignments], "b.mod-y")
b.refused[grants.ReadResolved] = true
if res := p.Compose(); res.Accounts == nil || len(res.Accounts.Deleted) != 0 || b.own[grants.StateAccounts]["b.mod-y"] == nil {
t.Fatalf("a pass whose read failed deleted %+v", res.Accounts)
+3 -19
View File
@@ -20,13 +20,7 @@ type Watcher interface {
// Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved
// config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
//
// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6).
var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover,
OwnLease}
// OwnLease is the part's own state lease as the follower watches it, `<module>.<state>`.
const OwnLease = Module + "." + StateLease
var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover}
// Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
// open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
@@ -127,23 +121,14 @@ func (f *Follower) heard(state, key string, value []byte) bool {
}
return false
}
if state == OwnLease {
// The marker of removal-due (its value gone: expired, or deleted by a hand-back, which the removal tells apart).
if key == KeyRemovalDue && value == nil && f.Part != nil {
f.Part.RemovalDueHeard()
return true
}
return false
}
if state == ReadLease {
// Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6).
// Once handed away nobody renews it, so any change is heard: it may be the lease altered (ADR 0582 clause 4).
on, _ := buscompose.LeaseHandedAway(value, len(value) > 0, nil)
f.mu.Lock()
defer f.mu.Unlock()
was, seen := f.leaseOn, f.leaseSeen
f.leaseOn, f.leaseSeen = on, true
return !seen || was != on || on
return !seen || was != on
}
if state != ReadHave {
return true
@@ -184,8 +169,7 @@ func (f *Follower) open() {
}
if f.Hear != nil && !f.heard_ {
err := f.Hear(func(event string, body []byte) {
// An account the vault minted, or a bucket's expiry set (the lease's, altered: ADR 0582 clause 4).
if IsAccountIssued(event) || IsBucketExpirySet(event) {
if IsAccountIssued(event) {
f.Wake()
return
}
+58 -17
View File
@@ -24,12 +24,13 @@ import (
// The states of other parts the grants part reads, as its manifest's reads name them (`<module>.<state>`).
const (
ReadModules = "mesh-delivery-indexer.modules"
ReadManifests = "mesh-delivery-indexer.manifests"
ReadHave = "node-engine.have"
ReadResolved = "mesh-config.resolved"
ReadWants = "mesh-delivery-assigner.wants"
ReadBuilds = "mesh-delivery-builder.builds"
ReadAssignments = "mesh-controller.assignments"
ReadModules = "mesh-delivery-indexer.modules"
ReadManifests = "mesh-delivery-indexer.manifests"
ReadHave = "node-engine.have"
ReadResolved = "mesh-config.resolved"
ReadWants = "mesh-delivery-assigner.wants"
ReadBuilds = "mesh-delivery-builder.builds"
// ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9).
ReadAccounts = "mesh-vault.bus-accounts"
// ReadPlacement is the seat mesh-placement's placement (ADR 0567 §1, §9): each node's modules and the seats
@@ -37,9 +38,10 @@ const (
ReadPlacement = delivery.PlacementRead
)
// Reads is every state the part reads, in the manifest's order. Not the controller's assignments: the seat's placement
// alone places modules (novox/hq ADR 0549 §5, ADR 0571 §10; issue 774).
var Reads = []string{ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
// Reads is every state the part reads, in the manifest's order. ReadAssignments stays (and in Followed) only while the
// controller holds the lease, when the assignments are unioned with the placement (ReadInputs); after the lease write
// they place nothing, and the first change after it takes them out (../assignments-readers names the part until then).
var Reads = []string{ReadAssignments, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
ReadAccounts, ReadPlacement, ReadLease, ReadTakeover}
// The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
@@ -50,7 +52,7 @@ const (
)
// Invokes is every tool the part calls.
var Invokes = append([]string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant}, LeaseCalls...)
var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant}
// Reader reads another part's state, named `<module>.<state>`. A refusal (a read not granted yet, a bucket not there) is
// an error the part names as a gap and composes past: it never stops for one.
@@ -81,6 +83,18 @@ var standingGaps = []Gap{
{Input: "announced states", Why: "the controller does not announce its states-announced readable, so every read is composed from `reads`, as when announcements cannot be read"},
}
// assignment is a key of the controller's assignments as composing needs it (mesh-controller internal/chain Assignment):
// a value that names no time assigned (a module pulled in), has ended, or waits in a state, is no assignment.
type assignment struct {
Node string `json:"node"`
Module string `json:"module"`
Assigned time.Time `json:"assigned,omitzero"`
Ended time.Time `json:"ended,omitzero"`
State string `json:"state,omitempty"`
}
func (a assignment) stands() bool { return !a.Assigned.IsZero() && a.Ended.IsZero() && a.State == "" }
// resolved is a key of mesh-config's resolved values, `<node>.<module>.<key>`.
type resolved struct {
Node string `json:"node"`
@@ -91,9 +105,10 @@ type resolved struct {
// Inputs is what the shadow composes from, read from the bus.
type Inputs struct {
// Nodes is every machine seen: in a node agent's have or the placement.
// Nodes is every machine seen: in a node agent's have or a standing assignment.
Nodes []string
// Assigned is what the seat's placement puts on each machine, sorted (issue 774: never the controller's assignments).
// Assigned is each machine's standing assignments and what the seat's placement puts there, sorted: while both
// stand (ADR 0549 §5) a module either names keeps its membership.
Assigned map[string][]string
// Placed is each node's placement as the seat mesh-placement holds it (ADR 0567 §1); empty before any is read.
// Where it is, a seat held once for the mesh is held where placement says, not wherever a module claiming it runs.
@@ -233,11 +248,26 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
NoFingerprint: map[string]string{}, Running: map[string]catalogue.Manifest{},
Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
nodes := map[string]bool{}
// **The seat's placement alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's
// assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed
// read: nothing is placed, so after take-over the list composed past it is not sealed.
// **After the lease write the assignments place nothing** (novox/hq ADR 0549 §3, §6): they are frozen then, so a
// module the catalogue takes off a node would keep its account, grant, membership and user-list entry. Once the lease
// is handed away and the placement is read, the placement alone places modules, as the assigner's Placed does with
// the same lease reading (buscompose.LeaseHandedAway); before, or with no placement on the bus, the two are unioned.
byAssignment, busByAssignment := map[string][]string{}, ""
for _, raw := range each(r, ReadAssignments, &in.Gaps) {
var a assignment
if json.Unmarshal(raw, &a) != nil || !a.stands() || a.Node == "" || a.Module == "" {
continue
}
byAssignment[a.Node] = append(byAssignment[a.Node], a.Module)
if a.Module == BusModule {
busByAssignment = a.Node
}
}
// The seat's placement (ADR 0567 §9): what it places is composed beside the assignments, and the seats held once
// for the mesh are held where it says. Not on the bus yet is a standing gap, never a failed read, so the list is still
// composed and sealed from the assignments alone.
if keys, err := r.Keys(ReadPlacement); err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true})
in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement, Why: "not read yet: the assignments alone place modules (" + err.Error() + ")"})
} else {
// Each node's keys in either form, assembled by the seat's one reader (delivery.PlacementsOf; research 072
// row 37): its node key and a key per module, or its legacy key `placement.<node>`.
@@ -264,6 +294,17 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
in.Placed[node] = p
}
}
away, _ := leaseHandedAway(r)
if away && len(in.Placed) > 0 {
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "not used: the controller's lease is handed away, " +
"so the seat's placement alone places modules (ADR 0549 §3, §6)"})
} else {
for node, ms := range byAssignment {
nodes[node] = true
in.Assigned[node] = append(in.Assigned[node], ms...)
}
in.BusNode = busByAssignment
}
for node, p := range in.Placed {
nodes[node] = true
for _, m := range p.Assigned() {
@@ -287,7 +328,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
nodes[node] = true
}
}
in.Gaps = append(in.Gaps, Gap{Input: "machines", Why: "read from the node agents' haves and the placement: a machine with neither is not seen"})
in.Gaps = append(in.Gaps, Gap{Input: "machines", Why: "read from the node agents' haves and the assignments: a machine with neither is not seen"})
for n := range nodes {
in.Nodes = append(in.Nodes, n)
}
+229 -30
View File
@@ -3,6 +3,7 @@ package grants
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
@@ -26,40 +27,231 @@ const ToolLeaseCheck = "grants_lease_check"
// Controller is the module and the seat the check is about.
const Controller = "mesh-controller"
// **The judge is mesh-sdk's** (catalogue.Dependencies, catalogue.Judge; issue 774): the same judgement the three
// repositories' manifest tests run through catalogue.ManifestRefusals (ADR 0571 §13), re-exported here by its names.
// The kinds of a dependency on the controller, as a manifest declares them.
const (
DependsRead = catalogue.DependsRead
DependsInvoke = catalogue.DependsInvoke
DependsConsume = catalogue.DependsConsume
DependsUses = catalogue.DependsUses
DependsWaits = catalogue.DependsWaits
PhaseBefore = catalogue.PhaseBefore
PhaseWrite = catalogue.PhaseWrite
DependsRead = "read"
DependsInvoke = "invoke"
DependsConsume = "consume"
DependsUses = "uses"
DependsWaits = "waits"
)
type (
// Dependency, Kept, Verdict and Coverage are mesh-sdk's.
Dependency = catalogue.Dependency
Kept = catalogue.KeptDependency
Verdict = catalogue.Verdict
Coverage = catalogue.Coverage
// The phases of the lease write's check (ADR 0571 §13): before the lease write's own delivery the whole named list is
// allowed; that delivery drops every entry but the three kept to the write itself, and its check runs with those alone.
const (
PhaseBefore = "before-delivery"
PhaseWrite = "write"
)
// KeepList and EndedBy are mesh-sdk's named list and the records that ended what it no longer keeps.
var (
KeepList = catalogue.KeepList
EndedBy = catalogue.EndedBy
)
// Dependencies are every declaration on the controller in the manifests given (catalogue.Dependencies).
func Dependencies(manifests map[string]catalogue.Manifest) []Dependency {
return catalogue.Dependencies(manifests)
// Dependency is one declaration of a manifest other than the controller's on the controller: a read of one of its
// states, an invoke of one of its verbs or tools, a consume or a wait on one of its events, or a send to its seat.
type Dependency struct {
Module string `json:"module"`
// By is the account declaring it when it is not the module's own: `process <id>` or `tools <binary>`.
By string `json:"by,omitempty"`
Kind string `json:"kind"`
What string `json:"what"`
// Kept is the record that keeps it until the lease write, from the named list; empty when none does.
Kept string `json:"kept,omitempty"`
// ToTheWrite says it is one of the three kept to the write itself, which the lease write's own delivery keeps.
ToTheWrite bool `json:"to_the_write,omitempty"`
}
// Judge runs the lease write's check in one phase (catalogue.Judge).
// Kept is one entry of the named, temporary list ADR 0571 §13 allows: a read, invoke or consume another accepted record
// keeps until the lease write, with that record. ToTheWrite marks the three kept to the write itself.
type Kept struct {
Module, Kind, What, Record string
ToTheWrite bool
}
// KeepList is the named list. Each entry names the clause that keeps it; a dependency not on it fails the check. The
// third entry kept to the write, the bus node agent's following of the controller's user list (ADR 0571 §8, ADR 0547
// §8), is no manifest's declaration but the node-engine's own, so no manifest matches it; it stays named here so the
// list reads as the record has it.
var KeepList = []Kept{
// Kept to the write itself (ADR 0571 §4, §8, §13).
{Module: "mesh-delivery-builder", Kind: DependsConsume, What: "mesh-controller.artifacts-gone", Record: "ADR 0571 §4, §13", ToTheWrite: true},
{Module: "artifact-store-tools", Kind: DependsInvoke, What: "seat:mesh-controller.collect", Record: "ADR 0571 §4, §13", ToTheWrite: true},
{Module: "node-engine", Kind: DependsRead, What: "the controller's user list of the bus module's composition", Record: "ADR 0571 §8, §13; ADR 0547 §8", ToTheWrite: true},
// Kept until the lease write's own delivery.
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.placement-export", Record: "ADR 0567 §7"},
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.assign", Record: "ADR 0567 §8"},
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.unassign", Record: "ADR 0567 §8"},
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.pin", Record: "ADR 0567 §8"},
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.unpin", Record: "ADR 0567 §8"},
{Module: "mesh-delivery-assigner", Kind: DependsRead, What: "mesh-controller.assignments", Record: "ADR 0567 §8"},
{Module: "mesh-delivery-assigner", Kind: DependsRead, What: "mesh-controller.composed", Record: "ADR 0571 §8"},
{Module: "mesh-grants", Kind: DependsRead, What: "mesh-controller.assignments", Record: "ADR 0549 §5"},
{Module: "mesh-grants", Kind: DependsConsume, What: "mesh-controller.condition-raised", Record: "ADR 0564 §9"},
{Module: "mesh-grants", Kind: DependsConsume, What: "mesh-controller.condition-cleared", Record: "ADR 0564 §9"},
{Module: "messenger", Kind: DependsConsume, What: "mesh-controller.condition-raised", Record: "ADR 0564 §9"},
{Module: "messenger", Kind: DependsConsume, What: "mesh-controller.condition-cleared", Record: "ADR 0564 §9"},
{Module: "messenger", Kind: DependsConsume, What: "mesh-controller.condition-changed", Record: "ADR 0564 §9"},
{Module: "messenger", Kind: DependsInvoke, What: "seat:mesh-controller.conditions", Record: "ADR 0564 §9"},
{Module: "mesh-watcher", Kind: DependsConsume, What: "mesh-controller.doctor-heartbeat", Record: "ADR 0564 §9"},
{Module: "mesh-conditions", Kind: DependsRead, What: "mesh-controller.conditions", Record: "ADR 0564 §9"},
{Module: "mesh-config", Kind: DependsRead, What: "mesh-controller.declared-config", Record: "ADR 0565 §4"},
{Module: "mesh-config", Kind: DependsRead, What: "mesh-controller.predefined", Record: "ADR 0565 §4"},
{Module: "mesh-healers", Kind: DependsRead, What: "mesh-controller.hand-acts", Record: "ADR 0565 §11"},
{Module: "mesh-hand-acts", Kind: DependsRead, What: "mesh-controller.hand-acts", Record: "ADR 0565 §11"},
}
// EndedBy names, for a declaration no entry keeps, the record that already ended it, so a refusal says where it moved.
// The controller's `grants` is one: ADR 0571 §9 has no manifest read it from L7's delivery, which changes ADR 0567 §10
// and ADR 0547 §5's shadow compare, so neither the assigner's nor the grants part's read of it is kept.
var EndedBy = map[string]string{
DependsRead + " mesh-controller.grants": "ADR 0571 §9: no manifest reads the controller's grants; the grants part's key is the one read",
DependsInvoke + " seat:mesh-controller.hand-act": "ADR 0565 §11: every recorder calls mesh-hand-acts",
DependsInvoke + " seat:mesh-controller.warranted": "ADR 0565 §11: every recorder calls mesh-hand-acts",
DependsInvoke + " seat:mesh-controller.root-free": "ADR 0571 §11: each machine's engine answers root-free (seat:node-engine.root-free@machine)",
DependsInvoke + " seat:mesh-controller.held": "ADR 0571 §2: held is the seat mesh-artifact-sweep's",
DependsRead + " mesh-controller.declared-packages": "ADR 0565 §6: each node agent writes its own node's packages",
}
// keptFor is the list's entry for one dependency, or nil.
func keptFor(d Dependency) *Kept {
for i, k := range KeepList {
if k.Module == d.Module && k.Kind == d.Kind && k.What == d.What {
return &KeepList[i]
}
}
return nil
}
// onController says whether a name `<owner>.<rest>` is the controller's.
func onController(name string) bool { return strings.HasPrefix(name, Controller+".") }
// invokesController says whether an invoke names a verb of the controller's seat or a tool of its module. `*` names none:
// it is the console's grant to every tool, judged by no record as a dependency.
func invokesController(invoke string) bool {
if seat, _, ok := catalogue.SeatInvoke(invoke); ok {
return seat == Controller
}
return onController(invoke)
}
// Dependencies are every declaration on the controller in the manifests given, by module, each with the list's entry
// that keeps it. The controller's own manifest is not judged, nor a read of its lease key: it names the lease's holder,
// whoever that is, and parts read it to learn that the write happened (ADR 0571 §13).
func Dependencies(manifests map[string]catalogue.Manifest) []Dependency {
var out []Dependency
add := func(module, by, kind, what string) {
d := Dependency{Module: module, By: by, Kind: kind, What: what}
if k := keptFor(d); k != nil {
d.Kept, d.ToTheWrite = k.Record, k.ToTheWrite
}
if !slices.Contains(out, d) {
out = append(out, d)
}
}
for name, m := range manifests {
if name == Controller || m.Module == Controller {
continue
}
for _, r := range m.Reads {
if onController(r) && r != buscompose.ControllerLease {
add(name, "", DependsRead, r)
}
}
for _, i := range m.Invokes {
if invokesController(i) {
add(name, "", DependsInvoke, i)
}
}
for _, c := range m.Consumes {
if onController(c) {
add(name, "", DependsConsume, c)
}
}
for _, w := range m.Waits {
if onController(w) {
add(name, "", DependsWaits, w)
}
}
for _, u := range m.Uses {
if u == Controller {
add(name, "", DependsUses, u)
}
}
for _, a := range m.BusAccounts() {
by := "process " + a.ID
if a.Tools {
by = "tools " + a.ID
}
for _, i := range a.Bus.Invokes {
if invokesController(i) {
add(name, by, DependsInvoke, i)
}
}
}
}
sort.Slice(out, func(i, j int) bool {
a, b := out[i], out[j]
if a.Module != b.Module {
return a.Module < b.Module
}
if a.Kind != b.Kind {
return a.Kind < b.Kind
}
if a.What != b.What {
return a.What < b.What
}
return a.By < b.By
})
return out
}
// Verdict is the lease write's check in one phase.
type Verdict struct {
Phase string `json:"phase"`
Passes bool `json:"passes"`
Reasons []string `json:"reasons"`
}
// Coverage is how much of the indexer the dependencies were derived from: the manifests read, the modules the indexer
// holds (-1 when its keys could not be listed, which Unread then names), and each manifest that could not be read.
type Coverage struct {
Read, Indexed int
Unread []Gap
}
// Judge runs the lease write's check over the dependencies (ADR 0571 §13) in one phase: before the lease write's own
// delivery, every dependency must be on the named list; at the write, only the three kept to the write itself may stay.
// It passes only over every module the indexer holds: no manifest read, fewer read than the indexer holds, or one that
// could not be read fails it, since what was not read could hide a dependency. A lease already handed away leaves
// nothing to write.
func Judge(deps []Dependency, phase string, cov Coverage, handedAway bool, leaseSays string) Verdict {
return catalogue.Judge(deps, phase, cov, handedAway, leaseSays)
v := Verdict{Phase: phase, Reasons: []string{}}
if handedAway {
v.Reasons = append(v.Reasons, "there is nothing to write: "+leaseSays)
}
switch {
case cov.Read == 0:
v.Reasons = append(v.Reasons, fmt.Sprintf("no manifest was read from the indexer (it holds %d modules): with nothing read, nothing on the controller can be judged absent", max(cov.Indexed, 0)))
case cov.Indexed >= 0 && cov.Read < cov.Indexed:
v.Reasons = append(v.Reasons, fmt.Sprintf("only %d of the %d modules the indexer holds had their manifest read, so what the others declare on the controller is not known", cov.Read, cov.Indexed))
}
for _, d := range deps {
who := d.Module
if d.By != "" {
who += " (" + d.By + ")"
}
switch {
case d.Kept == "":
why := fmt.Sprintf("%s declares %s %s, which no record keeps until the lease write (ADR 0571 §13)", who, d.Kind, d.What)
if ended := EndedBy[d.Kind+" "+d.What]; ended != "" {
why += "; ended by " + ended
}
v.Reasons = append(v.Reasons, why)
case phase == PhaseWrite && !d.ToTheWrite:
v.Reasons = append(v.Reasons, fmt.Sprintf("%s declares %s %s, kept by %s only until the lease write's own delivery, which drops it", who, d.Kind, d.What, d.Kept))
}
}
for _, g := range cov.Unread {
v.Reasons = append(v.Reasons, fmt.Sprintf("%s could not be read (%s), so what it declares on the controller is not known", g.Input, g.Why))
}
v.Passes = len(v.Reasons) == 0
return v
}
// LeaseNow is who holds the lease and what follows from it.
@@ -81,6 +273,7 @@ var (
"mesh-healers writes each act it would take in `would`, and acts on none (ADR 0564 §8)",
"the artifact sweep lets go of nothing; the controller's take-in sweeps and says `artifacts-gone` (ADR 0571 §4)",
"the watchman only lists dead letters; the controller keeps DEAD_LETTERS through its consumer of DEAD_LETTER_NOTICES (ADR 0564 §5)",
"placement proposals take only assignments, pins and config values, and the assigner keeps the controller's assignments in step (ADR 0567 §8, §9)",
"the controller keeps composing for the bus node, whose agent follows its user list, so the hand-backs of config, conditions, placement and the user list still answer (ADR 0547 §8, ADR 0564 §9, ADR 0565 §4, ADR 0567 §11)",
}
DoesHandedAway = []string{
@@ -220,6 +413,8 @@ type LeaseAnswer struct {
Streams []StreamUse `json:"streams,omitempty"`
Idle []string `json:"streams_idle,omitempty"`
StreamsError string `json:"streams_unread,omitempty"`
// Assignments is how many of the controller's assignments stand, which it publishes memberships and grants for.
Assignments int `json:"assignments_standing"`
// Check is the dry run at the write (the three alone), Before the same over the whole list before the write's
// delivery.
Check Verdict `json:"check"`
@@ -255,16 +450,20 @@ func (p *Part) LeaseCheck() (any, error) {
out.Unkept[d.Module] = append(out.Unkept[d.Module], d.Kind+" "+d.What)
}
}
var gaps []Gap
for _, raw := range each(p.Read, ReadAssignments, &gaps) {
var a assignment
if json.Unmarshal(raw, &a) == nil && a.stands() {
out.Assignments++
}
}
if used, idle, err := Duties(p.Call); err != nil {
out.StreamsError = err.Error()
} else {
out.Streams, out.Idle = used, idle
}
// Only the indexer's own modules count as read: a manifest the mesh provides itself must not stand in for one missing.
cov := Coverage{Indexed: -1}
for _, g := range out.Unread {
cov.Unread = append(cov.Unread, catalogue.Unread{Input: g.Input, Why: g.Why})
}
cov := Coverage{Indexed: -1, Unread: out.Unread}
if keys, err := p.Read.Keys(ReadModules); err == nil {
cov.Indexed = len(keys)
for _, k := range keys {
@@ -1,945 +0,0 @@
package grants
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
)
// **The lease write, a verb of the seat mesh-grants** (novox/hq ADR 0582, issue 774). `lease-write` takes a why and is
// refused, naming the check, until every precondition of clause 2 reads green; otherwise the operator is asked once, on
// a channel that proves who answered, with the checks shown and their digest bound to the yes (clause 1). On the
// warrant the checks are read again, and then, each step read back before the next (clause 3): the controller's service
// is stopped on its node and read back stopped; the bus's user list is composed without the controller's write on the
// lease (buscompose.Records.LeaseCut) and waited on until the bus node's have says it loaded it; the lease bucket's
// expiry is removed through the bus module's tool; the key `holder` names mesh-grants with the operator, the why, the
// ask and the time; `lease-handed` is said, the act recorded through mesh-hand-acts.warranted, and `removal-due` is
// written in the state `lease` with a day to live. A lease altered after it is said as `lease-altered`, once, and
// nothing is written again until the operator asks (clause 4). `lease-hand-back` reverses it for a day (clause 5). The
// expiry marker of `removal-due`, and nothing else, starts the day-later removal of the controller's consumer of
// DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (clause 6). Every step waits on an event (a warrant, the bus node's
// have, a marker), never on a timer.
// The verbs, events and states of the lease write.
const (
ActLeaseWrite = catalogue.GrantsVerbLeaseWrite
ActLeaseHandBack = catalogue.GrantsVerbLeaseHandBack
EventLeaseHanded = catalogue.GrantsEventLeaseHanded
// EventLeaseAltered and EventRemovalFailed are the seat's other two events.
EventLeaseAltered = catalogue.GrantsEventLeaseAltered
EventRemovalFailed = catalogue.GrantsEventRemovalFailed
// StateLease is the part's state with a time to live per key, holding KeyRemovalDue alone.
StateLease = "lease"
KeyRemovalDue = "removal-due"
// KeyLease is the lease write's record in StateTakeover: the cut, the write, the removal and every act.
KeyLease = "lease"
// ToolRemovalDue is the tool that writes removal-due again with a short time to live, the recovery for a marker
// missed while the part was down.
ToolRemovalDue = "grants_removal_due"
)
// RemovalAfter is how long after the write the controller's consumer and stream are removed (ADR 0582 clause 6).
const RemovalAfter = 24 * time.Hour
// The calls the lease write makes, as the manifest's invokes name them.
const (
// CallServiceStop, CallServiceStart and CallServiceStatus are the node seat's verbs, asked of the controller's node.
CallServiceStop = "seat:" + catalogue.ServiceManagerSeat + ".stop"
CallServiceStart = "seat:" + catalogue.ServiceManagerSeat + ".start"
CallServiceStatus = "seat:" + catalogue.ServiceManagerSeat + ".status"
// CallBucketExpiry sets and restores a bucket's expiry, recording the act itself; CallConsumerRemove and
// CallStreamRemove are the removal's two tools.
CallBucketExpiry = "nats.nats_bucket_expiry"
CallConsumerRemove = "nats.nats_consumer_remove"
CallStreamRemove = "nats.nats_stream_remove"
// CallRecordHandAct records the removals as hand acts (ADR 0582 clause 6, ADR 0565 §11).
CallRecordHandAct = "seat:" + handacts.Seat + ".hand-act"
// The checks' calls (clause 2).
CallNodeTrusts = "mesh-delivery-view.node_trusts"
CallConfigCompare = "seat:mesh-config.compare"
CallUpgrade = "seat:" + catalogue.WantsSeat + ".upgrade"
CallVaultStatus = "mesh-vault.vault_status"
CallIssue = "seat:issue-tracker.get"
)
// LeaseCalls are every call the lease write makes.
var LeaseCalls = []string{CallServiceStop, CallServiceStart, CallServiceStatus, CallBucketExpiry, CallConsumerRemove,
CallStreamRemove, CallRecordHandAct, CallNodeTrusts, CallConfigCompare, CallUpgrade, CallVaultStatus, CallIssue}
// ControllerUnit is the controller's service on its node.
const ControllerUnit = Controller + ".service"
// LeaseExpiry is the lease bucket's expiry a hand-back restores (buscompose.LeaseTTL).
var LeaseExpiry = buscompose.LeaseTTL
// RemovedTools are the bus module's tools the lease write and its removal need: the check reads them in its manifest.
var RemovedTools = []string{"nats_bucket_expiry", "nats_consumer_remove", "nats_stream_remove"}
// EarlierSteps are the issues of design 47's earlier steps the write waits on (ADR 0582 clause 2: issues 1952-1955 delivered).
var EarlierSteps = []uint64{1952, 1953, 1954, 1955}
// LeaseBus is the process's hand on the controller's lease: the key `holder` put and deleted, and the bucket's expiry
// read. Nil in the tools, which ask the process.
type LeaseBus interface {
PutHolder(value []byte) error
DeleteHolder() error
Expiry() (time.Duration, error)
}
// TTLStore is a Store that writes a key with its own time to live (the state lease).
type TTLStore interface {
PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error)
}
// SeatSayer says one of the seat's events, `mesh.seat.mesh-grants.event.<event>`.
type SeatSayer func(event string, body any) error
// Check is one precondition as read: its name, whether it reads green, and what it says.
type Check struct {
Name string `json:"name"`
Green bool `json:"green"`
Says string `json:"says"`
}
// checksDigest is the digest of the checks as shown: what the yes binds.
func checksDigest(cs []Check) string {
raw, _ := json.Marshal(cs)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// LeaseAct is one lease-write or lease-hand-back: asked, answered and each step it reached.
type LeaseAct struct {
Act string `json:"act"`
By string `json:"by"`
Why string `json:"why"`
At time.Time `json:"at"`
Ask string `json:"ask"`
// Checks are the preconditions as the ask showed them, and Digest their digest the yes binds.
Checks []Check `json:"checks,omitempty"`
Digest string `json:"digest,omitempty"`
Raised bool `json:"raised,omitempty"`
// Step is where it stands: asked; then on the warrant stopped, cut, loaded, expiry, written, done for a write and
// restored, uncut, done for a hand-back; or declined, expired, refused, failed.
Step string `json:"step"`
Said string `json:"said,omitempty"`
// List is the digest of the list composed at the cut (or without it, for a hand-back), awaited loaded.
List string `json:"list,omitempty"`
Done time.Time `json:"done,omitzero"`
}
// The steps an act reaches.
const (
StepAsked = "asked"
StepStopped = "stopped"
StepCut = "cut"
StepWritten = "written"
StepRestored = "restored"
StepUncut = "uncut"
StepDone = "done"
StepDeclined = "declined"
StepExpired = "expired"
StepRefused = "refused"
StepFailed = "failed"
)
// LeaseRecord is the key `lease` of StateTakeover.
type LeaseRecord struct {
V int `json:"v"`
// Cut composes the list with buscompose.Records.LeaseCut: from the write's second step until a hand-back's.
Cut bool `json:"cut"`
// Node is the controller's node, from the lease's holder at the write.
Node string `json:"node,omitempty"`
// Handed is when the key holder was written, zero when not (or handed back); Ask the write's ask.
Handed time.Time `json:"handed,omitzero"`
Ask string `json:"ask,omitempty"`
// Removed is when the day-later removal was done; zero until then.
Removed time.Time `json:"removed,omitzero"`
// Altered is what lease-altered said, empty while the lease reads as written.
Altered string `json:"altered,omitempty"`
// Open is the act asked or under way.
Open *LeaseAct `json:"open,omitempty"`
// Acts are the acts ended, newest last.
Acts []LeaseAct `json:"acts"`
}
func (p *Part) readLease() (LeaseRecord, error) {
r := LeaseRecord{V: 1, Acts: []LeaseAct{}}
raw, _, found, err := p.Store.Get(StateTakeover, KeyLease)
if err != nil {
return r, fmt.Errorf("the lease write's record cannot be read: %w", err)
}
if found {
if err := json.Unmarshal(raw, &r); err != nil {
return r, fmt.Errorf("the lease write's record does not read: %w", err)
}
}
return r, nil
}
func (p *Part) writeLease(r LeaseRecord) error {
if len(r.Acts) > keptActs {
r.Acts = r.Acts[len(r.Acts)-keptActs:]
}
if r.Acts == nil {
r.Acts = []LeaseAct{}
}
data, _ := json.Marshal(r)
if _, err := p.Store.Put(StateTakeover, KeyLease, data); err != nil {
return fmt.Errorf("the lease write's record cannot be written: %w", err)
}
return nil
}
// leaseCut says whether the list is composed with the cut.
func (p *Part) leaseCut() bool {
r, err := p.readLease()
return err == nil && r.Cut
}
// leaseHanded says whether the key holder names mesh-grants by the part's own write.
func (p *Part) leaseHanded() bool {
r, err := p.readLease()
return err == nil && !r.Handed.IsZero()
}
// end closes the open act with its step and why.
func (r *LeaseRecord) end(step, said string, at time.Time) {
if r.Open == nil {
return
}
a := *r.Open
a.Step, a.Said, a.Done = step, said, at
r.Acts = append(r.Acts, a)
r.Open = nil
}
// LeaseCheckNames are the checks LeaseChecks reads, in its order.
var LeaseCheckNames = []string{"lease-check", "earlier-steps", "node-trusts", "config-compare", "upgrade-compare",
"keeper-take-over", "vault-minting", "bus-tools"}
// LeaseChecks reads every precondition of the write (ADR 0582 clause 2), each named, in one order.
func (p *Part) LeaseChecks() []Check {
var out []Check
add := func(name string, green bool, says string) {
out = append(out, Check{Name: name, Green: green, Says: says})
}
// The lease check at the write, with the three kept to it alone (ADR 0571 §13).
if raw, err := p.LeaseCheck(); err != nil {
add("lease-check", false, "the lease check could not run: "+err.Error())
} else if a, ok := raw.(LeaseAnswer); !ok {
add("lease-check", false, "the lease check answered nothing it reads")
} else {
add("lease-check", a.Check.Passes, strings.Join(a.Check.Reasons, "; "))
}
// Design 47's earlier steps delivered.
var open []string
for _, n := range EarlierSteps {
raw, err := p.Call.Call(CallIssue, map[string]any{"number": n, "brief": true})
var is struct {
Status string `json:"status"`
}
switch {
case err != nil:
open = append(open, fmt.Sprintf("issue %d cannot be read (%v)", n, err))
case json.Unmarshal(unwrap(raw), &is) != nil || is.Status != "resolved":
open = append(open, fmt.Sprintf("issue %d is %q, not resolved", n, is.Status))
}
}
add("earlier-steps", len(open) == 0, said(open, "issues 1952 to 1955 are resolved"))
// Every node trusts the vault's key.
if raw, err := p.Call.Call(CallNodeTrusts, map[string]any{}); err != nil {
add("node-trusts", false, "node_trusts cannot be read: "+err.Error())
} else {
var t struct {
Every bool `json:"every"`
NotYet []string `json:"not_yet"`
}
_ = json.Unmarshal(unwrap(raw), &t)
add("node-trusts", t.Every, said(prefixed("not yet: ", t.NotYet), "every node's record trusts the vault's key"))
}
// mesh-config's compare empty and its take-over done.
if raw, err := p.Call.Call(CallConfigCompare, map[string]any{}); err != nil {
add("config-compare", false, "mesh-config's compare cannot be read: "+err.Error())
} else {
var c struct {
Source string `json:"source"`
Count int `json:"count"`
}
_ = json.Unmarshal(unwrap(raw), &c)
add("config-compare", c.Count == 0 && c.Source == "indexer",
fmt.Sprintf("mesh-config resolves from %q with %d differences", c.Source, c.Count))
}
// The upgrade compare empty.
if raw, err := p.Call.Call(CallUpgrade, map[string]any{"compare": "true"}); err != nil {
add("upgrade-compare", false, "mesh-wants' upgrade compare cannot be read: "+err.Error())
} else {
var u struct {
Empty bool `json:"empty"`
Splits []json.RawMessage `json:"splits"`
}
_ = json.Unmarshal(unwrap(raw), &u)
add("upgrade-compare", u.Empty, fmt.Sprintf("%d policy splits", len(u.Splits)))
}
// The condition keeper's take-over done.
raw, found, err := p.Read.Get(ReadTakeover, health.TakeoverKey)
t, terr := health.DecodeTakeover(raw, found)
switch {
case err != nil:
add("keeper-take-over", false, "the keeper's take-over cannot be read: "+err.Error())
case terr != nil:
add("keeper-take-over", false, "the keeper's take-over does not read: "+terr.Error())
default:
add("keeper-take-over", t.Follow != health.FollowController, "conditions follow "+t.Follow)
}
// The vault minting with nothing held back.
if raw, err := p.Call.Call(CallVaultStatus, map[string]any{}); err != nil {
add("vault-minting", false, "vault_status cannot be read: "+err.Error())
} else {
var v struct {
Ready bool `json:"ready"`
Minting struct {
On bool `json:"on"`
HeldBack map[string]string `json:"held_back"`
} `json:"minting"`
}
_ = json.Unmarshal(unwrap(raw), &v)
add("vault-minting", v.Ready && v.Minting.On && len(v.Minting.HeldBack) == 0,
fmt.Sprintf("ready %v, minting %v, %d held back", v.Ready, v.Minting.On, len(v.Minting.HeldBack)))
}
// The bus module's tools for the expiry and the removal live: its manifest at the indexer's trunk lists them.
in := ReadInputs(p.Read, &p.cache)
var missing []string
for _, tool := range RemovedTools {
if !slices.Contains(in.Manifests[BusModule].Tools, tool) {
missing = append(missing, tool)
}
}
add("bus-tools", len(missing) == 0, said(prefixed("the bus module lists no ", missing), "the bus module lists its three tools"))
return out
}
func said(problems []string, ok string) string {
if len(problems) == 0 {
return ok
}
return strings.Join(problems, "; ")
}
func prefixed(prefix string, items []string) []string {
if len(items) == 0 {
return nil
}
return []string{prefix + strings.Join(items, ", ")}
}
// notGreen is every check not green, `<name>: <says>`.
func notGreen(cs []Check) []string {
var out []string
for _, c := range cs {
if !c.Green {
out = append(out, c.Name+": "+c.Says)
}
}
return out
}
// changedChecks names each check whose reading differs from the one shown.
func changedChecks(shown, now []Check) []string {
was := map[string]Check{}
for _, c := range shown {
was[c.Name] = c
}
var out []string
for _, c := range now {
if w, ok := was[c.Name]; !ok || w != c {
out = append(out, c.Name+" (now: "+c.Says+")")
}
delete(was, c.Name)
}
for name := range was {
out = append(out, name+" (no longer read)")
}
slices.Sort(out)
return out
}
// LeaseWrite is the verb lease-write: refused, naming each check not green, or asked of the operator by the part's process.
func (p *Part) LeaseWrite(caller, why string) (any, error) {
return p.leaseRequest(ActLeaseWrite, caller, why)
}
// LeaseHandBack is the verb lease-hand-back.
func (p *Part) LeaseHandBack(caller, why string) (any, error) {
return p.leaseRequest(ActLeaseHandBack, caller, why)
}
func (p *Part) leaseRequest(act, caller, why string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("%s needs a why, shown to the operator and kept", act)
}
r, err := p.readLease()
if err != nil {
return nil, err
}
if r.Open != nil {
return nil, fmt.Errorf("%s is already under way: the ask %s stands at %s", r.Open.Act, r.Open.Ask, r.Open.Step)
}
var checks []Check
switch act {
case ActLeaseWrite:
if !r.Handed.IsZero() && r.Altered == "" {
return nil, errors.New("the lease is handed to mesh-grants already: nothing to write")
}
checks = p.LeaseChecks()
if bad := notGreen(checks); len(bad) > 0 {
return nil, fmt.Errorf("lease-write is refused until every check reads green (ADR 0582 clause 2): %s", strings.Join(bad, "; "))
}
case ActLeaseHandBack:
if refused := handBackRefused(r); refused != "" {
return nil, errors.New(refused)
}
}
a := LeaseAct{Act: act, By: caller, Why: why, At: p.now(), Checks: checks, Digest: checksDigest(checks), Step: StepAsked}
a.Ask = act + "-" + p.now().Format("20060102t150405") + "-" + p.nonce()
r.Open = &a
if err := p.writeLease(r); err != nil {
return nil, err
}
_, _ = p.AskCompose(caller)
return map[string]any{"asked": a.Ask, "checks": checks, "says": "the part's process asks the operator on their channel; " +
"only the operator's yes runs it, after the checks are read again. grants_lease_check shows the lease, and " +
"the record is the key lease of the state takeover"}, nil
}
// handBackRefused is why a hand-back cannot be asked, or empty.
func handBackRefused(r LeaseRecord) string {
switch {
case !r.Removed.IsZero():
return "lease-hand-back is refused: the day-later removal ran, so the controller would start without its " +
"consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (ADR 0582 clause 5)"
case r.Handed.IsZero() && !r.Cut:
return "lease-hand-back is refused: the lease is not handed to mesh-grants, so there is nothing to hand back"
}
return ""
}
// EventBucketExpirySet is the bus module's event of a bucket's expiry set, which the part consumes: a lease bucket's
// expiry changed is heard, never polled.
const EventBucketExpirySet = BusModule + ".bucket-expiry-set"
// IsBucketExpirySet says an event heard is the bus module's bucket-expiry-set.
func IsBucketExpirySet(event string) bool {
return event == EventBucketExpirySet || event == "mesh.mod."+BusModule+".event.bucket-expiry-set"
}
// isLeaseAsk says an ask is the lease write's.
func isLeaseAsk(id string) bool {
return strings.HasPrefix(id, ActLeaseWrite+"-") || strings.HasPrefix(id, ActLeaseHandBack+"-")
}
// leaseAskOf is the ask of an act, in the words the operator reads: what it does, the checks it showed, and the one
// option whose yes binds the act, the holder, the ask and the checks' digest (ADR 0582 clause 1).
func (p *Part) leaseAskOf(a LeaseAct) (asks.Ask, error) {
act := asks.Act{"verb": a.Act, "module": Module, "ask": a.Ask, "holder": Module, "checks": a.Digest}
bind, err := asks.ActDigest(act)
if err != nil {
return asks.Ask{}, err
}
var lines []string
for _, c := range a.Checks {
lines = append(lines, "✓ "+c.Name+": "+c.Says)
}
headline, does, stays := "Hand the controller's lease to mesh-grants?",
"The controller is stopped and the lease is handed to mesh-grants for good; hand-back stays open for a day.",
"The controller keeps its lease."
expl := "Every check of the lease write reads green. Your yes stops the controller, takes from it the right to write " +
"its lease, removes the lease's expiry and names mesh-grants as its holder. A controller started later fails on the lease."
if a.Act == ActLeaseHandBack {
headline, does, stays = "Give the controller its lease back?",
"The lease's expiry is restored, mesh-grants lets go of it, the controller gets its rights back and is started.",
"The lease stays with mesh-grants."
expl = "The lease was handed to mesh-grants less than a day ago. Your yes gives it back and starts the controller."
}
return asks.Ask{ID: a.Ask, Headline: headline, Explanation: expl, Who: asks.Operator,
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
Expires: a.At.Add(asks.ApproveLasts - time.Minute), OnExpiry: stays + " Nothing changes.", About: "mesh-grants-" + a.Act,
Details: "Asked by " + a.By + ": " + a.Why + "\n" + strings.Join(lines, "\n")}, nil
}
// raiseLease asks the operator the open act once; closes one past its expiry; catches up a warrant the process missed.
func (p *Part) raiseLease() error {
if p.Asker == nil {
return nil
}
r, err := p.readLease()
if err != nil || r.Open == nil || r.Open.Step != StepAsked {
return err
}
a := r.Open
if !a.Raised {
ask, err := p.leaseAskOf(*a)
if err != nil {
return err
}
if err := p.Asker.Ask(ask); err != nil {
if p.Forget != nil {
_ = p.Forget(a.Ask)
}
_ = p.Asker.Cancel(a.Ask)
r.end(StepRefused, "the operator could not be asked: "+err.Error(), p.now())
_ = p.writeLease(r)
return fmt.Errorf("the operator could not be asked the %s: %v", a.Act, err)
}
a.Raised = true
return p.writeLease(r)
}
_, w, err := p.Asker.Where(a.Ask)
if err != nil {
return fmt.Errorf("the router's record of the ask %s cannot be read: %v", a.Ask, err)
}
if w != nil {
h, herr := p.Asker.Take(*w, p.Warranted)
if h == "" && herr != nil {
return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", a.Ask, herr)
}
p.Said(h, *w, herr)
return nil
}
if p.now().After(a.At.Add(asks.ApproveLasts + askMargin)) {
r.end(StepExpired, "the ask expired: nothing was written", p.now())
return p.writeLease(r)
}
return nil
}
// leaseWarranted is the operator's answer to a lease act: a no or a refusal writes nothing; a yes runs the act.
func (p *Part) leaseWarranted(o asks.Option, w asks.Warrant) error {
r, err := p.readLease()
if err != nil {
return err
}
if r.Open == nil || r.Open.Ask != w.Ask || r.Open.Step != StepAsked {
return fmt.Errorf("the warrant answers the ask %s, which is not the lease act asked: nothing is written", w.Ask)
}
if o.ID != "yes" {
r.end(StepDeclined, "the operator said no: nothing was written", p.now())
return p.writeLease(r)
}
a := r.Open
if a.Act == ActLeaseWrite {
// Read again on the warrant (ADR 0582 clauses 1, 2): the yes binds the checks shown, so any check that reads
// otherwise now (one no longer green among them) refuses it, named.
if now := p.LeaseChecks(); checksDigest(now) != a.Digest {
r.end(StepRefused, "refused on the warrant, the checks changed since the ask: "+strings.Join(changedChecks(a.Checks, now), "; "), p.now())
p.sayLeaseRefused(a.Act, r.Acts[len(r.Acts)-1].Said)
return p.writeLease(r)
}
} else if refused := handBackRefused(r); refused != "" {
r.end(StepRefused, refused, p.now())
return p.writeLease(r)
}
a.Step = "warranted"
if err := p.writeLease(r); err != nil {
return err
}
return p.advanceLease()
}
// sayLeaseRefused tells the operator a yes was refused, as the take-over's refusal is said.
func (p *Part) sayLeaseRefused(act, why string) {
if p.Say == nil {
return
}
_ = p.Say(asks.Condition{Key: "mesh-grants." + act + "-refused", Severity: asks.SeverityWarning,
Subject: asks.Scope{Scope: "module", ID: Module}, State: asks.StateRaised, Summary: act + " refused: " + why,
Headline: "The lease write was refused", Needs: "a look at why",
Explanation: asks.NeedsYou + "a look at why. You said yes and it was refused: " + why, Resolved: "the act is asked again and goes through"})
}
// advanceLease takes the open act as far as it can now: each step read back before the next, and a step that waits (the
// list loaded) left for the pass that hears the bus node's have. A step that fails ends the act there, saying why, and
// writes nothing more.
func (p *Part) advanceLease() error {
r, err := p.readLease()
if err != nil || r.Open == nil || r.Open.Step == StepAsked {
return err
}
a := r.Open
fail := func(why string) error {
r.end(StepFailed, why, p.now())
p.sayLeaseRefused(a.Act, why)
return p.writeLease(r)
}
if a.Act == ActLeaseHandBack {
return p.advanceHandBack(&r, fail)
}
switch a.Step {
case "warranted":
// 1. Stop the controller's service on its node, read back stopped.
node, why := p.controllerNode()
if node == "" {
return fail(why)
}
if stopped, why := p.service(CallServiceStop, node); !stopped {
return fail("the controller's service on " + node + " did not read back stopped (" + why + "): nothing more is written")
}
r.Node, a.Step = node, StepStopped
fallthrough
case StepStopped:
// 2. Compose the list without the controller's write on the lease.
r.Cut, a.Step = true, StepCut
if err := p.writeLease(r); err != nil {
return err
}
res := p.composeQuiet()
g, _, _ := p.Sealed()
if res.Sealed == nil || g.Digest == "" {
return fail("the list composed with the cut could not be written sealed: " + strings.Join(res.Errors, "; "))
}
if r, err = p.readLease(); err != nil || r.Open == nil {
return err
}
a = r.Open
a.List = g.Digest
if err := p.writeLease(r); err != nil {
return err
}
fallthrough
case StepCut:
// 3. Wait until the bus node's have says it loaded that list.
if !p.loaded(a.List) {
return nil
}
// 4. Remove the lease bucket's expiry.
if ok, why := p.setExpiry("none", *a); !ok {
return fail("the lease's expiry was not removed (" + why + "): the holder is not written")
}
// 5. Create the key holder naming mesh-grants.
h := buscompose.LeaseHolder{HandedTo: Module, HandedBy: "operator", Why: a.Why, Ask: a.Ask, At: p.now(), Host: r.Node}
raw, _ := json.Marshal(h)
if p.Lease == nil {
return fail("the part holds no hand on the lease here")
}
if err := p.Lease.PutHolder(raw); err != nil {
return fail("the key holder could not be written: " + err.Error())
}
r.Handed, r.Ask, r.Altered, a.Step = p.now(), a.Ask, "", StepWritten
if err := p.writeLease(r); err != nil {
return err
}
// 6. Say lease-handed, record the act, and write removal-due with a day to live.
p.saySeat(EventLeaseHanded, map[string]any{"holder": Module, "ask": a.Ask, "why": a.Why, "node": r.Node, "at": r.Handed})
p.recordWarrant(a.Ask)
if err := p.putRemovalDue(RemovalAfter); err != nil {
r.end(StepDone, "the lease is handed; removal-due could not be written ("+err.Error()+"): grants_removal_due writes it", p.now())
} else {
r.end(StepDone, "the lease is handed to mesh-grants; the removal is due in a day", p.now())
}
return p.writeLease(r)
}
return nil
}
// advanceHandBack restores the expiry, deletes holder and removal-due, composes without the cut, and once that list is
// loaded starts the controller (ADR 0582 clause 5).
func (p *Part) advanceHandBack(r *LeaseRecord, fail func(string) error) error {
a := r.Open
switch a.Step {
case "warranted":
if ok, why := p.setExpiry(LeaseExpiry.String(), *a); !ok {
return fail("the lease's expiry was not restored (" + why + "): the lease stays with mesh-grants")
}
if p.Lease == nil {
return fail("the part holds no hand on the lease here")
}
// The record first: the delete of removal-due is then heard as no removal (Handed is zero).
r.Handed, a.Step = time.Time{}, StepRestored
if err := p.writeLease(*r); err != nil {
return err
}
if err := p.Lease.DeleteHolder(); err != nil {
return fail("the key holder could not be deleted: " + err.Error())
}
if err := p.Store.Delete(StateLease, KeyRemovalDue); err != nil {
return fail("removal-due could not be deleted: " + err.Error())
}
r.Cut, a.Step = false, StepUncut
if err := p.writeLease(*r); err != nil {
return err
}
p.composeQuiet()
g, _, _ := p.Sealed()
fresh, err := p.readLease()
if err != nil || fresh.Open == nil {
return err
}
*r = fresh
a = r.Open
a.List = g.Digest
if err := p.writeLease(*r); err != nil {
return err
}
fallthrough
case StepUncut:
if !p.loaded(a.List) {
return nil
}
if started, why := p.service(CallServiceStart, r.Node); !started {
return fail("the controller's service on " + r.Node + " did not read back started (" + why + ")")
}
p.recordWarrant(a.Ask)
r.end(StepDone, "the lease is given back; the controller is started", p.now())
return p.writeLease(*r)
}
return nil
}
// controllerNode is the node the controller holds its lease on, from the lease's holder.
func (p *Part) controllerNode() (string, string) {
raw, found, err := p.Read.Get(ReadLease, LeaseKey)
if err != nil || !found {
return "", "the lease cannot be read, so the controller's node is not known: nothing is stopped"
}
var h buscompose.LeaseHolder
if json.Unmarshal(raw, &h) != nil || h.Host == "" || h.HandedTo != "" {
return "", "the lease names no controller's host, so the controller's node is not known: nothing is stopped"
}
return h.Host, ""
}
// service asks the node seat's verb of the controller's unit and reads it back with status: stopped for a stop, started
// for a start.
func (p *Part) service(verb, node string) (bool, string) {
unit := map[string]any{"unit": ControllerUnit}
if _, err := p.Call.Call(verb+"@"+node, unit); err != nil {
return false, err.Error()
}
raw, err := p.Call.Call(CallServiceStatus+"@"+node, unit)
if err != nil {
return false, "its status cannot be read: " + err.Error()
}
var st struct {
Active string `json:"ActiveState"`
}
_ = json.Unmarshal(unwrap(raw), &st)
if verb == CallServiceStart {
return st.Active == "active", "it reads " + st.Active
}
return st.Active == "inactive" || st.Active == "failed", "it reads " + st.Active
}
// loaded says the bus node's have names the list of that digest loaded from the grants part's key.
func (p *Part) loaded(digest string) bool {
l := ReadInputs(p.Read, &p.cache).Loaded
return digest != "" && l != nil && l.Source == delivery.GrantsUserListSource && l.Digest == digest
}
// setExpiry asks the bus module's tool to set the lease bucket's expiry, and reads it back from the bucket.
func (p *Part) setExpiry(maxAge string, a LeaseAct) (bool, string) {
raw, err := p.Call.Call(CallBucketExpiry, map[string]any{"bucket": buscompose.LeaseBucket, "max_age": maxAge,
"why": a.Act + ": " + a.Why, "warrant": a.Ask})
if err != nil {
return false, err.Error()
}
var set struct {
Now string `json:"now"`
}
_ = json.Unmarshal(unwrap(raw), &set)
if set.Now != maxAge {
return false, "the tool answered " + set.Now
}
if p.Lease != nil {
got, err := p.Lease.Expiry()
want, _ := time.ParseDuration(strings.Replace(maxAge, "none", "0s", 1))
if err != nil || got != want {
return false, fmt.Sprintf("the bucket reads back %s (%v)", got, err)
}
}
return true, ""
}
// putRemovalDue writes removal-due with its time to live.
func (p *Part) putRemovalDue(ttl time.Duration) error {
s, ok := p.Store.(TTLStore)
if !ok {
return errors.New("the part's store writes no time to live here")
}
raw, _ := json.Marshal(map[string]any{"due": p.now().Add(ttl), "written": p.now()})
_, err := s.PutTTL(StateLease, KeyRemovalDue, raw, ttl)
return err
}
// saySeat says one of the seat's events; a failure is logged, never the end of the act.
func (p *Part) saySeat(event string, body any) {
if p.SaySeat == nil {
return
}
if err := p.SaySeat(event, body); err != nil && p.Logf != nil {
p.Logf("the event %s could not be said: %v", event, err)
}
}
// leaseStep is the pass's part in the lease write: the open act asked or advanced, the lease watched once handed, and
// the removal run when its marker was heard.
func (p *Part) leaseStep(res *Result) {
for _, step := range []func() error{p.raiseLease, p.advanceLease, p.watchLease} {
if err := step(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
}
if p.removalHeard.Swap(false) {
if err := p.removal(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
}
}
// RemovalDueHeard is the follower's: the marker of removal-due was heard (its delete, by expiry or by a hand-back).
func (p *Part) RemovalDueHeard() { p.removalHeard.Store(true) }
// watchLease says lease-altered once when the key or the bucket's expiry differs from what the part wrote (ADR 0582
// clause 4); a hand-back under way is the part's own change and is not watched.
func (p *Part) watchLease() error {
r, err := p.readLease()
if err != nil || r.Handed.IsZero() || r.Altered != "" || (r.Open != nil && r.Open.Act == ActLeaseHandBack && r.Open.Step != StepAsked) {
return err
}
var what []string
raw, found, err := p.Read.Get(ReadLease, LeaseKey)
var h buscompose.LeaseHolder
switch {
case err != nil:
return nil // not read is no change: the next pass reads it again
case !found:
what = append(what, "the key holder is gone")
case json.Unmarshal(raw, &h) != nil || h.HandedTo != Module || h.Ask != r.Ask:
what = append(what, "the key holder names "+firstOf(h.HandedTo, h.Instance, "something else")+", not mesh-grants by "+r.Ask)
}
if p.Lease != nil {
if d, err := p.Lease.Expiry(); err == nil && d != 0 {
what = append(what, "the lease's bucket expires after "+d.String())
}
}
if len(what) == 0 {
return nil
}
r.Altered = strings.Join(what, "; ")
p.saySeat(EventLeaseAltered, map[string]any{"altered": r.Altered, "ask": r.Ask, "at": p.now()})
return p.writeLease(r)
}
func firstOf(s ...string) string {
for _, x := range s {
if x != "" {
return x
}
}
return ""
}
// removal is the day-later removal, started by the marker of removal-due alone (ADR 0582 clause 6): when the lease still
// names mesh-grants, the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS are removed, each read
// back by the bus module's tool and recorded as a hand act. A failure is said as removal-failed and removal-due is
// written again with a day.
func (p *Part) removal() error {
r, err := p.readLease()
if err != nil || r.Handed.IsZero() || !r.Removed.IsZero() {
return err // handed back (a hand-back deletes the key), or done already
}
if raw, found, err := p.Read.Get(ReadLease, LeaseKey); err != nil || !found {
return p.removalFailed(r, "the lease cannot be read, so it is not known to name mesh-grants")
} else if away, _ := buscompose.LeaseHandedAway(raw, true, nil); !away {
return p.removalFailed(r, "the lease no longer names mesh-grants")
}
why := "a day after the lease write " + r.Ask + " (ADR 0582 clause 6)"
var failed []string
if raw, err := p.Call.Call(CallConsumerRemove, map[string]any{"stream": buscompose.DeadLetterNoticesStream,
"consumer": buscompose.ControllerName, "why": why}); err != nil && !strings.Contains(err.Error(), "has no consumer") {
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES: "+err.Error())
} else if err == nil && !removed(raw) {
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES does not read back removed")
} else {
p.recordHandAct("removed the controller's consumer of DEAD_LETTER_NOTICES", why)
}
if raw, err := p.Call.Call(CallStreamRemove, map[string]any{"stream": buscompose.AssignmentsStream, "why": why}); err != nil {
failed = append(failed, "the stream ASSIGNMENTS: "+err.Error())
} else if !removed(raw) {
failed = append(failed, "the stream ASSIGNMENTS does not read back removed")
} else {
p.recordHandAct("removed the stream ASSIGNMENTS", why)
}
if len(failed) > 0 {
return p.removalFailed(r, strings.Join(failed, "; "))
}
r.Removed = p.now()
return p.writeLease(r)
}
// removed reads a removal tool's answer: removed now, or not there to remove.
func removed(raw json.RawMessage) bool {
var a struct {
Removed bool `json:"removed"`
Present *bool `json:"present"`
}
_ = json.Unmarshal(unwrap(raw), &a)
return a.Removed || (a.Present != nil && !*a.Present)
}
func (p *Part) removalFailed(r LeaseRecord, why string) error {
p.saySeat(EventRemovalFailed, map[string]any{"failed": why, "ask": r.Ask, "at": p.now()})
if err := p.putRemovalDue(RemovalAfter); err != nil {
return fmt.Errorf("the removal failed (%s) and removal-due could not be written again: %v", why, err)
}
return fmt.Errorf("the day-later removal failed, tried again in a day: %s", why)
}
// recordHandAct records one removal in the hand-act log; said, never fatal.
func (p *Part) recordHandAct(what, why string) {
if p.Call == nil {
return
}
if _, err := p.Call.Call(CallRecordHandAct, map[string]any{"what": what, "why": why, "cause": "lease-removal"}); err != nil && p.Logf != nil {
p.Logf("%s, and the hand act could not be recorded: %v", what, err)
}
}
// RemovalDue is the tool grants_removal_due: removal-due written again with a short time to live, so its marker starts
// the removal (the recovery for a marker missed while the part was down). Recorded as a hand act.
func (p *Part) RemovalDue(caller, why, ttl string) (any, error) {
if strings.TrimSpace(why) == "" {
return nil, errors.New("grants_removal_due needs a why, kept with the hand act")
}
d, err := time.ParseDuration(ttl)
if err != nil || d < time.Second || d > RemovalAfter {
return nil, fmt.Errorf("ttl %q is no time to live between 1s and %s", ttl, RemovalAfter)
}
r, err := p.readLease()
if err != nil {
return nil, err
}
if r.Handed.IsZero() || !r.Removed.IsZero() {
return nil, errors.New("no removal is due: the lease is not handed to mesh-grants, or the removal ran")
}
if err := p.putRemovalDue(d); err != nil {
return nil, err
}
p.recordHandAct("wrote removal-due again with "+d.String()+" to live", why+" (by "+caller+")")
return map[string]any{"written": KeyRemovalDue, "ttl": d.String(), "says": "its marker starts the removal when it expires"}, nil
}
// LeaseState answers where the lease write stands, for grants_lease_check.
func (p *Part) LeaseState() (LeaseRecord, error) { return p.readLease() }
+5 -29
View File
@@ -34,7 +34,7 @@ const (
)
// States is every state the part keeps.
var States = []string{StateAccounts, StateAssertions, StateGrants, StateLease, StateMemberships, StateTakeover, StateUserList, StateWithheld}
var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList, StateWithheld}
// The events the part says.
const (
@@ -170,17 +170,9 @@ type Part struct {
// Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix.
Forget func(id string) error
Nonce func() string
// Lease is the process's hand on the controller's lease (leasewrite.go); nil in the tools, which ask the process.
Lease LeaseBus
// SaySeat says the seat mesh-grants' events (lease-handed, lease-altered, removal-failed): the process's; nil in the
// tools.
SaySeat SeatSayer
// deciding is set while a take-over is decided, or a lease act composes, so the composition it makes asks and
// reconciles nothing.
// deciding is set while a take-over is decided, so the composition it makes asks and reconciles nothing.
deciding atomic.Bool
// removalHeard is set when the marker of removal-due was heard, and taken by the next pass (leasewrite.go).
removalHeard atomic.Bool
Now func() time.Time
Now func() time.Time
cache Manifests
mu sync.Mutex
@@ -257,8 +249,6 @@ func (p *Part) Compose() Result {
// the trunk names, so the list grants a build's subjects before the want moves to it (issue 774).
recs, _ := Records(in)
listRecs, left := ListRecords(in)
// From the lease write's cut, the controller's write on the lease is taken from the list (ADR 0582 clause 3).
listRecs.LeaseCut = p.leaseCut()
holdings := HoldingsOf(in)
res.RootNotFree = WithRootFree(&listRecs, p.Call)
recs.RootFree = listRecs.RootFree
@@ -337,10 +327,6 @@ func (p *Part) Compose() Result {
members = append(members, m.Membership)
}
seated := holdings.ByMemberships(members...)
// From the lease write the memberships are written to the state alone and no longer said on their subject, which
// nothing reads any more (novox/hq ADR 0571 clause 10, issue 774).
notice, _ := leaseHandedAway(p.Read)
notice = !notice
for _, key := range sortedKeys(ms) {
// The last membership is the state's own (ADR 0571 clause 7), never the assignments stream's, which the bus
// module's tools remove at the lease write (research 072 row 33): a last read there would freeze every membership
@@ -360,10 +346,8 @@ func (p *Part) Compose() Result {
m.Published, now = "not published (tried again at the next event): its own "+StateMemberships+" cannot be written", false
} else {
res.MembershipsWritten++
if notice {
if err := p.Publish.PublishMembership(m.Node, m.Module, body); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("the membership of %s is written, its notice on its subject not said: %v", key, err))
}
if err := p.Publish.PublishMembership(m.Node, m.Module, body); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("the membership of %s is written, its notice on its subject not said: %v", key, err))
}
}
}
@@ -426,18 +410,10 @@ func (p *Part) Compose() Result {
if err := p.Reconcile(); err != nil {
res.Errors = append(res.Errors, err.Error())
}
p.leaseStep(&res)
}
return res
}
// composeQuiet composes as a step of an act: asking and reconciling nothing.
func (p *Part) composeQuiet() Result {
p.deciding.Store(true)
defer p.deciding.Store(false)
return p.Compose()
}
// Last is the result of the last pass this part made, kept under the lock alone.
func (p *Part) Last() Result {
p.mu.Lock()
@@ -49,9 +49,7 @@ func HeardFrom(event string) string {
const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent
// Consumes is every event the part hears, as its manifest's consumes name them.
//
// Not the controller's condition events: the keeper's alone (novox/hq ADR 0564 §9, issue 774).
var Consumes = []string{EventKeeperRaised, EventKeeperCleared, EventAccountIssued, EventBucketExpirySet}
var Consumes = []string{EventConditionRaised, EventConditionCleared, EventKeeperRaised, EventKeeperCleared, EventAccountIssued}
// IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject.
func IsAccountIssued(event string) bool {
@@ -95,8 +95,6 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped
cur, _, had := p.Sealed()
next := cur
next.Follow = followOf(p.SwitchState())
// The bus node follows this list alone once the lease is handed (ADR 0571 clause 8, ADR 0582 clause 3).
next.LeaseHanded = p.leaseHanded()
key := ""
if in.Loaded != nil {
key = in.Loaded.Key
@@ -6,7 +6,6 @@ import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
@@ -132,15 +131,8 @@ func (p *Part) SwitchState() string {
// the refusals of Decide. Asking again while an ask is open is refused.
func (p *Part) TakeOver(caller, why string) (any, error) { return p.request(ActTakeOver, caller, why) }
// HandBack is the tool: it asks the operator to switch the bus node back to the controller's list. Once the lease is
// handed to the grants part it is answered as gone: the grants part's list is the only one (ADR 0547 §11).
func (p *Part) HandBack(caller, why string) (any, error) {
if p.leaseHanded() {
return nil, errors.New("gone: the controller's lease is handed to mesh-grants, so the grants part's list is the only " +
"one and the user list is not handed back (ADR 0547 §11); lease-hand-back gives the lease back first")
}
return p.request(ActHandBack, caller, why)
}
// HandBack is the tool: it asks the operator to switch the bus node back to the controller's list.
func (p *Part) HandBack(caller, why string) (any, error) { return p.request(ActHandBack, caller, why) }
// nonce is four hex digits that keep two asks of one second apart.
func (p *Part) nonce() string {
@@ -348,9 +340,6 @@ func (p *Part) AskOf(r Request, st UserListState) (asks.Ask, error) {
// Warranted is the process's act on the operator's answer (asker.Client.Listen hands it once, after checking the ask, the
// option and its level). A no is recorded; a yes runs the switch after the refusals, and a refusal is said to the operator.
func (p *Part) Warranted(o asks.Option, w asks.Warrant) error {
if isLeaseAsk(w.Ask) {
return p.leaseWarranted(o, w)
}
s, err := p.readSwitch()
if err != nil {
return err
@@ -408,13 +397,6 @@ func (p *Part) Said(h asker.Handled, w asks.Warrant, err error) {
if err != nil && p.Logf != nil {
p.Logf("the answer to %s (%s) ended in an error: %v", w.Ask, h, err)
}
if isLeaseAsk(w.Ask) {
if r, err := p.readLease(); err == nil && r.Open != nil && r.Open.Ask == w.Ask && r.Open.Step == StepAsked {
r.end(string(w.Outcome), string(w.Outcome)+" ("+string(h)+"): nothing was written", p.now())
_ = p.writeLease(r)
}
return
}
s, err := p.readSwitch()
if err != nil || s.Open != w.Ask {
return
+1 -5
View File
@@ -7,7 +7,6 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/membership"
)
@@ -30,10 +29,7 @@ const (
// Tools is every tool, in the manifest's order.
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState,
ToolAccounts, ToolConsumerMatch, ToolAcceptHashes, ToolLeaseCheck, ToolRemovalDue}
// SeatTool is the tool serving one of the seat mesh-grants' verbs, `mesh-grants.<verb>`.
func SeatTool(verb string) string { return catalogue.GrantsSeat + "." + verb }
ToolAccounts, ToolConsumerMatch, ToolAcceptHashes, ToolLeaseCheck}
// WritesGrants is what grants says of the part's own grants: written once the list the bus loaded carries a build's
// accounts, and the one key the assigner reads (ADR 0571 clause 9, which changes ADR 0547 clause 5).
+40 -3
View File
@@ -13,9 +13,7 @@ import (
// Issue 774, design 47 L3 (ADR 0547 §11, ADR 0561 §3, ADR 0571 §13): grants_lease_check answers from bus state who holds
// the controller's lease, every manifest's dependency on the controller with what keeps it, the controller's streams in
// use, and the lease write's check run dry, at the write (the three kept to it alone) and before its delivery. The
// judgement of each kind of dependency is mesh-sdk's and tested there (catalogue/controller_dependence_test.go, its rows
// moved from here, 774/S4); these rows are the tool's own: what it read, the lease, the streams.
// use, and the lease write's check run dry, at the write (the three kept to it alone) and before its delivery.
func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
const mManifest = `{"module":"mod-m","version":"1","reads":["mesh-controller.declared-packages"]}`
controllerHolds := map[string]any{"instance": "controller@novox-be pid 1 since 2026-10-30T14:52:10Z", "epoch": 60,
@@ -41,6 +39,29 @@ func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
map[string]string{"mod-l": `{"module":"mod-l","version":"1","reads":["mesh-controller.lease"]}`,
"mesh-controller": `{"module":"mesh-controller","version":"1","invokes":["seat:mesh-controller.status"]}`},
controllerHolds, want{writePasses: true, beforePasses: true}, nil},
{"the builder's consume of artifacts-gone is kept to the write itself",
map[string]string{"mesh-delivery-builder": `{"module":"mesh-delivery-builder","version":"1","consumes":["mesh-controller.artifacts-gone"]}`},
controllerHolds, want{deps: []string{"mesh-delivery-builder consume mesh-controller.artifacts-gone kept"}, writePasses: true, beforePasses: true}, nil},
{"the assigner's read of composed is kept until the write's delivery, which drops it",
map[string]string{"mesh-delivery-assigner": `{"module":"mesh-delivery-assigner","version":"1","reads":["mesh-controller.composed","mesh-controller.lease"]}`},
controllerHolds, want{deps: []string{"mesh-delivery-assigner read mesh-controller.composed kept"}, beforePasses: true,
reason: "mesh-delivery-assigner declares read mesh-controller.composed, kept by ADR 0571 §8 only until the lease write's own delivery"}, nil},
{"a read no record keeps fails both phases, named",
map[string]string{"pacman": `{"module":"pacman","version":"1","reads":["mesh-controller.declared-packages"]}`},
controllerHolds, want{deps: []string{"pacman read mesh-controller.declared-packages unkept"},
reason: "pacman declares read mesh-controller.declared-packages, which no record keeps until the lease write"}, nil},
{"a seat verb and a module tool of the controller's, a consume and a wait are each one; every tool (*) is none",
map[string]string{"mod-i": `{"module":"mod-i","version":"1","invokes":["seat:mesh-controller.root-free","mesh-controller.status","*","nats.nats_users"],` +
`"consumes":["mesh-controller.checked","mod-x.thing"],"uses":["mesh-controller"]}`,
"node-tools": `{"module":"node-tools","version":"1","waits":["mesh-controller.checked"]}`},
controllerHolds, want{deps: []string{"mod-i consume mesh-controller.checked unkept", "mod-i invoke mesh-controller.status unkept",
"mod-i invoke seat:mesh-controller.root-free unkept", "mod-i uses mesh-controller unkept", "node-tools waits mesh-controller.checked unkept"},
reason: "mod-i declares invoke seat:mesh-controller.root-free, which no record keeps"}, nil},
{"a process's own invoke of the controller is its module's dependency, named by the process",
map[string]string{"mod-p": `{"module":"mod-p","version":"1","resources":[{"id":"account","type":"user","name":"mod-p"},{"id":"worker","type":"process","run":["./w"],"user":"mod-p",` +
`"bus":{"invokes":["seat:mesh-controller.held"]}}]}`},
controllerHolds, want{deps: []string{"mod-p invoke seat:mesh-controller.held unkept"},
reason: "mod-p (process worker) declares invoke seat:mesh-controller.held"}, nil},
{"no manifest read from the indexer fails both phases, though nothing read depends on the controller",
map[string]string{},
controllerHolds, want{reason: "no manifest was read from the indexer (it holds 0 modules)", bothPhases: true}, nil},
@@ -119,3 +140,19 @@ func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
})
}
}
// Every entry of the keep list names the record that keeps it, and exactly three are kept to the write itself (ADR 0571 §13).
func TestTheKeepListNamesItsRecordsAndThreeToTheWrite(t *testing.T) {
toTheWrite := 0
for _, k := range grants.KeepList {
if !strings.HasPrefix(k.Record, "ADR ") || k.Module == "" || k.What == "" {
t.Errorf("entry %+v names no record, module or declaration", k)
}
if k.ToTheWrite {
toTheWrite++
}
}
if toTheWrite != 3 {
t.Errorf("%d entries kept to the write itself, ADR 0571 §13 names three", toTheWrite)
}
}
-459
View File
@@ -1,459 +0,0 @@
package manifest
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/seal"
"mesh-grants/internal/grants"
)
// leaseFake is the bus as the lease write meets it: fakeBus's states and tools, and the seats and tools the write asks.
// Each answer is shaped as its writer's code answers it (issue 774): node-service-manager's status the systemd module's
// Manager.Status (ActiveState), nats_bucket_expiry nats-tools' BucketExpirySet (now), nats_consumer_remove its
// ConsumerRemoved (removed) and an absent consumer its error ("has no consumer"), nats_stream_remove its StreamRemoved
// (removed, present), issue-tracker.get mesh-issues' Brief (status), node_trusts mesh-delivery-view's TrustsAnswer
// (every), mesh-config's compare its Comparison (source, count), mesh-wants' upgrade compare the assigner's answer
// (empty), vault_status mesh-vault's answer (ready, minting.on, minting.held_back).
type leaseFake struct {
*fakeBus
answer map[string]func(map[string]any) (any, error)
active string // the controller's ActiveState
expiry time.Duration // the lease bucket's
ttls map[string]time.Duration
log []string // the write's acts on the bus, in order
}
func newLeaseFake(t *testing.T) (*leaseFake, *grants.Part) {
t.Helper()
b := fixture()
b.manifestOf("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`)
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 1", "host": "a", "epoch": 60,
"taken": "2026-10-30T14:52:10Z"})
b.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "the keeper took over"})
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
// The controller's hash in the vault, so the list carries the controller and the cut changes it.
b.put(grants.ReadAccounts, buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1,
"at": "2026-10-30T14:52:10Z"})
f := &leaseFake{fakeBus: b, active: "active", expiry: 15 * time.Second, ttls: map[string]time.Duration{}}
f.answer = map[string]func(map[string]any) (any, error){
grants.CallIssue: func(map[string]any) (any, error) { return map[string]any{"status": "resolved"}, nil },
grants.CallNodeTrusts: func(map[string]any) (any, error) { return map[string]any{"every": true, "not_yet": []string{}}, nil },
grants.CallConfigCompare: func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 0}, nil },
grants.CallUpgrade: func(map[string]any) (any, error) { return map[string]any{"empty": true, "splits": []any{}}, nil },
grants.CallVaultStatus: func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}, nil
},
grants.CallServiceStop: func(map[string]any) (any, error) { f.active = "inactive"; return map[string]any{"ok": true}, nil },
grants.CallServiceStart: func(map[string]any) (any, error) { f.active = "active"; return map[string]any{"ok": true}, nil },
grants.CallServiceStatus: func(map[string]any) (any, error) { return map[string]any{"ActiveState": f.active}, nil },
grants.CallBucketExpiry: func(a map[string]any) (any, error) {
now := fmt.Sprint(a["max_age"])
d, _ := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
f.expiry = d
f.log = append(f.log, "expiry "+now)
return map[string]any{"bucket": a["bucket"], "now": now, "changed": true}, nil
},
grants.CallRecordWarrant: func(a map[string]any) (any, error) { f.log = append(f.log, "warranted"); return map[string]any{}, nil },
grants.CallRecordHandAct: func(a map[string]any) (any, error) {
f.log = append(f.log, "hand-act "+fmt.Sprint(a["what"]))
return map[string]any{}, nil
},
grants.CallConsumerRemove: func(map[string]any) (any, error) {
f.log = append(f.log, "consumer removed")
return map[string]any{"removed": true}, nil
},
grants.CallStreamRemove: func(map[string]any) (any, error) {
f.log = append(f.log, "stream removed")
return map[string]any{"removed": true, "present": true}, nil
},
}
p := partOn(b)
p.Call, p.Store, p.Lease, p.Asker, p.Objects = f, ttlStore{ownStore{b}, f}, f, &fakeAsker{wheres: map[string]*asks.Warrant{}}, &fakeObjects{}
p.SaySeat = func(event string, _ any) error { f.log = append(f.log, "event "+event); return nil }
p.Nonce = func() string { return "0001" }
return f, p
}
func (f *leaseFake) Call(key string, body any) (json.RawMessage, error) {
base, _, _ := strings.Cut(key, "@")
if answer, ok := f.answer[base]; ok {
f.mu.Lock()
f.calls = append(f.calls, key)
f.mu.Unlock()
args, _ := body.(map[string]any)
out, err := answer(args)
if err != nil {
return nil, err
}
return json.Marshal(out)
}
return f.fakeBus.Call(key, body)
}
func (f *leaseFake) PutHolder(value []byte) error {
f.log = append(f.log, "holder")
f.states[grants.ReadLease][grants.LeaseKey] = value
return nil
}
func (f *leaseFake) DeleteHolder() error {
f.log = append(f.log, "holder deleted")
delete(f.states[grants.ReadLease], grants.LeaseKey)
return nil
}
func (f *leaseFake) Expiry() (time.Duration, error) { return f.expiry, nil }
// loads says the bus node loaded the list the part last sealed.
func (f *leaseFake) loads(p *grants.Part) {
g, _, _ := p.Sealed()
f.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Digest: g.Digest,
Users: g.Users, Key: keyOf(f)}})
}
func keyOf(f *leaseFake) string {
var h delivery.Have
_ = json.Unmarshal(f.states[grants.ReadHave]["a.nats"], &h)
return h.UserList.Key
}
// ttlStore is the part's own store with a time to live per key, recorded.
type ttlStore struct {
ownStore
f *leaseFake
}
func (s ttlStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
s.f.ttls[state+"."+key] = ttl
s.f.log = append(s.f.log, key+" "+ttl.String())
return s.Put(state, key, value)
}
// record is the part's lease record now.
func record(t *testing.T, p *grants.Part) grants.LeaseRecord {
t.Helper()
r, err := p.LeaseState()
if err != nil {
t.Fatal(err)
}
return r
}
// warrant answers the open act's ask with option, as the router's warrant hands it.
func warrant(t *testing.T, p *grants.Part, option string) error {
t.Helper()
r := record(t, p)
if r.Open == nil {
t.Fatal("no act is open")
}
return p.Warranted(asks.Option{ID: option}, asks.Warrant{Ask: r.Open.Ask, Option: option, Outcome: asks.OutcomeChosen})
}
// Issue 774 (novox/hq ADR 0582): the lease write, from its ask through the day-later removal, against stand-in seats.
func TestTheLeaseWrite(t *testing.T) {
// 774/S14/AC1: any one precondition not green refuses the verb naming it; all green asks.
t.Run("preconditions", func(t *testing.T) {
for _, row := range []struct {
name string
break_ func(f *leaseFake)
}{
{"lease-check", func(f *leaseFake) {
f.manifestOf("mod-u", `{"module":"mod-u","version":"1","reads":["mesh-controller.conditions"]}`)
}},
{"earlier-steps", func(f *leaseFake) {
f.answer[grants.CallIssue] = func(a map[string]any) (any, error) {
if fmt.Sprint(a["number"]) == "1954" {
return map[string]any{"status": "fixing"}, nil
}
return map[string]any{"status": "resolved"}, nil
}
}},
{"node-trusts", func(f *leaseFake) {
f.answer[grants.CallNodeTrusts] = func(map[string]any) (any, error) {
return map[string]any{"every": false, "not_yet": []string{"ace"}}, nil
}
}},
{"config-compare", func(f *leaseFake) {
f.answer[grants.CallConfigCompare] = func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 2}, nil }
}},
{"upgrade-compare", func(f *leaseFake) {
f.answer[grants.CallUpgrade] = func(map[string]any) (any, error) { return map[string]any{"empty": false, "splits": []any{1}}, nil }
}},
{"keeper-take-over", func(f *leaseFake) {
f.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowController})
}},
{"vault-minting", func(f *leaseFake) {
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
}
}},
{"bus-tools", func(f *leaseFake) { f.manifestOf("nats", natsManifest) }},
{"", func(*leaseFake) {}},
} {
f, p := newLeaseFake(t)
row.break_(f)
_, err := p.LeaseWrite("operator@shanks", "the controller goes")
switch {
case row.name == "" && err != nil:
t.Errorf("all green: refused: %v", err)
case row.name != "" && (err == nil || !strings.Contains(err.Error(), row.name+": ")):
t.Errorf("%s not green: %v, want a refusal naming it", row.name, err)
case row.name != "":
for _, other := range grants.LeaseCheckNames {
if other != row.name && strings.Contains(err.Error(), other+": ") {
t.Errorf("%s not green: the refusal names %s too: %v", row.name, other, err)
}
}
}
if r := record(t, p); (r.Open != nil) != (row.name == "") {
t.Errorf("%q: an act open %v", row.name, r.Open)
}
}
})
// 774/S15/AC1, AC2: a no, an expiry, or a warrant whose checks changed writes nothing, naming the changed check.
t.Run("the ask and the warrant", func(t *testing.T) {
for _, row := range []struct {
name, step, said string
answer func(t *testing.T, f *leaseFake, p *grants.Part)
}{
{"declined", grants.StepDeclined, "said no", func(t *testing.T, _ *leaseFake, p *grants.Part) {
if err := warrant(t, p, "no"); err != nil {
t.Fatal(err)
}
}},
{"expired", "expired", "expired", func(t *testing.T, _ *leaseFake, p *grants.Part) {
r := record(t, p)
p.Said("", asks.Warrant{Ask: r.Open.Ask, Outcome: asks.OutcomeExpired}, nil)
}},
{"checks changed", grants.StepRefused, "the checks changed since the ask: vault-minting (now: ready true, minting true, 1 held back)",
func(t *testing.T, f *leaseFake, p *grants.Part) {
// The vault holds one account back between the ask and the warrant.
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
}
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
}},
} {
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose() // the process asks it
if a := p.Asker.(*fakeAsker); len(a.asks) != 1 || len(a.asks[0].Options) != 2 || a.asks[0].Options[0].Binds == "" {
t.Fatalf("%s: asked %+v, want one ask whose yes binds the act", row.name, a.asks)
}
row.answer(t, f, p)
r := record(t, p)
if r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != row.step || !strings.Contains(r.Acts[0].Said, row.said) {
t.Errorf("%s: open %v, acts %+v; want one act %s saying %q", row.name, r.Open, r.Acts, row.step, row.said)
}
if f.count(grants.CallServiceStop+"@a") != 0 || len(f.log) != 0 || r.Cut {
t.Errorf("%s: something was written: stops %d, %v, cut %v", row.name, f.count(grants.CallServiceStop+"@a"), f.log, r.Cut)
}
}
})
// 774/S16/AC1, AC2; 774/S17/AC1, AC2: the write's six steps, in order, each read back before the next.
t.Run("the write", func(t *testing.T) {
f, p := newLeaseFake(t)
f.answer[grants.CallServiceStop] = func(map[string]any) (any, error) { return map[string]any{"ok": true}, nil } // reads back active
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
if r := record(t, p); r.Open != nil || r.Cut || r.Acts[0].Step != grants.StepFailed || !strings.Contains(r.Acts[0].Said, "did not read back stopped") || len(f.log) != 0 {
t.Fatalf("a stop not read back: %+v, %v; want the act failed there, nothing written", r, f.log)
}
f, p = newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
before, _, _ := p.Sealed()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
r := record(t, p)
g, _, _ := p.Sealed()
if f.active != "inactive" || !r.Cut || r.Open == nil || r.Open.Step != grants.StepCut || r.Open.List != g.Digest || g.Digest == before.Digest {
t.Fatalf("after the warrant: active %s, %+v, sealed %s (was %s); want stopped, cut and waiting on the cut list", f.active, r, g.Digest, before.Digest)
}
if len(f.log) != 0 {
t.Fatalf("the expiry or the key was touched before the cut list loaded: %v", f.log)
}
// The cut list takes the controller's write on the lease and gives mesh-grants the key holder.
var st grants.UserListState
raw, _, _, _ := ownStore{f.fakeBus}.Get(grants.StateUserList, grants.KeyList)
_ = json.Unmarshal(raw, &st)
p.Compose() // a pass with the bus node not loaded yet waits
if len(f.log) != 0 {
t.Fatalf("a pass before the load acted: %v", f.log)
}
f.loads(p)
p.Compose()
want := []string{"expiry none", "holder", "event lease-handed", "warranted", "removal-due 24h0m0s"}
if !slices.Equal(f.log, want) {
t.Errorf("the write's acts %v, want %v in that order", f.log, want)
}
var h buscompose.LeaseHolder
_ = json.Unmarshal(f.states[grants.ReadLease][grants.LeaseKey], &h)
r = record(t, p)
if h.HandedTo != "mesh-grants" || h.HandedBy != "operator" || h.Why != "the controller goes" || h.Ask != r.Ask || h.At.IsZero() ||
r.Open != nil || r.Handed.IsZero() || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
t.Errorf("holder %+v, record %+v", h, r)
}
p.Compose() // the pass the lease key's change wakes
if g, _, _ := p.Sealed(); !g.LeaseHanded {
t.Error("the user-list key does not say the lease is handed")
}
if _, err := p.HandBack("operator@shanks", "back"); err == nil || !strings.HasPrefix(err.Error(), "gone") {
t.Errorf("the user list's hand-back once the lease is handed: %v, want gone", err)
}
if _, err := p.LeaseWrite("operator@shanks", "again"); err == nil {
t.Error("a second lease-write with the lease handed was asked")
}
// 774/S18/AC1: the key or the expiry changed by another hand is said once, and nothing is written.
f.log = nil
f.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 9", "host": "a", "epoch": 61})
p.Compose()
p.Compose()
f.expiry = 15 * time.Second
p.Compose()
if !slices.Equal(f.log, []string{"event lease-altered"}) || !strings.Contains(record(t, p).Altered, "controller@a pid 9") {
t.Errorf("an altered lease said %v (%q), want lease-altered once naming it", f.log, record(t, p).Altered)
}
})
// 774/S19/AC1, AC2: hand-back restores the expiry, deletes holder and removal-due, and starts the controller only once
// the list with its rights is loaded; once the removal ran it is refused.
t.Run("hand-back", func(t *testing.T) {
f, p := handed(t)
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
t.Fatal(err)
}
p.Compose()
f.log = nil
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
r := record(t, p)
if f.expiry != 15*time.Second || f.states[grants.ReadLease][grants.LeaseKey] != nil || r.Cut || f.active != "inactive" {
t.Fatalf("after the warrant: expiry %s, holder %s, cut %v, controller %s", f.expiry, f.states[grants.ReadLease][grants.LeaseKey], r.Cut, f.active)
}
if _, _, found, _ := f.ownGet(grants.StateLease, grants.KeyRemovalDue); found {
t.Error("removal-due is not deleted")
}
f.loads(p)
p.Compose()
if r := record(t, p); f.active != "active" || r.Open != nil || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
t.Errorf("once loaded: controller %s, %+v", f.active, r)
}
// AC2: once the removal ran, refused naming why.
f, p = handed(t)
p.RemovalDueHeard()
p.Compose()
if _, err := p.LeaseHandBack("operator@shanks", "too late"); err == nil || !strings.Contains(err.Error(), "without its consumer") {
t.Errorf("hand-back after the removal: %v", err)
}
})
// 774/S20/AC1, AC2, AC3: the marker of removal-due starts the removal; a failure says removal-failed and writes the key
// again with a day; a hand-back's delete removes nothing.
t.Run("the day-later removal", func(t *testing.T) {
f, p := handed(t)
f.log = nil
p.RemovalDueHeard()
p.Compose()
want := []string{"consumer removed", "hand-act removed the controller's consumer of DEAD_LETTER_NOTICES", "stream removed",
"hand-act removed the stream ASSIGNMENTS"}
if !slices.Equal(f.log, want) || record(t, p).Removed.IsZero() {
t.Errorf("the removal %v (removed %v), want %v", f.log, record(t, p).Removed, want)
}
f, p = handed(t)
f.log = nil
f.answer[grants.CallStreamRemove] = func(map[string]any) (any, error) {
return nil, errors.New("ASSIGNMENTS has 1 consumers: a reader is attached, so it was not removed")
}
p.RemovalDueHeard()
p.Compose()
if !slices.Contains(f.log, "event removal-failed") || f.ttls[grants.StateLease+"."+grants.KeyRemovalDue] != grants.RemovalAfter ||
!record(t, p).Removed.IsZero() {
t.Errorf("a failed removal: %v, removal-due %s", f.log, f.ttls[grants.StateLease+"."+grants.KeyRemovalDue])
}
f, p = handed(t)
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
f.log = nil
p.RemovalDueHeard() // the hand-back's own delete of removal-due, heard
p.Compose()
if slices.ContainsFunc(f.log, func(s string) bool { return strings.Contains(s, "removed") }) {
t.Errorf("a hand-back's delete of removal-due removed %v", f.log)
}
})
// 774/S21/AC1: once the lease is handed away a membership is written to the state and not published on its subject.
t.Run("memberships after the write", func(t *testing.T) {
f, p := handed(t)
f.publishes = nil
f.assign("b", "mod-x")
p.Compose()
if len(f.publishes) != 0 {
t.Errorf("published on mesh.assignment after the lease write: %v", f.publishes)
}
if _, _, found, _ := f.ownGet(grants.StateMemberships, "b.mod-x"); !found {
t.Error("the membership is not written to the state")
}
})
}
// handed is a part whose lease write is done: the lease names mesh-grants and removal-due is written.
func handed(t *testing.T) (*leaseFake, *grants.Part) {
t.Helper()
f, p := newLeaseFake(t)
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
t.Fatal(err)
}
p.Compose()
if err := warrant(t, p, "yes"); err != nil {
t.Fatal(err)
}
f.loads(p)
p.Compose()
if r := record(t, p); r.Handed.IsZero() {
t.Fatalf("the write did not finish: %+v, %v", r, f.log)
}
return f, p
}
+6 -31
View File
@@ -2,18 +2,9 @@
"module": "mesh-grants",
"version": "1",
"slug": "grants",
"claims": [
{
"name": "mesh-grants",
"scope": "mesh",
"serves": [
"lease-write",
"lease-hand-back"
]
}
],
"reads": [
"mesh-config.resolved",
"mesh-controller.assignments",
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.manifests",
@@ -25,10 +16,11 @@
"mesh-conditions.takeover"
],
"consumes": [
"mesh-controller.condition-raised",
"mesh-controller.condition-cleared",
"mesh-conditions.condition-raised",
"mesh-conditions.condition-cleared",
"mesh-vault.account-issued",
"nats.bucket-expiry-set"
"mesh-vault.account-issued"
],
"uses": [
"operator-channel"
@@ -38,19 +30,7 @@
"nats.nats_user_can",
"nats.nats_users",
"seat:node-engine.root-free",
"seat:mesh-hand-acts.warranted",
"seat:node-service-manager.stop",
"seat:node-service-manager.start",
"seat:node-service-manager.status",
"nats.nats_bucket_expiry",
"nats.nats_consumer_remove",
"nats.nats_stream_remove",
"seat:mesh-hand-acts.hand-act",
"mesh-delivery-view.node_trusts",
"seat:mesh-config.compare",
"seat:mesh-wants.upgrade",
"mesh-vault.vault_status",
"seat:issue-tracker.get"
"seat:mesh-hand-acts.warranted"
],
"emits": [
"asserted",
@@ -71,8 +51,7 @@
"grants_accounts",
"grants_consumer_match",
"grants_accept_hashes",
"grants_lease_check",
"grants_removal_due"
"grants_lease_check"
],
"state": [
{
@@ -87,10 +66,6 @@
"name": "grants",
"history": 5
},
{
"name": "lease",
"key-ttl": true
},
{
"name": "memberships",
"history": 5