Compare commits
7
Commits
main
..
feat/l5-l7
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3ae3519714 | ||
|
|
04aa1a1ace | ||
|
|
b3a74e0bdc | ||
|
|
b8d4a77e75 | ||
|
|
8112fe4b13 | ||
|
|
b74c371ed6 | ||
|
|
f95d5b4d23 |
@@ -0,0 +1,7 @@
|
||||
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
|
||||
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
|
||||
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
|
||||
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
|
||||
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
|
||||
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
|
||||
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
|
||||
+3
-2
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
|
||||
@@ -102,9 +103,9 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.place(t, "g14", "web")
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
running := strings.Repeat("3", 40)
|
||||
b.place(t, "novox", "mesh-controller")
|
||||
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
|
||||
b.built(t, "mesh-controller", "novox/mesh-controller", running)
|
||||
sdkHead := strings.Repeat("4", 40)
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/judge"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -39,8 +40,6 @@ type bed struct {
|
||||
ctx context.Context
|
||||
dir string
|
||||
buckets map[string]jetstream.KeyValue
|
||||
// placed is, by node, the modules the bed's placement names on it (place).
|
||||
placed map[string][]string
|
||||
}
|
||||
|
||||
func startBed(t *testing.T, leaveOut string) *bed {
|
||||
@@ -236,7 +235,8 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
}
|
||||
|
||||
// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
|
||||
// it from the manifest) it is read with no restart: a node it names gets its own want, the node composing its own.
|
||||
// it from the manifest) it is read with no restart: a module only the placement names is
|
||||
// said as placed by one source only.
|
||||
pb := startBed(t, delivery.PlacementBucket)
|
||||
pdone := pb.start(t)
|
||||
select {
|
||||
@@ -253,8 +253,13 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nw, err := delivery.DecodeNodeWant(pb.get(t, delivery.AssignerWantsBucket, "want.n1.node")); err != nil || !nw.Self {
|
||||
t.Fatalf("the placement state made after the start was not read: n1's own want = %+v, %v", nw, err)
|
||||
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
|
||||
if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the placement state made after the start was not read")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,29 +271,6 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// place adds module to the seat's placement of node, written as the seat writes it: one node key and a key per module
|
||||
// (research 072 row 37). The placement alone names where modules run (ADR 0567 §8).
|
||||
func (b *bed) place(t *testing.T, node, module string) {
|
||||
t.Helper()
|
||||
if b.placed == nil {
|
||||
b.placed = map[string][]string{}
|
||||
}
|
||||
if !slices.Contains(b.placed[node], module) {
|
||||
b.placed[node] = append(b.placed[node], module)
|
||||
}
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range b.placed[node] {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
b.putJSON(t, delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// built appends the end of a done trunk build of module at commit to the builder's log, as the builder does.
|
||||
func (b *bed) built(t *testing.T, module, repository, commit string) {
|
||||
t.Helper()
|
||||
@@ -302,20 +284,24 @@ func (b *bed) built(t *testing.T, module, repository, commit string) {
|
||||
}
|
||||
}
|
||||
|
||||
// compose says the node's own composition is the digest d (a letter repeated), as its have names it (`<node>.node`): a
|
||||
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
|
||||
func (b *bed) compose(t *testing.T, node string, d byte) string {
|
||||
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
|
||||
// move to it, in mesh-controller_composed.
|
||||
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
|
||||
t.Helper()
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat(string(d), 64)
|
||||
v, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
|
||||
if judgement != delivery.JudgedNeither {
|
||||
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
|
||||
}
|
||||
v, err := delivery.EncodeComposedDigest(cd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.HaveBucket, key, v)
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
return digest
|
||||
}
|
||||
|
||||
@@ -343,8 +329,8 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.place(t, "g14", "web")
|
||||
b.place(t, "ace", "other")
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.other", []byte(`{"node":"ace","module":"other","build":"build-2"}`))
|
||||
heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
// 1927/AC1: the candidate carries the fingerprint set the SDK computes, as the indexer writes it.
|
||||
@@ -425,14 +411,14 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
}
|
||||
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
b.compose(t, node, 'c')
|
||||
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
||||
b.place(t, node, "web")
|
||||
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
|
||||
}
|
||||
b.compose(t, "shanks", 'a')
|
||||
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
b.start(t)
|
||||
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
@@ -440,8 +426,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if err != nil || want.FP != fNew {
|
||||
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||
}
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w)
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
|
||||
}
|
||||
var neither asks.Condition
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
@@ -465,8 +451,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
// there by itself: the condition is cleared.
|
||||
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F, the node composing it", w)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F with its composition", w)
|
||||
}
|
||||
cleared := false
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
|
||||
@@ -517,7 +503,7 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.place(t, "ace", "web")
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
|
||||
@@ -580,7 +566,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
wantKey, _ := delivery.WantKey("ace", "mail")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
|
||||
b.place(t, "ace", "mail")
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
|
||||
b.start(t)
|
||||
|
||||
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
@@ -632,8 +618,8 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", 'c')
|
||||
b.place(t, "ace", name)
|
||||
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
|
||||
}
|
||||
count := func() int {
|
||||
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
|
||||
@@ -663,6 +649,58 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
|
||||
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
|
||||
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
|
||||
func TestAWantWaitsForItsComposition(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
time.Sleep(time.Second)
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
t.Fatalf("a want was written before the composition existed: %s", e.Value())
|
||||
}
|
||||
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
|
||||
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
|
||||
}
|
||||
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
|
||||
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
|
||||
}
|
||||
// the node's own want
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
|
||||
}
|
||||
ownKey, _ := delivery.ComposedNodeKey("ace")
|
||||
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
|
||||
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the node's own want = %+v, %v", w, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
|
||||
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
t.Helper()
|
||||
@@ -677,7 +715,7 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.place(t, "ace", name)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
|
||||
}
|
||||
|
||||
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
|
||||
@@ -740,8 +778,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", 'b')
|
||||
b.compose(t, "ace", 'a')
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
@@ -756,8 +794,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b.get(t, delivery.AssignerAsksBucket, hold)
|
||||
b.answer(t, hold, "release")
|
||||
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
|
||||
if w.Composed != "" || !w.Self || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v, want F by the rules, the node composing it", w)
|
||||
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
cleared := false
|
||||
@@ -787,19 +825,19 @@ func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk
|
||||
|
||||
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
|
||||
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
|
||||
// value, so the node retries.
|
||||
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
|
||||
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", 'b')
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Self: true}) // a placed node's want names the build alone
|
||||
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
|
||||
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
|
||||
b.start(t)
|
||||
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
|
||||
a := b.askNow(t, akey, 0)
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") {
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
|
||||
t.Fatalf("the backup ask = %+v", a)
|
||||
}
|
||||
// an answer to another ask is no answer to this one
|
||||
@@ -810,7 +848,7 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
}
|
||||
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
|
||||
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
|
||||
if w1.FP != fNew || w1.Composed != "" || !w1.Self {
|
||||
if w1.FP != fNew || w1.Composed != first {
|
||||
t.Errorf("the want = %+v", w1)
|
||||
}
|
||||
// the node retries and the backup fails again: a new have value, asked again under the next number
|
||||
@@ -825,10 +863,91 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
if w2.Words == w1.Words {
|
||||
t.Errorf("the second answer left the want as it was: %+v", w2)
|
||||
}
|
||||
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
|
||||
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
|
||||
a3 := b.askNow(t, akey, 3)
|
||||
if !strings.Contains(a3.Ask.Explanation, second) {
|
||||
t.Errorf("the ask for the new revision = %+v", a3)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
|
||||
t.Errorf("the old answer was taken for the new revision: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
|
||||
|
||||
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
|
||||
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
|
||||
// neither is written at once.
|
||||
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, recorded)
|
||||
old := "sha256:" + strings.Repeat("a", 64)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
|
||||
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
|
||||
b.start(t)
|
||||
akey, _ := delivery.RecordAskKey("ace", "web", nw)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
|
||||
t.Fatalf("the record ask = %+v, %v", a, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the move was written before the yes: %+v", w)
|
||||
}
|
||||
b.answer(t, akey, assign.OptionYes)
|
||||
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
|
||||
// a new revision at the same fingerprint, judged to move data, waits as well
|
||||
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
|
||||
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
|
||||
b.get(t, delivery.AssignerAsksBucket, rkey)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
|
||||
t.Fatalf("the revision was written before the yes: %+v", w)
|
||||
}
|
||||
// and a revision judged neither is written at once
|
||||
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
|
||||
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
|
||||
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
|
||||
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
|
||||
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
|
||||
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
|
||||
b.start(t)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
|
||||
}
|
||||
// the controller composes the bus at the fingerprint it runs with a new user list
|
||||
key, _ := delivery.ComposedKey("ace", "bus", fOld)
|
||||
free := "sha256:" + strings.Repeat("e", 64)
|
||||
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
|
||||
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
if w.FP != fOld {
|
||||
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
|
||||
}
|
||||
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
|
||||
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
|
||||
t.Errorf("the user-list revision was asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
|
||||
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
|
||||
// cleared saying why, and the want stays at the fingerprint.
|
||||
@@ -836,7 +955,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", 'b')
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
@@ -869,7 +988,7 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", 'b')
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
@@ -57,7 +58,10 @@ func main() {
|
||||
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||
var readBuckets = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
buscompose.LeaseBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
@@ -123,14 +127,40 @@ type buckets map[string]jetstream.KeyValue
|
||||
// without it and takes no put further until it starts again (ADR 0546).
|
||||
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||
out := buckets{}
|
||||
// The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names
|
||||
// where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the
|
||||
// controller holds it, as before.
|
||||
away, why := false, ""
|
||||
if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err)
|
||||
} else {
|
||||
out[buscompose.LeaseBucket] = kv
|
||||
e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey)
|
||||
var raw []byte
|
||||
if gerr == nil {
|
||||
raw = e.Value()
|
||||
}
|
||||
if errors.Is(gerr, jetstream.ErrKeyNotFound) {
|
||||
gerr = nil
|
||||
}
|
||||
away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr)
|
||||
}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
if name == buscompose.LeaseBucket {
|
||||
continue
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if name == provider.AssignmentsBucket && err != nil && away {
|
||||
logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why)
|
||||
continue
|
||||
}
|
||||
if name == delivery.AssignerPutsBucket && err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if slices.Contains(seatBuckets, name) && err != nil {
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement no module is placed.
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement the controller's
|
||||
// assignments alone name where modules run, as before any placement was written (ADR 0549 §5).
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
@@ -163,9 +193,10 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.PartGrantsBucket && err != nil {
|
||||
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
|
||||
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
|
||||
if sourceOfGrants(name) && err != nil {
|
||||
// The grants part's grants and the controller's lease choose where grants are read from (issue 774): without
|
||||
// them the controller's grants are read, as before the lease is handed away.
|
||||
logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
@@ -184,7 +215,13 @@ func startsWithout(name string) bool {
|
||||
delivery.AssignerPutsBucket:
|
||||
return true
|
||||
}
|
||||
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
|
||||
return slices.Contains(seatBuckets, name) || sourceOfGrants(name)
|
||||
}
|
||||
|
||||
// sourceOfGrants says the bucket is the grants part's grants or the controller's lease: neither stops the assigner when it
|
||||
// is not there.
|
||||
func sourceOfGrants(name string) bool {
|
||||
return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
@@ -294,7 +331,10 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
buscompose.LeaseBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
@@ -377,7 +417,12 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
before := w.handedAway()
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
if name == buscompose.LeaseBucket && w.handedAway() == before {
|
||||
// The controller renews its lease every few seconds: only the lease's being handed away makes a pass.
|
||||
continue
|
||||
}
|
||||
wake()
|
||||
}
|
||||
}
|
||||
@@ -413,13 +458,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
return
|
||||
}
|
||||
if bus != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && startsWithout(name) {
|
||||
if kvs[name] == nil && (startsWithout(name) || name == provider.AssignmentsBucket) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
@@ -435,8 +480,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
return err
|
||||
}
|
||||
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
|
||||
// module is placed.
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then the
|
||||
// controller's assignments alone name where modules run (ADR 0549 §5).
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
@@ -449,7 +494,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
}
|
||||
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
|
||||
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
|
||||
for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
|
||||
if kvs[name] == nil && js != nil {
|
||||
name := name
|
||||
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
|
||||
@@ -476,10 +521,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
|
||||
}
|
||||
if bus != nil {
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf)
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf, w.handedAway)
|
||||
}
|
||||
if js != nil {
|
||||
p.publish = streamPublisher(ctx, js, thisNode)
|
||||
|
||||
@@ -19,8 +19,10 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
@@ -72,21 +74,28 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
|
||||
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
|
||||
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
lease []byte
|
||||
said string
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
|
||||
// Issue 774: the grants part's grants not there do not stop the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
|
||||
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
|
||||
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
@@ -103,14 +112,12 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
placed, err := delivery.PlacementValues(delivery.Placement{Node: "ace"})
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range placed {
|
||||
kv := kvs[delivery.PlacementBucket].(*watchKV)
|
||||
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
|
||||
}
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
|
||||
@@ -129,13 +136,18 @@ type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
lease []byte // the controller's lease, when it holds one
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
}
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
kv := &memKV{values: map[string][]byte{}}
|
||||
if name == buscompose.LeaseBucket && r.lease != nil {
|
||||
kv.values[buscompose.ControllerLeaseKey] = r.lease
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
|
||||
@@ -59,10 +59,12 @@ type passer struct {
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// callsCleared says the first pass cleared the conditions of refused controller calls a previous process raised
|
||||
// (clearStaleCalls): the assigner makes no such call now (issue 774).
|
||||
callsCleared bool
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its acts (nil: none is made), how it says
|
||||
// callSaid is, by the key of a refused controller call's condition, what is raised (issue 774): by this process, or by
|
||||
// a previous one and taken over from the findings.
|
||||
callSaid map[string]*callSaid
|
||||
// callsSwept says the first pass took over the refused calls' conditions a previous process raised (sweepCalls).
|
||||
callsSwept bool
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its calls of the controller (nil: none is made), how it says
|
||||
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
|
||||
seat *seatActs
|
||||
publish publisher
|
||||
@@ -90,7 +92,8 @@ func (p *passer) pass(ctx context.Context) {
|
||||
p.backups(ctx)
|
||||
}
|
||||
|
||||
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
placed := p.w.placedNodes()
|
||||
@@ -106,6 +109,20 @@ func (p *passer) nodeWants(ctx context.Context) {
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
if placed[node] {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
}
|
||||
|
||||
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
|
||||
@@ -113,7 +130,7 @@ func (p *passer) nodeWants(ctx context.Context) {
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned := p.w.assigned()
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
manifests := map[string]catalogue.Manifest{}
|
||||
for name, mod := range trunk {
|
||||
@@ -139,7 +156,9 @@ func (p *passer) wants(ctx context.Context) {
|
||||
defer p.unplacedFindings()
|
||||
defer p.seatHolders(ctx)
|
||||
defer p.policySplits(manifests, chosen)
|
||||
p.clearStaleCalls()
|
||||
differ := p.withCalls(p.w.placementDiffers())
|
||||
p.sayCalls(differ)
|
||||
frozen := p.sayPlacement(differ)
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
@@ -167,7 +186,15 @@ func (p *passer) wants(ctx context.Context) {
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Self: func(node string) bool { return placed[node] },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
|
||||
if placed[node] && p.w.handedAway() {
|
||||
return p.w.ownComposed(node) // from the lease write: the digest its own have names (ADR 0571 §6)
|
||||
}
|
||||
if placed[node] {
|
||||
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
|
||||
// names the build alone, and the node judges the move itself.
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
|
||||
}
|
||||
return p.w.composed(node, name, fp)
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
@@ -240,6 +267,9 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.recordAsk(ctx, name, fp, need)
|
||||
}
|
||||
for node, want := range out.Wants {
|
||||
if frozen[node+"."+name] {
|
||||
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
|
||||
}
|
||||
key, err := delivery.WantKey(node, name)
|
||||
if err != nil {
|
||||
p.logf("no want of %s on %s is written: %v", name, node, err)
|
||||
@@ -873,6 +903,42 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
|
||||
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
|
||||
// for them and keeps the one written before. A say that fails is made again at the next pass.
|
||||
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
|
||||
frozen := map[string]bool{}
|
||||
now := map[string]bool{}
|
||||
for _, d := range differ {
|
||||
frozen[d.Node+"."+d.Module] = true
|
||||
c := assign.PlacementCondition(d, true)
|
||||
now[c.Key] = true
|
||||
if p.said[c.Key+".raised"] {
|
||||
continue
|
||||
}
|
||||
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
|
||||
continue
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
}
|
||||
prefix := delivery.AssignerModule + ".placement."
|
||||
for key := range p.said {
|
||||
c, raised := strings.CutSuffix(key, ".raised")
|
||||
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
|
||||
continue
|
||||
}
|
||||
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
|
||||
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
delete(p.said, key)
|
||||
}
|
||||
return frozen
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
@@ -937,8 +1003,11 @@ func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen ma
|
||||
now = p.now
|
||||
}
|
||||
split := map[string]bool{}
|
||||
// The controller writes no policy any more (ADR 0571 §6): nothing is compared with.
|
||||
for _, s := range assign.Splits(manifests, chosen, nil) {
|
||||
theirs := map[string][]string{}
|
||||
if !p.w.handedAway() { // the controller writes no policy from the lease write (ADR 0571 §6)
|
||||
theirs = assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket))
|
||||
}
|
||||
for _, s := range assign.Splits(manifests, chosen, theirs) {
|
||||
split[s.Module] = true
|
||||
if p.splits[s.Module] {
|
||||
continue
|
||||
|
||||
@@ -8,14 +8,17 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -49,34 +52,6 @@ func newPasser(kvs buckets, at time.Time) *passer {
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
// placeOn puts the seat's placement of node, naming the modules, into the world: one node key and a key per module
|
||||
// (research 072 row 37).
|
||||
func placeOn(t *testing.T, w *world, node string, modules ...string) {
|
||||
t.Helper()
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
|
||||
func composes(t *testing.T, w *world, node, digest string) {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
w.put(delivery.HaveBucket, key, raw)
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
@@ -221,7 +196,7 @@ func putOfTheBus(t *testing.T) {
|
||||
p := newPasser(kvs, now)
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, nil)
|
||||
p.acts.snapshot = func(context.Context, string, string) (string, error) {
|
||||
return "novox's restore point of nats (streams) taken 22:01", nil
|
||||
}
|
||||
@@ -236,7 +211,7 @@ func putOfTheBus(t *testing.T) {
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "nats")
|
||||
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
@@ -292,7 +267,9 @@ func putOfTheBus(t *testing.T) {
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
composes(t, p.w, "novox", digest)
|
||||
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
world(delivery.PartGrantsBucket, gkey, g)
|
||||
@@ -379,7 +356,7 @@ func putOfTheBus(t *testing.T) {
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPlacementAloneNamesWhereModulesRun(t)
|
||||
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
@@ -553,7 +530,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
@@ -591,7 +568,7 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
placeOn(t, p.w, "shanks", "systemd-resolved")
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
@@ -655,13 +632,15 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
name string
|
||||
trunk map[string]delivery.Module
|
||||
gone []string // the keys taken away
|
||||
noDigit bool // the node's own composition is not read
|
||||
noDigit bool // the controller's composition is not read
|
||||
heldAt string // the node's want is at this fingerprint, whose composition is digOld
|
||||
}{
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true},
|
||||
{"the trunk is not read at all", nil, nil, false},
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
|
||||
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
|
||||
{"the trunk is not read at all", nil, nil, false, ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
@@ -688,9 +667,21 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
asksKV.values[key] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
|
||||
if !c.noDigit {
|
||||
composes(t, p.w, "novox", digNew)
|
||||
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
|
||||
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
p.w.put(delivery.ControllerComposedBucket, ckey, v)
|
||||
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
|
||||
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
if c.heldAt != "" {
|
||||
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
|
||||
}
|
||||
p.w.put(delivery.ControllerComposedBucket, okey, old)
|
||||
}
|
||||
if c.heldAt != "" {
|
||||
wkey, _ := delivery.WantKey("novox", "web")
|
||||
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
|
||||
p.w.put(delivery.AssignerWantsBucket, wkey, w)
|
||||
}
|
||||
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||
wantsKV.values[heldOld] = []byte("x")
|
||||
@@ -788,38 +779,134 @@ func mustKey(key string, err error) string {
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
|
||||
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
|
||||
if cerr != nil {
|
||||
t.Fatal(cerr)
|
||||
}
|
||||
put(delivery.ControllerComposedBucket, ckey, cd)
|
||||
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
|
||||
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, tkey, tcd)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", revised)
|
||||
for i := 0; i < 2; i++ {
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
|
||||
}
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != revised {
|
||||
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
|
||||
}
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Error("the answered record ask was taken away")
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
|
||||
// module in an assignment only is no module of its node; before any placement is read nothing is placed and the check composes
|
||||
// no input; placed, the check composes over the input the nodes compose from (frombus, 1126, ADR 0547 §2).
|
||||
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
|
||||
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
|
||||
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
|
||||
// assignments are the one source and nothing is raised.
|
||||
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
|
||||
if got := p.w.assigned(); len(got) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v", got)
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
assigned := func(node, module string) {
|
||||
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
|
||||
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
|
||||
}
|
||||
place := func(node string, modules ...string) {
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
// One node key and a key per module (research 072 row 37); a module taken off leaves its key, which the node
|
||||
// key no longer names.
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
p.w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
assigned("g14", "a")
|
||||
assigned("g14", "b")
|
||||
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v, said %v", got, said)
|
||||
}
|
||||
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
|
||||
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
placeOn(t, p.w, "g14", "c", "a")
|
||||
placeOn(t, p.w, "n2", "d")
|
||||
if got := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "c"}) || !slices.Equal(got["n2"], []string{"d"}) || len(got) != 2 {
|
||||
t.Errorf("the placement alone: assigned %v", got)
|
||||
place("g14", "a", "c")
|
||||
got, _ := p.w.assigned()
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
p.sayPlacement(p.w.placementDiffers())
|
||||
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
|
||||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
|
||||
!strings.Contains(said[1].Explanation, "the seat mesh-placement's placement names c on g14") {
|
||||
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
|
||||
}
|
||||
place("g14", "a", "b", "c")
|
||||
assigned("g14", "c")
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
|
||||
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
|
||||
// wrote its record; until then it says why and composes the assignments alone.
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
|
||||
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
|
||||
}
|
||||
placeOn(t, p.w, "g14", "a", "b", "c")
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
|
||||
p.w.put(delivery.HaveBucket, "g14.node", rec)
|
||||
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n2", TunnelKey: "k2"}})
|
||||
p.w.put(delivery.HaveBucket, "n2.node", rec2)
|
||||
m := p.w.mesh(context.Background(), none)
|
||||
if m.Input == nil || len(m.Input.Nodes) != 2 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
|
||||
// Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules
|
||||
// run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of
|
||||
// the node. A lease that goes back to a controller brings the comparison back.
|
||||
place("n2", "d")
|
||||
assigned("g14", "e")
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`))
|
||||
got, all := p.w.assigned()
|
||||
if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 {
|
||||
t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all)
|
||||
}
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 {
|
||||
t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:])
|
||||
}
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`))
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] {
|
||||
t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
@@ -834,7 +921,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
@@ -871,10 +958,15 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
// Issue 774, 2026-10-31: the node's own composition is not the digest its have waits on; the ask stays, and a pass that
|
||||
// changes nothing puts nothing: on novox the asks of postgres, keycloak and mongodb were taken away and made again every
|
||||
// 8 s, and a want put unchanged each pass made the node compose again each time.
|
||||
composes(t, p.w, "g14", "sha256:"+strings.Repeat("c", 64))
|
||||
// Issue 774, 2026-10-31: while the controller still composes the placed node's module (its lease not handed away), its
|
||||
// digests are not the one the node's have waits on; the ask stays, and a pass that changes nothing puts nothing: on novox
|
||||
// the asks of postgres, keycloak and mongodb were taken away and made again every 8 s, and a want put unchanged each
|
||||
// pass made the node compose again each time.
|
||||
for _, fp := range []string{fpTest, fpOld} {
|
||||
ck, _ := delivery.ComposedKey("g14", "web", fp)
|
||||
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, ck, cv)
|
||||
}
|
||||
asked := asksKV.values[akey]
|
||||
p.wants(context.Background())
|
||||
if !bytes.Equal(asksKV.values[akey], asked) {
|
||||
@@ -911,37 +1003,178 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S25/AC1 (ADR 0567 §8): an approved placement change is written and followed with its warrant recorded and
|
||||
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
|
||||
// cleared once, and said again at a later pass while the findings cannot be read.
|
||||
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
|
||||
var said []string
|
||||
p.publish = func(event string, _ any, _ string) error { said = append(said, event); return nil }
|
||||
prop := delivery.PlacementProposal{ID: "p-one", By: "operator@laptop", Why: "move web"}
|
||||
change := delivery.ChangeOf(delivery.Placement{Node: "anchor"}, delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
p.followWrite(prop, delivery.Answer{}, []delivery.PlacementChange{change})
|
||||
if len(bus.calls) != 0 || !slices.Equal(said, []string{delivery.PlacementChangedEvent}) {
|
||||
t.Fatalf("calls = %d, said = %v", len(bus.calls), said)
|
||||
}
|
||||
// refusingBus is the assigner's process account against a controller that refuses the calls whose verb is in refuse.
|
||||
type refusingBus struct {
|
||||
fakeBus
|
||||
refuse map[string]string
|
||||
// pins are the controller's pins its placement-export answers, by node.
|
||||
mu sync.Mutex
|
||||
pins map[string]map[string]delivery.Provider
|
||||
}
|
||||
|
||||
func (b *refusingBus) setPins(node string, pins map[string]delivery.Provider) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.pins == nil {
|
||||
b.pins = map[string]map[string]delivery.Provider{}
|
||||
}
|
||||
b.pins[node] = pins
|
||||
}
|
||||
|
||||
func (b *refusingBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
if key == exportCall {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
node, _ := body.(map[string]any)["node"].(string)
|
||||
answer, _ := json.Marshal(map[string]delivery.Placement{node: {V: delivery.Version, Node: node, Pins: b.pins[node]}})
|
||||
return json.Marshal(map[string]any{"ok": true, "answer": json.RawMessage(answer)})
|
||||
}
|
||||
for verb, words := range b.refuse {
|
||||
if strings.HasSuffix(key, "."+verb) {
|
||||
return nil, errors.New(words)
|
||||
}
|
||||
}
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
}
|
||||
|
||||
// Issue 774: a call of the controller it refuses is a condition, never only a journal line: raised once, urgent, naming the verb,
|
||||
// the node, the module and the refusal; cleared when a later call for the pair is answered, or, for a module's place, once the
|
||||
// pair differed and the two sources agree.
|
||||
func TestARefusedControllerCallIsACondition(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fails := 1
|
||||
p.findings.Keys = func() ([]string, error) {
|
||||
if fails > 0 {
|
||||
fails--
|
||||
return nil, errors.New("the bucket is not there yet")
|
||||
}
|
||||
return []string{"module.anchor.web.controller-call", "module.laptop.db.controller-pin", "module.anchor.web.neither"}, nil
|
||||
bus := &refusingBus{refuse: map[string]string{"assign": `assign takes no argument "why" — it takes "build", "module", "node"; nothing was done`}}
|
||||
woke := make(chan struct{}, 4)
|
||||
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
|
||||
assignWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
|
||||
call := func(c assign.ControllerCall) {
|
||||
p.seat.callAll([]assign.ControllerCall{c})
|
||||
<-woke
|
||||
}
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
if len(conds) != 2 || conds[0].State != asks.StateCleared || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
conds[1].Key != "mesh-delivery-assigner.controller-pin.laptop.db" {
|
||||
t.Fatalf("the old conditions cleared = %+v", conds)
|
||||
differ := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
|
||||
|
||||
call(assignWeb)
|
||||
p.sayCalls(differ)
|
||||
p.sayCalls(differ)
|
||||
if len(conds) != 1 || conds[0].State != asks.StateRaised || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
!strings.Contains(conds[0].Explanation, `takes no argument "why"`) || !strings.Contains(conds[0].Headline, "assign web on anchor") {
|
||||
t.Fatalf("after a refused assign, said = %+v", conds)
|
||||
}
|
||||
|
||||
bus.refuse = nil
|
||||
call(assignWeb)
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != conds[0].Key {
|
||||
t.Fatalf("after the assign is answered, said = %+v", conds)
|
||||
}
|
||||
|
||||
// Refused again, then assigned by hand: cleared once the pair, seen differing, agrees; not before it was seen differing.
|
||||
bus.refuse = map[string]string{"assign": "refused"}
|
||||
call(assignWeb)
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 3 || conds[2].State != asks.StateRaised {
|
||||
t.Fatalf("a second refusal is not raised: %+v", conds)
|
||||
}
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 3 {
|
||||
t.Fatalf("cleared before the pair was seen differing: %+v", conds)
|
||||
}
|
||||
p.sayCalls(differ)
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 4 || conds[3].State != asks.StateCleared || !strings.Contains(conds[3].Resolved, "agree") {
|
||||
t.Fatalf("the hand assign does not clear it: %+v", conds)
|
||||
}
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 4 {
|
||||
t.Fatalf("said again after the clear: %+v", conds)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774: a refused pin's condition, and one a previous process raised, clear once the pair agrees after it was seen differing:
|
||||
// the controller's pins (its placement-export, read again when its compositions change) against the placement's, the controller's
|
||||
// assignments against the placement for a module's place. A start whose findings cannot be read takes them over at a later pass.
|
||||
func TestARefusedCallClearsWhenTheSourcesAgree(t *testing.T) {
|
||||
store := delivery.Provider{Node: "anchor", Module: "store"}
|
||||
pinDB := assign.ControllerCall{Verb: "pin", Node: "laptop", Module: "db",
|
||||
Args: map[string]any{"node": "laptop", "provision": "db", "from": "anchor", "module": "store"}}
|
||||
placeWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
|
||||
webDiffers := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
|
||||
for name, c := range map[string]struct {
|
||||
call *assign.ControllerCall // refused in this process; nil: raised by a previous one
|
||||
findings []string // the findings a previous process left
|
||||
keysFail int // how many first reads of the findings fail
|
||||
differ []assign.Pair // the pairs the two sources name apart, while they differ
|
||||
agree func(*refusingBus) // the person's hand call
|
||||
key string
|
||||
}{
|
||||
"a pin refused, pinned by hand": {call: &pinDB, agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
|
||||
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
|
||||
"a place refused, assigned by hand": {call: &placeWeb, differ: webDiffers, agree: func(*refusingBus) {},
|
||||
key: "mesh-delivery-assigner.controller-call.anchor.web"},
|
||||
"a pin raised before a start": {findings: []string{"module.laptop.db.controller-pin"}, keysFail: 1,
|
||||
agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
|
||||
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
|
||||
"a place raised before a start": {findings: []string{"module.anchor.web.controller-call"}, differ: webDiffers, agree: func(*refusingBus) {},
|
||||
key: "mesh-delivery-assigner.controller-call.anchor.web"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fails := c.keysFail
|
||||
p.findings.Keys = func() ([]string, error) {
|
||||
if fails > 0 {
|
||||
fails--
|
||||
return nil, errors.New("the bucket is not there yet")
|
||||
}
|
||||
return c.findings, nil
|
||||
}
|
||||
bus := &refusingBus{refuse: map[string]string{"pin": "refused", "assign": "refused"}}
|
||||
woke := make(chan struct{}, 16)
|
||||
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
|
||||
// The placement pins db on laptop; the controller does not.
|
||||
raw, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "laptop", Pins: map[string]delivery.Provider{"db": store}})
|
||||
key, _ := delivery.LegacyPlacementKey("laptop")
|
||||
p.w.put(delivery.PlacementBucket, key, raw)
|
||||
// pass is a pass, then one more for each background read it started.
|
||||
pass := func(differ []assign.Pair) {
|
||||
p.sayCalls(differ)
|
||||
for {
|
||||
select {
|
||||
case <-woke:
|
||||
p.sayCalls(differ)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.call != nil {
|
||||
p.seat.callAll([]assign.ControllerCall{*c.call})
|
||||
<-woke
|
||||
}
|
||||
pass(c.differ)
|
||||
pass(c.differ)
|
||||
for _, cond := range conds {
|
||||
if cond.State == asks.StateCleared {
|
||||
t.Fatalf("cleared while the sources differ: %+v", conds)
|
||||
}
|
||||
}
|
||||
c.agree(bus)
|
||||
p.w.put(delivery.ControllerComposedBucket, "laptop.x", []byte("moved")) // a pin changes a composition
|
||||
pass(nil)
|
||||
if len(conds) == 0 {
|
||||
t.Fatal("nothing was raised")
|
||||
}
|
||||
last := conds[len(conds)-1]
|
||||
if last.Key != c.key || last.State != asks.StateCleared || !strings.Contains(last.Resolved, "agree") {
|
||||
t.Fatalf("the hand call does not clear it: %+v", conds)
|
||||
}
|
||||
n := len(conds)
|
||||
pass(nil)
|
||||
if len(conds) != n {
|
||||
t.Errorf("said again after the clear: %+v", conds[n:])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,8 +55,8 @@ type putActs struct {
|
||||
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs {
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
@@ -97,7 +97,7 @@ func (p *passer) puts(ctx context.Context) {
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned := p.w.assigned()
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
|
||||
@@ -10,12 +10,14 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/judge"
|
||||
@@ -57,23 +59,80 @@ func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) p
|
||||
}
|
||||
}
|
||||
|
||||
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
|
||||
// the controller: placement is the one source (ADR 0567 §8, issue 774).
|
||||
// seatActs keeps what the seat's calls of the controller came to while this process runs: the export read again at a
|
||||
// seed's warrant, and each call that keeps the controller's assignments in step (§8).
|
||||
type seatActs struct {
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
export map[string]exportCheck
|
||||
// calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to.
|
||||
calls map[string]string
|
||||
// outcomes is, by the key of a call's condition, the last call made for its pair once answered or refused: refused
|
||||
// "" when the controller answered it (issue 774: a refused call is a condition, never only a journal line).
|
||||
outcomes map[string]callOutcome
|
||||
// pins is, by node, the controller's pins as its placement-export last answered them (pinsOf).
|
||||
pins map[string]pinRead
|
||||
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, rec: handacts.Recorder{Ask: bus.Call}}
|
||||
// callOutcome is what one call of the controller came to: the call, and the controller's refusal ("" when it answered).
|
||||
type callOutcome struct {
|
||||
call assign.ControllerCall
|
||||
refused string
|
||||
}
|
||||
|
||||
// exportCheck is the export read again for one proposal: running, its answer, or why it was not read.
|
||||
type exportCheck struct {
|
||||
running bool
|
||||
export map[string]delivery.Placement
|
||||
err string
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{},
|
||||
outcomes: map[string]callOutcome{}, pins: map[string]pinRead{}, rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}}
|
||||
}
|
||||
|
||||
// exportOf is the export read again for a proposal, asked once in the background.
|
||||
func (a *seatActs) exportOf(id string) exportCheck {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if c, ok := a.export[id]; ok {
|
||||
return c
|
||||
}
|
||||
a.export[id] = exportCheck{running: true}
|
||||
go func() {
|
||||
defer a.wake()
|
||||
raw, err := a.bus.Call(exportCall, map[string]any{})
|
||||
c := exportCheck{}
|
||||
if err != nil {
|
||||
c.err = "the controller's placement-export did not answer: " + err.Error()
|
||||
} else if c.export, err = assign.ExportOf(raw); err != nil {
|
||||
c.err = err.Error()
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.export[id] = c
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
return exportCheck{running: true}
|
||||
}
|
||||
|
||||
// forget drops what is kept of a proposal's export, so a later warrant reads it again.
|
||||
func (a *seatActs) forget(id string) {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
delete(a.export, id)
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
|
||||
// the controller's while that seat has no holder (ADR 0565 §11, handacts.Recorder).
|
||||
// the controller's while that seat has no holder and the controller holds its lease (ADR 0565 §11, handacts.Recorder).
|
||||
func (a *seatActs) record(warrant string) {
|
||||
if a == nil || warrant == "" {
|
||||
return
|
||||
@@ -88,35 +147,257 @@ func (a *seatActs) record(warrant string) {
|
||||
}()
|
||||
}
|
||||
|
||||
// clearStaleCalls clears, once, the conditions of the refused controller calls a previous process raised (findings of the
|
||||
// kinds controller-call and controller-pin): the assigner calls no verb of the controller any more (ADR 0567 §8, issue 774),
|
||||
// so nothing would clear them. It is tried again at the next pass while the findings cannot be read.
|
||||
func (p *passer) clearStaleCalls() {
|
||||
if p.callsCleared || p.say == nil || p.findings.Keys == nil {
|
||||
// handAct records a hand-back in the hand-act log (§7), as record does.
|
||||
func (a *seatActs) handAct(prop delivery.PlacementProposal) {
|
||||
go func() {
|
||||
if taken, err := a.rec.Record(handacts.Request{What: "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "),
|
||||
Why: prop.Why, Cause: "asked by " + prop.By + ", approved by the operator"}); err != nil {
|
||||
a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err)
|
||||
} else {
|
||||
a.logf("the hand-back %s is recorded in the log of %s", prop.ID, taken.Log)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// callAll makes the controller's calls in order in the background, each said in the journal and kept by its pair.
|
||||
func (a *seatActs) callAll(calls []assign.ControllerCall) {
|
||||
if len(calls) == 0 {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
for _, c := range calls {
|
||||
a.calls[c.Node+"."+c.Module] = c.Key() + " is asked"
|
||||
}
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
for _, c := range calls {
|
||||
words, refused := c.Key()+" answered", ""
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+"."+c.Verb, c.Args); err != nil {
|
||||
refused = err.Error()
|
||||
words = c.Key() + " refused: " + refused
|
||||
}
|
||||
a.logf("keeping the controller in step with placement: %s", words)
|
||||
a.mu.Lock()
|
||||
a.calls[c.Node+"."+c.Module] = words
|
||||
a.outcomes[assign.CallConditionKey(c)] = callOutcome{call: c, refused: refused}
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// outcomesNow is a copy of every call's last outcome, by the key of its condition.
|
||||
func (a *seatActs) outcomesNow() map[string]callOutcome {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return maps.Clone(a.outcomes)
|
||||
}
|
||||
|
||||
// pinRead is the controller's pins of one node as its placement-export answered them, for the generation of the
|
||||
// controller's compositions it was asked at: a pin made at the controller changes a composition, which the assigner
|
||||
// watches, so a read is asked again only after such a change, never on a timer.
|
||||
type pinRead struct {
|
||||
gen uint64
|
||||
running bool
|
||||
known bool
|
||||
pins map[string]delivery.Provider
|
||||
}
|
||||
|
||||
// pinsOf is the controller's pins of node as last read, and whether a read answered; a read is asked in the background
|
||||
// when none was made at gen, and wakes a pass when it answers.
|
||||
func (a *seatActs) pinsOf(node string, gen uint64) (map[string]delivery.Provider, bool) {
|
||||
if a == nil {
|
||||
return nil, false
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
r, ok := a.pins[node]
|
||||
if !ok || (r.gen != gen && !r.running) {
|
||||
r.gen, r.running = gen, true
|
||||
a.pins[node] = r
|
||||
go func() {
|
||||
defer a.wake()
|
||||
raw, err := a.bus.Call(exportCall, map[string]any{"node": node})
|
||||
var pins map[string]delivery.Provider
|
||||
var known bool
|
||||
if err == nil {
|
||||
var export map[string]delivery.Placement
|
||||
if export, err = assign.ExportOf(raw); err == nil {
|
||||
pins, known = export[node].Pins, true
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
a.logf("the controller's pins of %s were not read, so a refused pin's condition there waits: %v", node, err)
|
||||
}
|
||||
a.mu.Lock()
|
||||
now := a.pins[node]
|
||||
now.running = false
|
||||
if known {
|
||||
now.pins, now.known = pins, true
|
||||
}
|
||||
a.pins[node] = now
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
}
|
||||
return r.pins, r.known
|
||||
}
|
||||
|
||||
// sayCalls makes a refused call of the controller loud (issue 774): the condition of each pair whose last call was refused
|
||||
// is raised once per refusal, naming the verb, the node, the module and the controller's words. It is cleared when a later
|
||||
// call for the pair is answered, or once the pair, seen differing since the refusal, agrees between the two sources (a
|
||||
// person made the call by hand): for a module's place the controller's assignments and the placement, for a pin the
|
||||
// controller's pins (its placement-export, read again after its compositions change) and the placement's. The first pass
|
||||
// after a start takes over what a previous process raised (its findings) and clears each whose pair agrees; until the
|
||||
// findings are read it is tried again at every pass. A say that fails is made again at the next pass.
|
||||
func (p *passer) sayCalls(differ []assign.Pair) {
|
||||
if p.say == nil {
|
||||
return
|
||||
}
|
||||
if p.callSaid == nil {
|
||||
p.callSaid = map[string]*callSaid{}
|
||||
}
|
||||
differs := map[string]bool{}
|
||||
for _, d := range differ {
|
||||
differs[d.Node+"."+d.Module] = true
|
||||
}
|
||||
outcomes := p.seat.outcomesNow()
|
||||
if !p.callsSwept {
|
||||
p.callsSwept = p.sweepCalls(outcomes)
|
||||
}
|
||||
for key, o := range outcomes {
|
||||
said := p.callSaid[key]
|
||||
switch {
|
||||
case o.refused != "" && (said == nil || said.adopted || said.refused != o.refused):
|
||||
if err := p.say(assign.CallRefusedCondition(o.call, o.refused, true, "")); err != nil {
|
||||
p.logf("that %s was refused could not be said, and is said at the next pass: %v", o.call.Key(), err)
|
||||
continue
|
||||
}
|
||||
p.callSaid[key] = &callSaid{call: o.call, refused: o.refused}
|
||||
case o.refused == "" && said != nil:
|
||||
if err := p.say(assign.CallRefusedCondition(o.call, "", false, o.call.Key()+" was answered.")); err != nil {
|
||||
p.logf("the condition of %s could not be cleared, and is cleared at the next pass: %v", o.call.Key(), err)
|
||||
continue
|
||||
}
|
||||
delete(p.callSaid, key)
|
||||
}
|
||||
}
|
||||
var placements map[string]delivery.Placement
|
||||
for key, said := range p.callSaid {
|
||||
c := said.call
|
||||
var apart bool
|
||||
if c.CallKind() == assign.KindControllerCall {
|
||||
apart = differs[c.Node+"."+c.Module]
|
||||
} else {
|
||||
theirs, known := p.seat.pinsOf(c.Node, p.w.bucketGen(delivery.ControllerComposedBucket))
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
if placements == nil {
|
||||
placements, _ = p.seatPlacements()
|
||||
}
|
||||
ours, has := placements[c.Node].Pins[c.Module]
|
||||
to, had := theirs[c.Module]
|
||||
apart = has != had || ours != to
|
||||
}
|
||||
if apart {
|
||||
said.differed = true
|
||||
continue
|
||||
}
|
||||
if !said.differed {
|
||||
continue // not seen differing since the refusal: the world may not hold the write yet
|
||||
}
|
||||
if err := p.say(assign.CallRefusedCondition(c, "", false, agreeWords(c))); err != nil {
|
||||
p.logf("the condition of %s could not be cleared, and is cleared at the next pass: %v", c.Key(), err)
|
||||
continue
|
||||
}
|
||||
if o, ok := outcomes[key]; ok {
|
||||
p.seat.forgetOutcome(key, o)
|
||||
}
|
||||
delete(p.callSaid, key)
|
||||
}
|
||||
}
|
||||
|
||||
// sweepCalls takes over the refused calls' conditions a previous process raised, from the findings: each is followed as
|
||||
// one seen differing, so it clears at the first pass its pair agrees. It answers whether the findings were read.
|
||||
func (p *passer) sweepCalls(outcomes map[string]callOutcome) bool {
|
||||
if p.findings.Keys == nil {
|
||||
return true
|
||||
}
|
||||
keys, err := p.findings.Keys()
|
||||
if err != nil {
|
||||
p.logf("the findings were not read, so the refused controller calls a previous process raised are cleared at the next pass: %v", err)
|
||||
return
|
||||
p.logf("the findings were not read, so the refused calls a previous process raised are taken over at the next pass: %v", err)
|
||||
return false
|
||||
}
|
||||
p.callsCleared = true
|
||||
for _, k := range keys {
|
||||
rest, ok := strings.CutPrefix(k, "module.")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, kind := range []string{assign.KindControllerCall, assign.KindControllerPin} {
|
||||
if pair, ok := strings.CutSuffix(rest, "."+kind); ok {
|
||||
if node, module, two := strings.Cut(pair, "."); two {
|
||||
if err := p.say(assign.CallClearedCondition(kind, node, module)); err != nil {
|
||||
p.callsCleared = false
|
||||
p.logf("the condition of the old %s call for %s on %s could not be cleared, and is cleared at the next pass: %v", kind, module, node, err)
|
||||
}
|
||||
}
|
||||
pair, ok := strings.CutSuffix(rest, "."+kind)
|
||||
node, module, two := strings.Cut(pair, ".")
|
||||
if !ok || !two {
|
||||
continue
|
||||
}
|
||||
verb := "assign"
|
||||
if kind == assign.KindControllerPin {
|
||||
verb = "pin"
|
||||
}
|
||||
c := assign.ControllerCall{Verb: verb, Node: node, Module: module}
|
||||
key := assign.CallConditionKey(c)
|
||||
if _, made := outcomes[key]; made || p.callSaid[key] != nil {
|
||||
continue
|
||||
}
|
||||
p.callSaid[key] = &callSaid{call: c, differed: true, adopted: true}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func agreeWords(c assign.ControllerCall) string {
|
||||
if c.CallKind() == assign.KindControllerPin {
|
||||
return "the controller's pins and the placement agree on " + c.Module + " for " + c.Node + "."
|
||||
}
|
||||
return "the controller's assignments and the placement agree on " + c.Module + " on " + c.Node + "."
|
||||
}
|
||||
|
||||
// callSaid is a raised call's condition: its call, the refusal it said, whether its pair was seen differing since, and
|
||||
// whether it was taken over from a previous process.
|
||||
type callSaid struct {
|
||||
call assign.ControllerCall
|
||||
refused string
|
||||
differed bool
|
||||
adopted bool
|
||||
}
|
||||
|
||||
// forgetOutcome drops a refused call's outcome once its condition is cleared, unless a later call replaced it.
|
||||
func (a *seatActs) forgetOutcome(key string, o callOutcome) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if now, ok := a.outcomes[key]; ok && now.refused == o.refused && now.call.Verb == o.call.Verb {
|
||||
delete(a.outcomes, key)
|
||||
}
|
||||
}
|
||||
|
||||
// callOf is what the last call for a pair came to, "" when none was made.
|
||||
func (a *seatActs) callOf(node, module string) string {
|
||||
if a == nil {
|
||||
return ""
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.calls[node+"."+module]
|
||||
}
|
||||
|
||||
// withCalls names, on each pair the two sources do not both name, the call the assigner made for it (§8).
|
||||
func (p *passer) withCalls(pairs []assign.Pair) []assign.Pair {
|
||||
for i := range pairs {
|
||||
pairs[i].Call = p.seat.callOf(pairs[i].Node, pairs[i].Module)
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
|
||||
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
|
||||
@@ -263,18 +544,17 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos
|
||||
p.endProposal(ctx, prop, prop.Withdrawn)
|
||||
return
|
||||
}
|
||||
if prop.Seed || prop.HandBack {
|
||||
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
|
||||
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone")
|
||||
return
|
||||
}
|
||||
akey, _ := delivery.PlacementAskKey(prop.ID)
|
||||
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
|
||||
if prop.State == delivery.ProposalAsked {
|
||||
askPulse, _ := delivery.PulseAskKey(prop.ID)
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
if prop.HandBack {
|
||||
p.handBack(ctx, prop, akey, revs)
|
||||
} else {
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
}
|
||||
return
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
p.endProposal(ctx, prop, "declined by the operator")
|
||||
@@ -284,6 +564,19 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos
|
||||
return
|
||||
}
|
||||
}
|
||||
if prop.HandBack {
|
||||
// The operator's alone (§7): no node answers it; its ask is raised at once, and again on again.
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
brief := fmt.Sprintf("%s (asked by %s): every node's placement key is deleted, %s, and each node returns to the "+
|
||||
"controller's compositions; the history is kept", prop.Why, prop.By, strings.Join(prop.Nodes(), ", "))
|
||||
now := p.clock().UTC()
|
||||
prop.Verdict = &delivery.PlacementVerdict{State: delivery.VerdictPass, At: now}
|
||||
if prop, ok := p.ask(ctx, prop, akey, brief); ok {
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
|
||||
return
|
||||
@@ -454,7 +747,8 @@ func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal
|
||||
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded,
|
||||
Seed: prop.Seed, ExportDigest: prop.ExportDigest, Leased: p.w.handedAway()})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
@@ -467,6 +761,7 @@ func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal,
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
|
||||
p.seat.forget(prop.ID)
|
||||
p.writeProposal(ctx, again, rev)
|
||||
}
|
||||
|
||||
@@ -483,14 +778,35 @@ func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
|
||||
}
|
||||
|
||||
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, and placement-changed said.
|
||||
// warranted writes an approved proposal (§5): for the seed, after the export is read again and found unchanged (§7);
|
||||
// each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, placement-changed said, and the controller kept in step (§8).
|
||||
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
if prop.Seed {
|
||||
if p.seat == nil {
|
||||
return
|
||||
}
|
||||
c := p.seat.exportOf(prop.ID)
|
||||
switch {
|
||||
case c.running:
|
||||
return // its answer wakes the next pass
|
||||
case c.err != "":
|
||||
if words := "the export cannot be read again at the warrant: " + c.err; prop.Words != words {
|
||||
prop.Words = words
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
return
|
||||
case delivery.ExportDigest(c.export) != prop.ExportDigest:
|
||||
p.endProposal(ctx, prop, "the controller's export changed since the seed was proposed: the new export is proposed as the seed")
|
||||
p.proposeSeed(ctx, prop, c.export)
|
||||
return
|
||||
}
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
@@ -515,7 +831,7 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal,
|
||||
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
|
||||
// again over the keys as this write left them.
|
||||
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
|
||||
p.followWrite(prop, answer, changes)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
if rest, ok := assign.Rest(prop, writtenLaid); ok {
|
||||
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
|
||||
} else {
|
||||
@@ -541,17 +857,22 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal,
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.followWrite(prop, answer, changes)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the
|
||||
// controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded, placement-changed said, and, until the lease
|
||||
// write, the controller's assignments kept in step for them (§8). The seed calls no controller verb: it is what the
|
||||
// controller holds already (§9).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange,
|
||||
before map[string]delivery.Placement, written map[string]*delivery.Placement) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
if p.seat != nil && !p.w.handedAway() && !prop.Seed {
|
||||
p.seat.callAll(assign.ControllerCalls(before, written))
|
||||
}
|
||||
}
|
||||
|
||||
func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
@@ -563,6 +884,44 @@ func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// handBack deletes every key on the operator's warrant (§7) at the revision the hand-back read, records it in the
|
||||
// controller's hand-act log and says placement-changed; every node returns to the controller's compositions. A key moved
|
||||
// since ends it, and the operator hands back again.
|
||||
func (p *passer) handBack(ctx context.Context, prop delivery.PlacementProposal, akey string, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind this hand-back, so nothing is deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
current, held := p.seatPlacements()
|
||||
var changes []delivery.PlacementChange
|
||||
for _, node := range prop.Nodes() {
|
||||
key, _ := delivery.PlacementNodeKey(node)
|
||||
err := error(nil)
|
||||
if held[node] != prop.Base[node] {
|
||||
err = fmt.Errorf("its placement is at revision %d, the hand-back read %d", held[node], prop.Base[node])
|
||||
} else {
|
||||
_, err = p.writeNode(ctx, node, nil)
|
||||
}
|
||||
if err != nil {
|
||||
p.sayChanged(prop, answer, changes)
|
||||
p.endProposal(ctx, prop, fmt.Sprintf("%s moved since the hand-back was asked (%v): hand back again", key, err))
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(current[node], delivery.Placement{Node: node})
|
||||
c.Before = prop.Base[node]
|
||||
changes = append(changes, c)
|
||||
}
|
||||
if p.seat != nil {
|
||||
p.seat.handAct(prop)
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
}
|
||||
|
||||
// sayChanged says placement-changed for the keys written.
|
||||
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
@@ -579,6 +938,26 @@ func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Ans
|
||||
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
|
||||
}
|
||||
|
||||
// proposeSeed proposes an export as the seed again, by the same proposer and why.
|
||||
func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal, export map[string]delivery.Placement) {
|
||||
held, recorded := p.lays(ctx)
|
||||
now := p.clock()
|
||||
seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: p.w.handedAway()})
|
||||
if len(findings) > 0 {
|
||||
p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
for _, node := range seed.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(seed.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: seed.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
seed.Written = now.UTC()
|
||||
p.writeProposal(ctx, seed, 0)
|
||||
}
|
||||
|
||||
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
|
||||
// whole with why it ended.
|
||||
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
|
||||
@@ -594,6 +973,7 @@ func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposa
|
||||
return
|
||||
}
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.seat.forget(prop.ID)
|
||||
p.logf("the proposal %s ends: %s", prop.ID, why)
|
||||
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -206,7 +207,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
current, revs, err := s.placements()
|
||||
@@ -219,7 +220,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
@@ -269,15 +270,92 @@ func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// doAdopt is the verb adopt (§7): gone since the lease write. The seed was the controller's export proposed whole, and the
|
||||
// mesh has no controller export to take over (ADR 0547 §11, ADR 0567 §11); nothing is written.
|
||||
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
|
||||
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
|
||||
|
||||
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
|
||||
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
|
||||
return nil, errors.New("adopt is gone: placement is the mesh's alone since the lease write, and there is no controller to take it over from (ADR 0547 §11, ADR 0567 §11)")
|
||||
current, _, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(current) > 0 {
|
||||
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
|
||||
}
|
||||
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
|
||||
for _, id := range ids {
|
||||
if p, err := s.proposal(id); err == nil && p.Seed {
|
||||
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := s.ask(exportCall, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
|
||||
}
|
||||
export, err := assign.ExportOf(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// doHandBack is the verb hand-back (§7): gone since the lease write, as adopt is; nothing is deleted.
|
||||
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
|
||||
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
|
||||
// unreadable, is not handed away.
|
||||
func (s *seatTools) leased() bool {
|
||||
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
|
||||
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
|
||||
// records the hand act and says placement-changed; every node returns to the controller's compositions.
|
||||
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
|
||||
return nil, errors.New("hand-back is gone: placement is the mesh's alone since the lease write, and there is no controller to hand it back to (ADR 0547 §11, ADR 0567 §11)")
|
||||
if s.leased() {
|
||||
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
|
||||
}
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("hand-back says why: it is recorded with the hand act")
|
||||
}
|
||||
if by == "" {
|
||||
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
|
||||
}
|
||||
_, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(revs) == 0 {
|
||||
return nil, errors.New("there is no placement key to hand back")
|
||||
}
|
||||
now := s.now()
|
||||
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
|
||||
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the hand-back is not written: %w", err)
|
||||
}
|
||||
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
|
||||
"node returns to the controller's compositions"}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
@@ -36,8 +37,8 @@ func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
}
|
||||
|
||||
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt and hand-back are
|
||||
// answered as gone and change nothing (ADR 0547 §11, ADR 0567 §11); placement and
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt proposes the
|
||||
// controller's export whole as the seed, once; hand-back records the hand act and deletes every key; placement and
|
||||
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
|
||||
func TestTheSeatsVerbs(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
@@ -52,35 +53,32 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
|
||||
st[haveState].values[n+".node"] = raw
|
||||
}
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor})
|
||||
var asked []string
|
||||
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
if key == exportCall {
|
||||
out, _ := json.Marshal(map[string]any{"ok": true, "output": string(export)})
|
||||
return out, nil
|
||||
}
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
// 774/S28/AC1: adopt and hand-back are answered as gone, naming their records, and change nothing and ask nobody.
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
for verb, call := range map[string]func() (any, error){
|
||||
"adopt": func() (any, error) { return s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop") },
|
||||
"hand-back": func() (any, error) { return s.doHandBack(map[string]any{"why": "back"}, "op") },
|
||||
} {
|
||||
if _, err := call(); err == nil || !strings.Contains(err.Error(), verb+" is gone") || !strings.Contains(err.Error(), "ADR 0567 §11") {
|
||||
t.Errorf("%s: %v", verb, err)
|
||||
}
|
||||
}
|
||||
if len(asked) != 0 || len(st[delivery.PlacementProposalsState].values) != 0 || len(st[delivery.PlacementCandidatesState].values) != 0 ||
|
||||
len(st[delivery.PlacementState].values) != 1 {
|
||||
t.Fatalf("adopt or hand-back changed something: asked %v", asked)
|
||||
}
|
||||
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
|
||||
// adopt: the export proposed whole as the seed, once.
|
||||
got, err := s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := got.(proposed).Proposal
|
||||
if !seed.Seed || seed.ExportDigest != delivery.ExportDigest(map[string]delivery.Placement{"anchor": anchor}) || !slices.Equal(asked, []string{exportCall}) ||
|
||||
len(st[delivery.PlacementCandidatesState].values) != 1 || st[delivery.PlacementProposalsState].values[seed.ID] == nil {
|
||||
t.Fatalf("adopt = %+v, asked %v", seed, asked)
|
||||
}
|
||||
if _, err := s.doAdopt(map[string]any{"why": "again"}, "op"); err == nil || !strings.Contains(err.Error(), "is open") {
|
||||
t.Errorf("a second adopt with a seed open: %v", err)
|
||||
}
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -97,8 +95,10 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
|
||||
|
||||
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
|
||||
!strings.Contains(err.Error(), "ghost") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
!strings.Contains(err.Error(), "ghost") || !strings.Contains(err.Error(), "lease write") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
t.Errorf("a refused proposal: %v", err)
|
||||
}
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
|
||||
@@ -146,20 +146,32 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand placement back to.
|
||||
// hand-back: the operator's alone, written as a hand-back proposal naming every key at its revision, for the assigner
|
||||
// to ask; nothing is deleted by the verb.
|
||||
got, err = s.doHandBack(map[string]any{"why": "back"}, "op")
|
||||
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
|
||||
t.Errorf("hand-back = %+v, %v", got, err)
|
||||
}
|
||||
// Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and
|
||||
// there is no controller to hand placement back to.
|
||||
st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}}
|
||||
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
|
||||
t.Errorf("a hold: %v", err)
|
||||
t.Errorf("a hold after the lease write: %v", err)
|
||||
}
|
||||
if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") {
|
||||
t.Errorf("a hand-back: %v", err)
|
||||
t.Errorf("a hand-back after the lease write: %v", err)
|
||||
}
|
||||
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the controller's composition until the lease write, the node's own after; an unplaced node is said", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
|
||||
ckey, _ := delivery.ComposedKey("ace", "web", fp)
|
||||
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedNeither,
|
||||
Policy: catalogue.PolicyRoll, PolicyFrom: catalogue.FromDefault})
|
||||
w.putEntry(delivery.ControllerComposedBucket, ckey, cv, 1, time.Now())
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey, _ := delivery.LegacyPlacementKey("ace")
|
||||
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
|
||||
@@ -168,11 +180,15 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
hk, _ := delivery.HaveNodeKey(node)
|
||||
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
|
||||
}
|
||||
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("c", 64) || len(w.unplacedNodes()) != 0 {
|
||||
t.Fatalf("before the lease write: %+v %v, unplaced %v", d, ok, w.unplacedNodes())
|
||||
}
|
||||
w.putEntry(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator"}`), 2, time.Now())
|
||||
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Errorf("ace: %+v %v", d, ok)
|
||||
t.Errorf("after the lease write, ace: %+v %v", d, ok)
|
||||
}
|
||||
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
|
||||
t.Errorf("g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
|
||||
t.Errorf("after the lease write, g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
|
||||
}
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
|
||||
@@ -340,7 +356,7 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
func wantsVerbs(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
|
||||
st := memStore{}
|
||||
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings"} {
|
||||
for _, state := range []string{modulesState, manifestsState, composedState, policiesState, haveState, putsState, "findings"} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
manifest := func(name, extra string) {
|
||||
@@ -355,7 +371,13 @@ func wantsVerbs(t *testing.T) {
|
||||
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
|
||||
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
}
|
||||
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
composed := func(module, policy string) {
|
||||
key, _ := delivery.ComposedKey("anchor", module, fpNew)
|
||||
st[composedState].values[key], _ = json.Marshal(delivery.ComposedDigest{V: delivery.Version, Digest: "sha256:" + strings.Repeat("c", 64),
|
||||
Judgement: delivery.JudgedNeither, Policy: policy, PolicyFrom: catalogue.FromPerson, PolicyWhy: "chosen at the controller"})
|
||||
}
|
||||
composed("web", catalogue.PolicyRecord)
|
||||
composed("photos", catalogue.PolicyRecord)
|
||||
var asked []string
|
||||
var said []string
|
||||
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -375,11 +397,10 @@ func wantsVerbs(t *testing.T) {
|
||||
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
|
||||
t.Errorf("nats = %+v", p)
|
||||
}
|
||||
// compare: the controller writes no policy, so no module splits; each node's own composition is the digest its have names.
|
||||
// compare: web rolls at the assigner and records at the controller.
|
||||
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if got := out.(map[string]any); len(got["splits"].([]assign.Split)) != 0 || got["empty"] != true || got["findings"] == nil ||
|
||||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
|
||||
t.Errorf("compare = %+v", got)
|
||||
if splits := out.(map[string]any)["splits"].([]assign.Split); len(splits) != 1 || splits[0].Module != "web" || out.(map[string]any)["findings"] == nil {
|
||||
t.Errorf("splits = %+v", splits)
|
||||
}
|
||||
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
|
||||
// large states (the modules, their manifests, the composed record).
|
||||
@@ -401,14 +422,14 @@ func wantsVerbs(t *testing.T) {
|
||||
if werr != nil {
|
||||
t.Fatal(werr)
|
||||
}
|
||||
if got := wout.(map[string]any); got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) {
|
||||
t.Errorf("compare by watch = %+v", got)
|
||||
if ws := wout.(map[string]any)["splits"].([]assign.Split); len(ws) != 1 || ws[0].Module != "web" {
|
||||
t.Errorf("splits by watch = %+v", ws)
|
||||
}
|
||||
}
|
||||
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
|
||||
t.Errorf("the manifests were read by key %d times, want each once", n)
|
||||
}
|
||||
for _, state := range []string{modulesState, haveState, health.FindingsState} {
|
||||
for _, state := range []string{modulesState, composedState, health.FindingsState} {
|
||||
if watches[state] != 1 {
|
||||
t.Errorf("%s was watched %d times, want once", state, watches[state])
|
||||
}
|
||||
@@ -428,6 +449,9 @@ func wantsVerbs(t *testing.T) {
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil },
|
||||
Keys: func() ([]string, error) { return []string{"module.gone.policy-split"}, nil }}}
|
||||
for k, v := range st[composedState].values {
|
||||
p.w.putEntry(delivery.ControllerComposedBucket, k, v, 1, time.Now())
|
||||
}
|
||||
man := map[string]catalogue.Manifest{}
|
||||
for _, name := range []string{"web", "photos"} {
|
||||
raw, _, _, _ := st.get(modulesState, name)
|
||||
@@ -437,10 +461,9 @@ func wantsVerbs(t *testing.T) {
|
||||
man[name], _ = catalogue.Decode([]byte(kept.JSON))
|
||||
}
|
||||
findings["module.gone.policy-split"] = []byte("{}")
|
||||
// The controller writes no policy now (ADR 0571 §6): the process compares with nothing, and deletes a finding left.
|
||||
p.policySplits(man, nil)
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("findings = %v, want a stale one deleted", findings)
|
||||
if _, ok := findings[assign.PolicySplitKey("web")]; !ok || len(findings) != 1 {
|
||||
t.Errorf("findings = %v, want web's split alone (a stale one deleted)", findings)
|
||||
}
|
||||
// a person's record for web: written, recorded, said; the split is gone.
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
@@ -19,14 +20,15 @@ import (
|
||||
|
||||
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
|
||||
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
|
||||
// own have names (the controller writes no policy any more, so no module's policy differs); `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from
|
||||
// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// `policies` when it judges a move and writes the finding policy-split while a split stands.
|
||||
|
||||
// The states the verbs read, as the runtime names them.
|
||||
const (
|
||||
policiesState = delivery.AssignerPoliciesState
|
||||
manifestsState = delivery.IndexerModule + ".manifests"
|
||||
composedState = "mesh-controller.composed"
|
||||
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
|
||||
PolicyChangedEvent = "policy-changed"
|
||||
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
|
||||
@@ -54,6 +56,13 @@ func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error),
|
||||
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
|
||||
}
|
||||
|
||||
// leased says the controller's lease is handed away.
|
||||
func (s *wantsTools) leased() bool {
|
||||
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// values reads every key of a state: from the watched copy when there is one, else key by key.
|
||||
func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
@@ -145,23 +154,15 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
chosen, unreadable := assign.DecodePolicies(raw)
|
||||
if arg(args, "compare") == "true" {
|
||||
haves, err := s.values(haveState)
|
||||
composed, err := s.values(composedState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
composed := map[string]string{}
|
||||
for key, raw := range haves {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
|
||||
composed[node] = h.Composed
|
||||
}
|
||||
}
|
||||
}
|
||||
splits := assign.Splits(manifests, chosen, nil) // the controller writes no policy to differ from (ADR 0571 §6)
|
||||
splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed))
|
||||
if splits == nil {
|
||||
splits = []assign.Split{}
|
||||
}
|
||||
out := map[string]any{"splits": splits, "empty": len(splits) == 0, "composed": composed}
|
||||
out := map[string]any{"splits": splits, "empty": len(splits) == 0}
|
||||
// The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15).
|
||||
if found, err := s.values(health.FindingsState); err != nil {
|
||||
out["findings-unread"] = err.Error()
|
||||
@@ -228,7 +229,7 @@ func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[str
|
||||
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
|
||||
}
|
||||
}
|
||||
rec := handacts.Recorder{Ask: s.ask}
|
||||
rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased}
|
||||
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
|
||||
recorded := taken.Log
|
||||
if rerr != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -26,6 +28,18 @@ import (
|
||||
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
|
||||
const noModuleWords = delivery.CandidateNoneNoModule
|
||||
|
||||
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
|
||||
// internal/view/view.go): an assignment stands while it has not ended and has no state.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Build string `json:"build"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
|
||||
// digest. The watches write it; a pass reads it.
|
||||
type world struct {
|
||||
@@ -196,15 +210,38 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules the seat mesh-placement's placement names, in name order (ADR 0549 §5, ADR 0567 §8): the
|
||||
// placement is the one source of where modules run. The controller's assignments are read no more (issue 774).
|
||||
func (w *world) assigned() map[string][]string {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
|
||||
// the lease write's own check refuses that order (ADR 0547 §11).
|
||||
placed, seeded := w.placed()
|
||||
byAssignment, all := w.assignedByController()
|
||||
if away := w.handedAway(); away && seeded {
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, away)
|
||||
return both, map[string]assignment{}
|
||||
}
|
||||
return out
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, false)
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
|
||||
// the controller's lease is handed away, since the placement is then the one source.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
placed, seeded := w.placed()
|
||||
byAssignment, _ := w.assignedByController()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
|
||||
return differ
|
||||
}
|
||||
|
||||
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
|
||||
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
|
||||
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
|
||||
func (w *world) handedAway() bool {
|
||||
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
|
||||
return away
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
@@ -234,6 +271,24 @@ func (w *world) placedNodes() map[string]bool {
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
|
||||
// name order.
|
||||
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
|
||||
continue
|
||||
}
|
||||
out[a.Node] = append(out[a.Node], a.Module)
|
||||
all[a.Node+"."+a.Module] = a
|
||||
}
|
||||
for _, mods := range out {
|
||||
sort.Strings(mods)
|
||||
}
|
||||
return out, all
|
||||
}
|
||||
|
||||
// trunk is every module the indexer holds at the trunk, with its value.
|
||||
func (w *world) trunk() map[string]delivery.Module {
|
||||
out := map[string]delivery.Module{}
|
||||
@@ -252,7 +307,7 @@ func (w *world) trunk() map[string]delivery.Module {
|
||||
// mesh is the mesh as the trunk holds it, for the composition check.
|
||||
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
|
||||
m.Assigned = w.assigned()
|
||||
m.Assigned, _ = w.assigned()
|
||||
for name, mod := range w.trunk() {
|
||||
if mod.Manifest == "" {
|
||||
continue
|
||||
@@ -557,14 +612,49 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
|
||||
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
|
||||
// controller's copy (mesh-controller_composed) is read no more (issue 774).
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
//
|
||||
// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a
|
||||
// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets
|
||||
// no want (and the pass says so, unplacedFindings).
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
if !w.placedNodes()[node] {
|
||||
if w.handedAway() {
|
||||
if !w.placedNodes()[node] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return w.ownComposed(node)
|
||||
}
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return w.ownComposed(node)
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
if w.handedAway() {
|
||||
return out // the controller composes no node from the lease write (ADR 0571 §6)
|
||||
}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
@@ -629,7 +719,7 @@ func (w *world) runningCommits(module string) []string {
|
||||
var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState
|
||||
|
||||
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
|
||||
// what rule 7 reads. A node whose have names none is not composed yet.
|
||||
// what rule 7 reads from the lease write. A node whose have names none is not composed yet.
|
||||
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
@@ -647,9 +737,12 @@ func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
Words: "the node composes its own"}, true
|
||||
}
|
||||
|
||||
// unplacedNodes are the nodes whose engine has a have and that no placement names: they get no
|
||||
// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no
|
||||
// want, since nothing composes them.
|
||||
func (w *world) unplacedNodes() []string {
|
||||
if !w.handedAway() {
|
||||
return nil
|
||||
}
|
||||
placed := w.placedNodes()
|
||||
var out []string
|
||||
for key := range w.read(delivery.HaveBucket) {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,23 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027 h1:a33dsavoaqllh40eVICbhxi6Kero1EGSjt1V+2fa2Hk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101085317-790d1c82d027/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364 h1:UUDPWfUjyerptar+wPwI891jTUAtbbuXori0j3phTE4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3 h1:lb6hz+SBsoCAzsiCCvdtPKyE66Y5Ng/DBAkonAexPzA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -117,6 +117,7 @@ func saidConditionsAreFindings(t *testing.T) {
|
||||
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
|
||||
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
|
||||
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
|
||||
{PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"},
|
||||
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
|
||||
} {
|
||||
|
||||
@@ -1,12 +1,122 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
|
||||
// the seat mesh-placement's placement (ADR 0549 §5, ADR 0567 §8). The assigner writes wants only for a pair both name:
|
||||
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
|
||||
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
|
||||
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
|
||||
// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease
|
||||
// handed away) the placement is the one source.
|
||||
|
||||
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
|
||||
// OnlyPlacement.
|
||||
type Pair struct {
|
||||
Node, Module, Only string
|
||||
// Call is what became of the assigner's call of the controller's verb for the pair, in words, when it made one.
|
||||
Call string
|
||||
}
|
||||
|
||||
// The source that names a pair alone.
|
||||
const (
|
||||
OnlyAssignments = "the controller's assignments"
|
||||
OnlyPlacement = "the seat mesh-placement's placement"
|
||||
)
|
||||
|
||||
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
|
||||
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's
|
||||
// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the
|
||||
// assignments are then a source of nothing.
|
||||
func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) {
|
||||
if handedAway && seeded {
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
if !seeded {
|
||||
return assigned, nil
|
||||
}
|
||||
set := func(m map[string][]string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for node, modules := range m {
|
||||
for _, module := range modules {
|
||||
out[node+"\x00"+module] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
inAssigned, inPlaced := set(assigned), set(placed)
|
||||
both := map[string][]string{}
|
||||
var differ []Pair
|
||||
for node, modules := range assigned {
|
||||
for _, module := range modules {
|
||||
if inPlaced[node+"\x00"+module] {
|
||||
both[node] = append(both[node], module)
|
||||
} else {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
|
||||
}
|
||||
}
|
||||
}
|
||||
for node, modules := range placed {
|
||||
for _, module := range modules {
|
||||
if !inAssigned[node+"\x00"+module] {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, modules := range both {
|
||||
sort.Strings(modules)
|
||||
}
|
||||
sort.Slice(differ, func(i, j int) bool {
|
||||
if differ[i].Node != differ[j].Node {
|
||||
return differ[i].Node < differ[j].Node
|
||||
}
|
||||
return differ[i].Module < differ[j].Module
|
||||
})
|
||||
return both, differ
|
||||
}
|
||||
|
||||
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
|
||||
// both name it or neither does.
|
||||
func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
|
||||
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("the controller's assignments and the seat's placement agree on %s on %s.", p.Module, p.Node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
var needs string
|
||||
if p.Only == OnlyAssignments {
|
||||
needs = fmt.Sprintf("propose placing %s on %s (mesh-placement.propose), or unassign it at the controller", p.Module, p.Node)
|
||||
} else {
|
||||
needs = fmt.Sprintf("assign %s to %s at the controller, as the approved placement says, or propose taking it off", p.Module, p.Node)
|
||||
}
|
||||
if p.Call != "" {
|
||||
needs += "; the assigner's call: " + p.Call
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
|
||||
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
|
||||
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
|
||||
}
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) asks.Condition {
|
||||
|
||||
@@ -48,6 +48,26 @@ func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (p
|
||||
return catalogue.Upgrade(m, c)
|
||||
}
|
||||
|
||||
// ControllerPolicies is, by module, every policy the controller writes beside a composition's digest of it.
|
||||
func ControllerPolicies(composed map[string][]byte) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for key, raw := range composed {
|
||||
_, module, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(raw)
|
||||
if err != nil || d.Policy == "" {
|
||||
continue
|
||||
}
|
||||
if !contains(out[module], d.Policy) {
|
||||
out[module] = append(out[module], d.Policy)
|
||||
sort.Strings(out[module])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
|
||||
@@ -68,7 +68,7 @@ func StepPut(w PutWorld) PutNext {
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -17,23 +19,30 @@ import (
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
type ProposeIn struct {
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
Seed bool
|
||||
ExportDigest string
|
||||
// Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk
|
||||
// buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then
|
||||
// the holder keeps the controller in step and refuses what its verbs cannot take.
|
||||
Leased bool
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change the
|
||||
// controller's verbs cannot take before the lease write (§8), a change that does not fit its node, and what
|
||||
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
|
||||
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
|
||||
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
|
||||
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
|
||||
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
|
||||
Changes: in.Changes, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
Changes: in.Changes, Seed: in.Seed, ExportDigest: in.ExportDigest, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
var findings []string
|
||||
if p.By == "" {
|
||||
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
|
||||
@@ -45,8 +54,14 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla
|
||||
findings = append(findings, "a proposal changes something: it names no change")
|
||||
}
|
||||
for i, c := range in.Changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
|
||||
if (c.Op == delivery.OpSeed) != in.Seed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
|
||||
continue
|
||||
}
|
||||
if !in.Leased {
|
||||
if why := c.BeforeLease(); why != "" {
|
||||
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
|
||||
}
|
||||
}
|
||||
}
|
||||
laid, lay := delivery.LayChanges(in.Current, in.Changes)
|
||||
@@ -103,19 +118,112 @@ func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) (
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The kinds of the condition (and finding) of a refused controller call, which the assigner raised until it stopped calling
|
||||
// the controller's verbs (ADR 0567 §8, issue 774).
|
||||
// ControllerCall is one call of the controller's seat verbs the holder makes after writing an approved proposal, until
|
||||
// the lease write (§8): assign, unassign, pin or unpin, with its arguments.
|
||||
type ControllerCall struct {
|
||||
Verb string
|
||||
Args map[string]any
|
||||
// Node and Module name the pair the call keeps in step (a pin names no module).
|
||||
Node, Module string
|
||||
}
|
||||
|
||||
// Key names the call, as the condition of its pair says it.
|
||||
func (c ControllerCall) Key() string {
|
||||
return fmt.Sprintf("mesh-controller.%s %s %s", c.Verb, c.Node, c.Module)
|
||||
}
|
||||
|
||||
// ControllerCalls is every call that brings the controller's assignments and pins to the placement after: assign for a
|
||||
// module placed on a node before not, unassign for one taken off, pin for a pin set or changed, unpin for one removed. In
|
||||
// node order, unassigns before assigns, so a module moved between nodes is first taken off.
|
||||
//
|
||||
// Each call carries only the arguments the controller's seat contract declares for its verb (catalogue.ControllerVerbs):
|
||||
// the controller refuses any other argument and does nothing (issue 774: a "why" added to every call had every approved
|
||||
// placement refused since 2026-10-28). The proposal and its warrant are in the assigner's own record instead: the
|
||||
// placement-changed event and the hand-act log.
|
||||
func ControllerCalls(before map[string]delivery.Placement, after map[string]*delivery.Placement) []ControllerCall {
|
||||
nodes := make([]string, 0, len(after))
|
||||
for node := range after {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var off, on, pins []ControllerCall
|
||||
for _, node := range nodes {
|
||||
was, now := before[node], delivery.Placement{}
|
||||
if a := after[node]; a != nil {
|
||||
now = *a
|
||||
}
|
||||
had, has := map[string]bool{}, map[string]bool{}
|
||||
for _, m := range was.Modules {
|
||||
had[m.Module] = true
|
||||
}
|
||||
for _, m := range now.Modules {
|
||||
has[m.Module] = true
|
||||
if !had[m.Module] {
|
||||
on = append(on, ControllerCall{Verb: "assign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, m := range was.Modules {
|
||||
if !has[m.Module] {
|
||||
off = append(off, ControllerCall{Verb: "unassign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(now.Pins) {
|
||||
to := now.Pins[provision]
|
||||
if old, ok := was.Pins[provision]; !ok || old != to {
|
||||
pins = append(pins, ControllerCall{Verb: "pin", Node: node, Module: provision,
|
||||
Args: map[string]any{"node": node, "provision": provision, "from": to.Node, "module": to.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(was.Pins) {
|
||||
if _, ok := now.Pins[provision]; !ok {
|
||||
pins = append(pins, ControllerCall{Verb: "unpin", Node: node, Module: provision, Args: map[string]any{"node": node, "provision": provision}})
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(append(off, on...), pins...)
|
||||
}
|
||||
|
||||
// CallKind is the kind of the condition of a refused call: one for a module's place (assign, unassign), one for a pin.
|
||||
func (c ControllerCall) CallKind() string {
|
||||
if c.Verb == "pin" || c.Verb == "unpin" {
|
||||
return KindControllerPin
|
||||
}
|
||||
return KindControllerCall
|
||||
}
|
||||
|
||||
// The kinds of a refused controller call's condition (and finding).
|
||||
const (
|
||||
KindControllerCall = "controller-call"
|
||||
KindControllerPin = "controller-pin"
|
||||
)
|
||||
|
||||
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
|
||||
// module, or node and provision for a pin).
|
||||
func CallClearedCondition(kind, node, module string) asks.Condition {
|
||||
return asks.Condition{Key: delivery.AssignerModule + "." + kind + "." + node + "." + module,
|
||||
Subject: asks.Scope{Scope: "module", ID: module, Machine: node}, State: asks.StateCleared,
|
||||
Resolved: "the assigner calls no verb of the controller any more: placement is the one source."}
|
||||
// CallConditionKey is the key of the condition of a call refused for its pair (node and module, or node and provision).
|
||||
func CallConditionKey(c ControllerCall) string {
|
||||
return delivery.AssignerModule + "." + c.CallKind() + "." + c.Node + "." + c.Module
|
||||
}
|
||||
|
||||
// CallRefusedCondition is the condition the assigner raises when the controller refused (or did not answer) a call that
|
||||
// keeps its assignments in step with an approved placement (ADR 0567 §8, issue 774): the placement is written, the
|
||||
// controller's assignments are not, so nothing new reaches the node (no bus account, no composition). It names the verb,
|
||||
// the node, the module (or provision) and the controller's words, keyed by the pair, and is cleared (raised false) when a
|
||||
// later call for the pair is answered, or both sources agree on it, with resolved saying which.
|
||||
func CallRefusedCondition(c ControllerCall, refusal string, raised bool, resolved string) asks.Condition {
|
||||
key := CallConditionKey(c)
|
||||
subject := asks.Scope{Scope: "module", ID: c.Module, Machine: c.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: resolved}
|
||||
}
|
||||
headline := fmt.Sprintf("the controller refused %s %s on %s", c.Verb, c.Module, c.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
needs := fmt.Sprintf("call mesh-controller.%s for %s on %s by hand once the refusal's cause is fixed", c.Verb, c.Module, c.Node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. An approved placement was written, and the assigner's call %s to keep the controller's "+
|
||||
"assignments in step was refused: %s. Until the controller holds it, the placement and the assignments differ, so "+
|
||||
"the assigner writes no want for the pair and nothing new reaches %s. This clears when a call for it is answered "+
|
||||
"or both sources agree.", asks.NeedsYou, needs, c.Key(), refusal, c.Node)}
|
||||
}
|
||||
|
||||
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
|
||||
@@ -138,6 +246,9 @@ func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking
|
||||
}
|
||||
nodes := p.Nodes()
|
||||
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
|
||||
if p.Seed {
|
||||
headline = "Take placement over from the controller?"
|
||||
}
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
|
||||
}
|
||||
@@ -157,6 +268,57 @@ func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking
|
||||
}, Asking: asking}, nil
|
||||
}
|
||||
|
||||
// ExportOf reads the controller's answer to placement-export (§7): its verb answers what the command printed, the
|
||||
// JSON of every node's placement, in `output` (the controller's verbAnswer; `answer` when it carries it as data). A
|
||||
// refused or unreadable answer is an error naming it.
|
||||
func ExportOf(raw []byte) (map[string]delivery.Placement, error) {
|
||||
var a struct {
|
||||
Output string `json:"output"`
|
||||
OK *bool `json:"ok"`
|
||||
Answer json.RawMessage `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export answered no verb answer: %w", err)
|
||||
}
|
||||
if a.OK != nil && !*a.OK {
|
||||
return nil, fmt.Errorf("the controller's placement-export refused: %s", strings.TrimSpace(a.Output))
|
||||
}
|
||||
var byNode map[string]json.RawMessage
|
||||
if len(a.Answer) > 0 && string(a.Answer) != "null" {
|
||||
if err := json.Unmarshal(a.Answer, &byNode); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export is no placement by node: %w", err)
|
||||
}
|
||||
} else if err := json.NewDecoder(strings.NewReader(a.Output)).Decode(&byNode); err != nil {
|
||||
// The command prints its JSON first; what its standard error said after it is left.
|
||||
return nil, fmt.Errorf("the controller's placement-export printed no placement by node: %w", err)
|
||||
}
|
||||
out := map[string]delivery.Placement{}
|
||||
for node, rawP := range byNode {
|
||||
p, err := delivery.DecodePlacement(rawP)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if p.Node != node {
|
||||
return nil, fmt.Errorf("the controller's placement-export names %s under %s", p.Node, node)
|
||||
}
|
||||
out[node] = p
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("the controller's placement-export names no node")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SeedChanges is the seed's changes: every node's placement of the export, whole, in node order.
|
||||
func SeedChanges(export map[string]delivery.Placement) []delivery.PlacementProposalChange {
|
||||
var out []delivery.PlacementProposalChange
|
||||
for _, node := range sortedKeys(export) {
|
||||
p := export[node]
|
||||
out = append(out, delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: node, Whole: &p})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
|
||||
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
|
||||
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -37,30 +38,39 @@ func TestThePlacementSeatDecides(t *testing.T) {
|
||||
in ProposeIn
|
||||
want string
|
||||
}{
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
"a port before the lease": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}}), "lease write"},
|
||||
"a holding": {in(delivery.PlacementProposalChange{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "mesh-store", Scope: "mesh"}), "lease write"},
|
||||
"a binding": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}), "lease write"},
|
||||
"a config place": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}), "lease write"},
|
||||
"a config access": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}}), "lease write"},
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "adopt's alone"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
} {
|
||||
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
|
||||
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
|
||||
// A module's config values may be proposed before the lease write (§13); once the lease is written every change is
|
||||
// the state's alone, and none waits.
|
||||
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
|
||||
t.Errorf("config values before the lease write: %v", f)
|
||||
}
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
t.Errorf("%s is refused: %v", c.Op, f)
|
||||
leased := in(c)
|
||||
leased.Leased = true
|
||||
if _, _, f := Propose(leased); len(f) != 0 {
|
||||
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,10 +92,32 @@ func TestThePlacementSeatDecides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126: the seed places the private network, which no repository holds; it is the mesh's own, and passes.
|
||||
wired := laptop
|
||||
wired.Modules = append(slices.Clone(laptop.Modules), delivery.PlacedModule{Module: "mesh-wireguard"})
|
||||
seed := in(SeedChanges(map[string]delivery.Placement{"laptop": wired})...)
|
||||
seed.Seed, seed.ExportDigest, seed.Current, seed.Revisions = true, "sha256:x", map[string]delivery.Placement{}, map[string]uint64{}
|
||||
seed.Held = func(m string) bool { return m != "mesh-wireguard" } // the indexer holds every module but it
|
||||
if _, _, f := Propose(seed); len(f) != 0 {
|
||||
t.Errorf("a seed placing the private network is refused: %v", f)
|
||||
}
|
||||
|
||||
writes, err := Writes(p, laid)
|
||||
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
|
||||
t.Fatalf("writes = %+v, %v", writes, err)
|
||||
}
|
||||
pinned := *laid["laptop"]
|
||||
pinned.Pins = map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
|
||||
calls := ControllerCalls(map[string]delivery.Placement{"anchor": anchor, "laptop": laptop},
|
||||
map[string]*delivery.Placement{"anchor": laid["anchor"], "laptop": &pinned})
|
||||
var said []string
|
||||
for _, c := range calls {
|
||||
said = append(said, c.Key())
|
||||
}
|
||||
if !slices.Equal(said, []string{"mesh-controller.unassign anchor web", "mesh-controller.assign laptop web", "mesh-controller.pin laptop db"}) {
|
||||
t.Errorf("the controller's calls = %v", said)
|
||||
}
|
||||
|
||||
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
|
||||
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
@@ -95,4 +127,112 @@ func TestThePlacementSeatDecides(t *testing.T) {
|
||||
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
|
||||
t.Error("the ask's yes binds changes it was not asked about")
|
||||
}
|
||||
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor, "laptop": laptop})
|
||||
for name, c := range map[string]struct {
|
||||
raw string
|
||||
want int
|
||||
}{
|
||||
"in output": {`{"ok":true,"output":` + quote(string(export)+"\n") + `}`, 2},
|
||||
"as answer": {`{"ok":true,"output":"","answer":` + string(export) + `}`, 2},
|
||||
"refused": {`{"ok":false,"output":"no store"}`, 0},
|
||||
"no node": {`{"ok":true,"output":"{}"}`, 0},
|
||||
"no version": {`{"ok":true,"output":"{\"anchor\":{\"node\":\"anchor\",\"modules\":[]}}"}`, 0},
|
||||
} {
|
||||
got, err := ExportOf([]byte(c.raw))
|
||||
if len(got) != c.want || (c.want == 0) != (err != nil) {
|
||||
t.Errorf("%s: %d nodes, %v", name, len(got), err)
|
||||
}
|
||||
}
|
||||
if seed := SeedChanges(map[string]delivery.Placement{"laptop": laptop, "anchor": anchor}); len(seed) != 2 || seed[0].Node != "anchor" || seed[0].Op != delivery.OpSeed {
|
||||
t.Errorf("the seed's changes = %+v", seed)
|
||||
}
|
||||
}
|
||||
|
||||
func quote(s string) string {
|
||||
raw, _ := json.Marshal(s)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// Issue 774: every call the assigner makes of the controller carries only the arguments the controller's seat contract
|
||||
// declares for its verb, and every one it requires; the controller refuses any other argument and does nothing. A "why"
|
||||
// added to every call (2026-10-28) had each approved placement refused.
|
||||
func TestControllerCallsFollowTheContract(t *testing.T) {
|
||||
declared := map[string]map[string]any{}
|
||||
required := map[string][]string{}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
declared[v.Name] = props
|
||||
switch r := v.Input["required"].(type) {
|
||||
case []string:
|
||||
required[v.Name] = r
|
||||
}
|
||||
}
|
||||
db := map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
|
||||
moved := map[string]delivery.Provider{"db": {Node: "laptop", Module: "store"}}
|
||||
placed := func(node string, pins map[string]delivery.Provider, modules ...string) delivery.Placement {
|
||||
p := delivery.Placement{Node: node, Pins: pins}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
return p
|
||||
}
|
||||
ptr := func(p delivery.Placement) *delivery.Placement { return &p }
|
||||
for name, c := range map[string]struct {
|
||||
before map[string]delivery.Placement
|
||||
after map[string]*delivery.Placement
|
||||
verbs []string
|
||||
}{
|
||||
"assign": {map[string]delivery.Placement{}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil, "web"))}, []string{"assign"}},
|
||||
"unassign": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil))}, []string{"unassign"}},
|
||||
"key deleted": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": nil}, []string{"unassign", "unpin"}},
|
||||
"pin set": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", db, "web"))}, []string{"pin"}},
|
||||
"pin changed": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", moved, "web"))}, []string{"pin"}},
|
||||
"moved and pinned": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil)), "laptop": ptr(placed("laptop", db, "web"))}, []string{"unassign", "assign", "pin"}},
|
||||
} {
|
||||
calls := ControllerCalls(c.before, c.after)
|
||||
var verbs []string
|
||||
for _, call := range calls {
|
||||
verbs = append(verbs, call.Verb)
|
||||
props, ok := declared[call.Verb]
|
||||
if !ok {
|
||||
t.Errorf("%s: the controller's seat has no verb %q", name, call.Verb)
|
||||
continue
|
||||
}
|
||||
for arg := range call.Args {
|
||||
if _, ok := props[arg]; !ok {
|
||||
t.Errorf("%s: %s gives %q, which mesh-controller.%s does not take, so the controller refuses it", name, call.Key(), arg, call.Verb)
|
||||
}
|
||||
}
|
||||
for _, arg := range required[call.Verb] {
|
||||
if _, ok := call.Args[arg]; !ok {
|
||||
t.Errorf("%s: %s lacks %q, which mesh-controller.%s requires", name, call.Key(), arg, call.Verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !slices.Equal(verbs, c.verbs) {
|
||||
t.Errorf("%s: the calls = %v, want %v", name, verbs, c.verbs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774: a refused call is a condition keyed by its pair, a module's place apart from a pin, naming the verb, the node, the
|
||||
// module and the refusal; its clear says why.
|
||||
func TestCallRefusedCondition(t *testing.T) {
|
||||
assign := ControllerCall{Verb: "assign", Node: "anchor", Module: "web"}
|
||||
pin := ControllerCall{Verb: "pin", Node: "anchor", Module: "web"}
|
||||
c := CallRefusedCondition(assign, `assign takes no argument "why"`, true, "")
|
||||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
!strings.Contains(c.Headline, "assign web on anchor") || !strings.Contains(c.Explanation, `takes no argument "why"`) {
|
||||
t.Errorf("the raised condition = %+v", c)
|
||||
}
|
||||
if CallConditionKey(pin) == c.Key {
|
||||
t.Error("a pin's condition shares the key of a module's place")
|
||||
}
|
||||
if cl := CallRefusedCondition(assign, "", false, "answered"); cl.State != asks.StateCleared || cl.Key != c.Key || cl.Resolved != "answered" {
|
||||
t.Errorf("the clear = %+v", cl)
|
||||
}
|
||||
if _, _, raised, err := FindingOf(c, time.Unix(0, 0)); err != nil || !raised {
|
||||
t.Errorf("the condition is no finding: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,8 +39,8 @@ type Module struct {
|
||||
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
||||
Chosen *catalogue.PersonPolicy
|
||||
Nodes []string
|
||||
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
|
||||
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
@@ -131,7 +131,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
}
|
||||
if words, waiting := m.provisionWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
@@ -412,8 +412,7 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
// A placed node's want names the build alone: it is the same target while the build is.
|
||||
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
|
||||
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
@@ -535,7 +534,7 @@ func (m Module) selfDerived(node string, before delivery.Want, had bool, next de
|
||||
d, there := m.composition(node, next.FP)
|
||||
if !there || d.Digest == "" {
|
||||
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
||||
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
|
||||
"and the controller has not composed it at %s, so its want stays", m.Name, node, short(next.FP))}
|
||||
return next, false
|
||||
}
|
||||
at := m
|
||||
|
||||
@@ -815,13 +815,6 @@ func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
||||
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer was erased: %+v", w)
|
||||
}
|
||||
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
|
||||
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
|
||||
m.Self = func(string) bool { return true }
|
||||
placed := map[string]delivery.Want{"ace": {FP: fNew, Self: true, Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
||||
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer in a placed node's want was erased: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved",
|
||||
want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.composed", "mesh-controller.lease",
|
||||
"mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
@@ -82,7 +82,8 @@ func TestManifest(t *testing.T) {
|
||||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
|
||||
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
|
||||
}
|
||||
if want := []string{
|
||||
if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin",
|
||||
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
|
||||
"seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
|
||||
@@ -30,6 +30,9 @@
|
||||
"mesh-conditions.takeover",
|
||||
"mesh-config.kept",
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.lease",
|
||||
"mesh-delivery-builder.build-log",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
@@ -46,6 +49,11 @@
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.assign",
|
||||
"seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export",
|
||||
"seat:mesh-controller.unassign",
|
||||
"seat:mesh-controller.unpin",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-hand-acts.hand-acts",
|
||||
"seat:mesh-hand-acts.warranted",
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -20,7 +19,7 @@ import (
|
||||
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
|
||||
// controller had never taken in, and the builder kept them done.
|
||||
|
||||
// heldVerb answers the seat mesh-artifact-sweep's verb held on the bus, as its holder does: {result: {output, ok,
|
||||
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
|
||||
// answer}}; a reference is gone while gone says so.
|
||||
type heldVerb struct {
|
||||
mu sync.Mutex
|
||||
@@ -30,7 +29,7 @@ type heldVerb struct {
|
||||
|
||||
func (h *heldVerb) serve(t *testing.T, b *bed) {
|
||||
t.Helper()
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat."+catalogue.ArtifactSweepSeat+".call.held.>", func(m *nats.Msg) {
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
|
||||
var args struct {
|
||||
References []string `json:"references"`
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -178,42 +177,6 @@ func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the
|
||||
// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined
|
||||
// ask again, so a lone node off the agreed toolchain cannot spin on it.
|
||||
const (
|
||||
DefaultDeclineBound = 30 * time.Minute
|
||||
DefaultDeclinePause = 5 * time.Second
|
||||
)
|
||||
|
||||
func (b *Builder) declineBound() time.Duration {
|
||||
if b.DeclineBound > 0 {
|
||||
return b.DeclineBound
|
||||
}
|
||||
return DefaultDeclineBound
|
||||
}
|
||||
|
||||
func (b *Builder) declinePause() time.Duration {
|
||||
if b.DeclinePause != 0 {
|
||||
return max(b.DeclinePause, 0)
|
||||
}
|
||||
return DefaultDeclinePause
|
||||
}
|
||||
|
||||
// declinedWords names the nodes that declined an ask and why, in the order of their names.
|
||||
func declinedWords(by map[string]string) string {
|
||||
nodes := make([]string, 0, len(by))
|
||||
for n := range by {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var parts []string
|
||||
for _, n := range nodes {
|
||||
parts = append(parts, n+": "+by[n])
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
@@ -292,8 +255,6 @@ func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
// errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2).
|
||||
errDeclined = errors.New("declined")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
@@ -307,42 +268,11 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
var firstDecline time.Time
|
||||
declines := map[string]string{} // node -> why it declined, the last
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errDeclined):
|
||||
// Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once.
|
||||
// Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the
|
||||
// checker's decline loop is, issue 890).
|
||||
now := time.Now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
declines[step.Node] = why
|
||||
if bound := b.declineBound(); now.Sub(firstDecline) > bound {
|
||||
return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines))
|
||||
}
|
||||
// The pause is a context's bound, not a timer (ADR 0437 §8).
|
||||
pause, stop := context.WithTimeout(ctx, b.declinePause())
|
||||
<-pause.Done()
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return step, ask, ctx.Err()
|
||||
}
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why)
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
@@ -387,11 +317,6 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if _, declined := delivery.DeclinedOf(step); err == nil && declined {
|
||||
// A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again
|
||||
// after a pause, within DeclineBound.
|
||||
return step, step.Reason, errDeclined
|
||||
}
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
|
||||
@@ -67,9 +67,6 @@ type Builder struct {
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
|
||||
// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
|
||||
DeclineBound, DeclinePause time.Duration
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
|
||||
@@ -418,7 +415,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
if planErr == nil && !nothing {
|
||||
if why := b.stale(m, plan); why != "" {
|
||||
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
|
||||
// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
|
||||
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
|
||||
b.logf("%s at %s is not built: %s", module, m.FP2, why)
|
||||
return
|
||||
}
|
||||
@@ -509,11 +506,10 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
|
||||
continue // an archive is packed with no toolchain
|
||||
}
|
||||
language := buildrun.LanguageOf[bm]
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
want, ok := m.Toolchains[language]
|
||||
if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
|
||||
@@ -547,8 +543,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
repository := delivery.RepositoryName(m.Repository)
|
||||
var lanes []*lane
|
||||
used := map[string]delivery.ToolchainUsed{}
|
||||
var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
for _, bm := range plan.Modules() {
|
||||
if prior != nil && len(only) > 0 && !only[bm] {
|
||||
kept = append(kept, bm)
|
||||
@@ -558,12 +553,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
|
||||
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
|
||||
language := buildrun.LanguageOf[bm]
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, none)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
sitting = append(sitting, SittingOutWords(language, ag)...)
|
||||
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
||||
}
|
||||
@@ -660,9 +653,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
}
|
||||
}
|
||||
step := "built " + strings.Join(steps, "; ")
|
||||
if len(sitting) > 0 {
|
||||
step += " (" + strings.Join(sitting, "; ") + ")"
|
||||
}
|
||||
if len(used) == 0 {
|
||||
used = nil
|
||||
}
|
||||
@@ -952,13 +942,3 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
|
||||
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
|
||||
return e
|
||||
}
|
||||
|
||||
// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
|
||||
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
|
||||
func SittingOutWords(language string, a delivery.Agreement) []string {
|
||||
var out []string
|
||||
for _, s := range a.SittingOut {
|
||||
out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -223,14 +223,6 @@ var (
|
||||
|
||||
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
|
||||
r := newRig(t, goBundle, archiveA, tsBundle)
|
||||
// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
|
||||
r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false)
|
||||
defer func() {
|
||||
if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") {
|
||||
t.Errorf("the key does not say ace sits out: %s", st)
|
||||
}
|
||||
}()
|
||||
var logged []delivery.BuildLogEntry
|
||||
var lmu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
|
||||
@@ -738,25 +730,6 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
})
|
||||
t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.DeclinePause, r.b.DeclineBound = -1, 150*time.Millisecond
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}})
|
||||
waitForAsks(t, r, 1)
|
||||
for ended, deadline := 0, time.Now().Add(5*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn
|
||||
r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined})
|
||||
ended++
|
||||
}
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") {
|
||||
t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings)
|
||||
}
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
askings int
|
||||
|
||||
-1
@@ -36,7 +36,6 @@ func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerd
|
||||
// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not
|
||||
// "not yet run": a pass names its machine, a fail fails the check with the assigner's words.
|
||||
func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) {
|
||||
t.Run("the read awaits the assigner's composition", checkerAwaitsTheAssignersComposition)
|
||||
for _, c := range []struct {
|
||||
state, owed, words string
|
||||
}{
|
||||
|
||||
+1
-2
@@ -21,8 +21,7 @@ import (
|
||||
|
||||
// 1297/AC3 (its bus half): on the real bus, the checker's read of the assigner's composition waits while the key is
|
||||
// absent and while the assigner says it waits, and answers the verdict the assigner then writes under the check's key.
|
||||
// A case of TestTheCheckersLineSaysTheAssignersComposition (merged into it for the test budget, issue 774).
|
||||
func checkerAwaitsTheAssignersComposition(t *testing.T) {
|
||||
func TestTheCheckerAwaitsTheAssignersComposition(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "comp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -342,11 +342,6 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
|
||||
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
|
||||
}
|
||||
for _, a := range asks { // ADR 0584: every check ask names the agreed toolchain, so a node off it declines
|
||||
if a.Check.Toolchain == nil || a.Check.Toolchain.Version != "go1.27.1" || a.Check.Toolchain.Recipe != "7" {
|
||||
t.Errorf("a check ask names toolchain %+v, want go1.27.1 recipe 7", a.Check.Toolchain)
|
||||
}
|
||||
}
|
||||
for _, a := range asks[1:] {
|
||||
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
|
||||
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
|
||||
|
||||
@@ -166,9 +166,7 @@ type target struct {
|
||||
module string
|
||||
node string
|
||||
toolchain string
|
||||
// agreed is the toolchain the ask names, so a node off it declines the check (ADR 0584); nil when none is agreed.
|
||||
agreed *delivery.ToolchainUsed
|
||||
fetched Fetched
|
||||
fetched Fetched
|
||||
// fetchArgs are the arguments of the fetch its check ask runs.
|
||||
fetchArgs map[string]any
|
||||
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
|
||||
@@ -376,7 +374,6 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
t := &target{pull: p, module: bm}
|
||||
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
||||
t.toolchain = tc.Version
|
||||
t.agreed = &delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
|
||||
}
|
||||
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
||||
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
|
||||
@@ -431,7 +428,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
|
||||
p := t.pull
|
||||
placed := len(needs) > 0
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages, Toolchain: t.agreed}
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
||||
if st.j.Nodes != nil {
|
||||
names, err := st.j.Nodes.Names(ctx)
|
||||
if err == nil && len(names) == 0 {
|
||||
@@ -498,12 +495,8 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
bound = DefaultDeclineBound
|
||||
}
|
||||
if now.Sub(firstDecline) > bound {
|
||||
lacks := strings.Join(needs, ", ")
|
||||
if lacks == "" {
|
||||
lacks = strings.Join(d.Lacks, ", ") // a node off the agreed toolchain declined it (ADR 0584)
|
||||
}
|
||||
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
||||
p.Repository, p.Number, lacks, bound, step.Node))
|
||||
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
|
||||
}
|
||||
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
||||
a.ID, a.At = buildstep.NewID(), now.UTC()
|
||||
|
||||
+1
-3
@@ -423,9 +423,7 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
|
||||
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}},
|
||||
// The mesh packages its go.sum names (ADR 0573 §2, issue 1585): this module's own go.sum names mesh-sdk.
|
||||
Packages: map[string][]string{"modules/a": {"git.novox.be/novox/mesh-sdk/go"}}}
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0 h1:q6ywchpTM9fwaJ1UNwDSuOjb5FYDFcsaeiTrKoB4scc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a h1:AChSJUmHyOZ5smiuTrtDZNjetYNTFb+anRFjXIQHbSA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101143343-67354f8e019a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -50,37 +50,23 @@ const Env = "MESH_CHECK_BESIDE"
|
||||
// CapturedSuffix is what a captured file's name carries that the repository's own does not.
|
||||
const CapturedSuffix = ".captured"
|
||||
|
||||
// notPlaced are the repositories the checker places beside no mesh-delivery check (besideRules in
|
||||
// mesh-delivery-checker/internal/check/beside.go places mesh-sdk and the catalogue): a test reads them from the captured
|
||||
// copy even in a merge check, and says so (issue 1585). Every other sibling missing from a merge check fails the test.
|
||||
var notPlaced = map[string]bool{"mesh-host": true}
|
||||
|
||||
// Dir is the named repository a test reads — the clone beside a merge check, or the captured copy — and
|
||||
// says which in the test's log.
|
||||
func Dir(t testing.TB, repository string) string {
|
||||
t.Helper()
|
||||
root := os.Getenv(Env)
|
||||
if root != "" {
|
||||
if root := os.Getenv(Env); root != "" {
|
||||
dir := filepath.Join(root, repository)
|
||||
_, err := os.Stat(dir)
|
||||
switch {
|
||||
case err == nil:
|
||||
t.Logf("judged against %s as cloned beside this check", dir)
|
||||
return dir
|
||||
case !notPlaced[repository]:
|
||||
// A sibling the checker places that is missing is the checker's fault, never judged against a stale copy.
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
t.Fatalf("%s is not beside this check in %s=%s, so its agreement with this repository cannot be "+
|
||||
"judged here: %v", repository, Env, root, err)
|
||||
}
|
||||
t.Logf("judged against %s as cloned beside this check", dir)
|
||||
return dir
|
||||
}
|
||||
from := filepath.Join(Captured(), repository)
|
||||
if _, err := os.Stat(from); err != nil {
|
||||
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
|
||||
}
|
||||
if root != "" {
|
||||
t.Logf("%s is not beside this check in %s=%s (the checker places it beside no mesh-delivery check): read from "+
|
||||
"its captured copy", repository, Env, root)
|
||||
}
|
||||
dir, err := layOut(repository, from)
|
||||
if err != nil {
|
||||
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
|
||||
|
||||
@@ -5013,10 +5013,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
// Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1);
|
||||
// fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts
|
||||
// (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2);
|
||||
// fifty-seven with mesh-grants (ADR 0582 clause 1).
|
||||
if len(catalogue.Seats()) != 57 {
|
||||
t.Errorf("the mesh defines %d seats rather than 57; the set is closed, so a change here is "+
|
||||
// (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2).
|
||||
if len(catalogue.Seats()) != 56 {
|
||||
t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(catalogue.Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
|
||||
+6
-8
@@ -7,22 +7,20 @@ The copy carries the catalogue's private details: domains, first names in module
|
||||
private addresses. That is acceptable while mesh-sdk is private; if it ever goes public, this copy is
|
||||
a second place to clean besides the catalogue itself.
|
||||
|
||||
Only what go/catalogue's tests read is captured (R4b, issue 1379), copied here from mesh-sdk with the judge's tests
|
||||
(issue 1585). mesh-host refreshed 2026-11-01 from its main, the catalogue 2026-10-21 from its main; before that it
|
||||
was mesh-controller's copy, which still held two modules that had left the catalogue.
|
||||
Only what go/catalogue's tests read is captured (R4b, issue 1379). Refreshed 2026-10-21 from the mains of both
|
||||
repositories; before that it was mesh-controller's copy, which still held two modules that had left the catalogue.
|
||||
|
||||
mesh-catalog b5dbfbb784507e211372a7c9640d91f7d6dadd84 modules/*/module.json, modules/*/files/**, modules/*/templates/**, predefined.json
|
||||
mesh-host 233bb23bbcb1a829ce2a6164a31a5c5cc4f3d950 internal/apply/placement_guard.go
|
||||
mesh-host cdcba9ca73755854d21e4e657d7e46c32b9b5265 internal/apply/placement_guard.go
|
||||
|
||||
To move them, from mesh-delivery's modules/mesh-delivery-indexer directory, with the two repositories checked out
|
||||
beside mesh-delivery:
|
||||
To move them, from this repository's go/ directory, with the two repositories checked out beside mesh-sdk:
|
||||
|
||||
rm -rf testdata/beside/mesh-catalog testdata/beside/mesh-host
|
||||
mkdir -p testdata/beside/mesh-catalog testdata/beside/mesh-host/internal/apply
|
||||
git -C ../../../mesh-catalog archive <commit> -- predefined.json ':(glob)modules/*/module.json' \
|
||||
git -C ../../mesh-catalog archive <commit> -- predefined.json ':(glob)modules/*/module.json' \
|
||||
':(glob)modules/*/files/**' ':(glob)modules/*/templates/**' | tar -x -C testdata/beside/mesh-catalog
|
||||
find testdata/beside -name module.json -exec mv {} {}.captured \;
|
||||
git -C ../../../mesh-host show <commit>:internal/apply/placement_guard.go \
|
||||
git -C ../../mesh-host show <commit>:internal/apply/placement_guard.go \
|
||||
> testdata/beside/mesh-host/internal/apply/placement_guard.go.captured
|
||||
|
||||
and write the commits here. The placement guard is kept as .captured so no Go tool reads it as this
|
||||
|
||||
Vendored
+4
-99
@@ -56,7 +56,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// protectedRoots are paths no directory, access or mount source may be, nor hold.
|
||||
@@ -384,9 +384,6 @@ func ownedByAccount(owner string, a homeAccount) bool {
|
||||
if owner == a.Name {
|
||||
return true
|
||||
}
|
||||
if a.UID < 0 {
|
||||
return false // an account the declaration makes has no uid yet: by name alone, never by a number
|
||||
}
|
||||
user, _, _ := strings.Cut(owner, ":")
|
||||
if uid, err := strconv.Atoi(user); err == nil {
|
||||
return uid == a.UID
|
||||
@@ -416,91 +413,12 @@ const (
|
||||
// refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module,
|
||||
// owner a directory's declared owner.
|
||||
func refusePath(path string, kind placing, module, owner string) error {
|
||||
return refusePathWith(path, kind, module, owner, nil)
|
||||
}
|
||||
|
||||
// declaredHomes is the home of each account the declaration itself makes (a user resource with a home), by the
|
||||
// account's name; the account is not in the user database yet when the declaration is judged, and the engine makes
|
||||
// it where it is declared, before the directory that is owned by it is reached (novox/hq issue 1542). Its uid is
|
||||
// not known, so a directory is owned by it by name alone. Only for the directory placement guard; no other
|
||||
// reader of accountsOfHomes sees these.
|
||||
func declaredHomes(resources []declaration.Resource) map[string]homeAccount {
|
||||
var accounts []passwdEntry
|
||||
var out map[string]homeAccount
|
||||
for _, r := range resources {
|
||||
u, ok := r.(*declaration.User)
|
||||
if !ok || u.Name == "" {
|
||||
continue
|
||||
}
|
||||
home := filepath.Clean(u.Home)
|
||||
if u.Home == "" || !filepath.IsAbs(home) || home == "/" {
|
||||
continue
|
||||
}
|
||||
if accounts == nil {
|
||||
accounts = passwdEntries()
|
||||
}
|
||||
// Only ever adds a home. An account the machine has keeps the home it has (applyUser never moves it), so
|
||||
// a declared home of one is not its home, unless it is the very home it has: the account an earlier apply
|
||||
// made, as declared, which must be judged the same at every later apply. And a home at or below another
|
||||
// account's, or at or above a protected root, is not a new one.
|
||||
taken := false
|
||||
for _, e := range accounts {
|
||||
if e.name == u.Name && e.home != home {
|
||||
taken = true
|
||||
}
|
||||
if e.name != u.Name && e.home != "" && atOrBelow(home, e.home) {
|
||||
taken = true
|
||||
}
|
||||
}
|
||||
for _, root := range protectedRoots {
|
||||
taken = taken || home == root || below(root, home)
|
||||
}
|
||||
if taken {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]homeAccount{}
|
||||
}
|
||||
out[home] = homeAccount{Name: u.Name, UID: -1}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// passwdEntry is an account of the user database and its home, whatever its uid.
|
||||
type passwdEntry struct{ name, home string }
|
||||
|
||||
// passwdEntries is every account of the user database with its home (empty and root homes as "").
|
||||
func passwdEntries() []passwdEntry {
|
||||
out := []passwdEntry{}
|
||||
f, err := os.Open(passwdFile)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Split(sc.Text(), ":")
|
||||
if len(fields) < 6 {
|
||||
continue
|
||||
}
|
||||
home := filepath.Clean(fields[5])
|
||||
if home == "." || home == "/" || home == "/nonexistent" {
|
||||
home = ""
|
||||
}
|
||||
out = append(out, passwdEntry{fields[0], home})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// refusePathWith is refusePath with the homes of the accounts the declaration makes (declaredHomes) counted
|
||||
// among the machine's.
|
||||
func refusePathWith(path string, kind placing, module, owner string, declared map[string]homeAccount) error {
|
||||
clean := filepath.Clean(path)
|
||||
if !filepath.IsAbs(clean) {
|
||||
return nil // the declaration refuses a relative path already; a named volume is not a path
|
||||
}
|
||||
for _, p := range []string{clean, resolved(clean)} {
|
||||
if err := refuseOne(p, kind, module, owner, declared); err != nil {
|
||||
if err := refuseOne(p, kind, module, owner); err != nil {
|
||||
if p != clean {
|
||||
err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why)
|
||||
err.Path = clean
|
||||
@@ -511,7 +429,7 @@ func refusePathWith(path string, kind placing, module, owner string, declared ma
|
||||
return nil
|
||||
}
|
||||
|
||||
func refuseOne(path string, kind placing, module, owner string, declared map[string]homeAccount) *PlacementRefusedError {
|
||||
func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError {
|
||||
for _, root := range protectedRoots {
|
||||
if path == root || below(root, path) {
|
||||
return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " +
|
||||
@@ -564,18 +482,6 @@ func refuseOne(path string, kind placing, module, owner string, declared map[str
|
||||
return nil
|
||||
}
|
||||
homes := accountsOfHomes()
|
||||
if ssh && len(declared) > 0 {
|
||||
merged := map[string]homeAccount{}
|
||||
for home, a := range homes {
|
||||
merged[home] = a
|
||||
}
|
||||
for home, a := range declared {
|
||||
if _, held := merged[home]; !held {
|
||||
merged[home] = a
|
||||
}
|
||||
}
|
||||
homes = merged
|
||||
}
|
||||
var deepest string
|
||||
for home := range homes {
|
||||
if below(path, home) && len(home) > len(deepest) {
|
||||
@@ -612,12 +518,11 @@ func moduleOfID(id string) string {
|
||||
// a file into it, because nothing judged the file's path.
|
||||
func refusedPlaces(resources []declaration.Resource) map[string]error {
|
||||
out := map[string]error{}
|
||||
declared := declaredHomes(resources)
|
||||
for _, r := range resources {
|
||||
var err error
|
||||
switch res := r.(type) {
|
||||
case *declaration.Directory:
|
||||
err = refusePathWith(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner, declared)
|
||||
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
|
||||
// And outside every home, the walk's judgement before anything is applied (novox/hq issue 358): a
|
||||
// link another account could make in its path refuses it, and with it what is below it, whatever
|
||||
// order they are declared in. The apply walks again at the moment of use. A directory handed over
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
|
||||
"mesh-delivery-watchman/internal/deadletters"
|
||||
@@ -39,13 +38,6 @@ const (
|
||||
routeDead = "dead-letter"
|
||||
routeBusFault = "bus-fault"
|
||||
routeProv = "provisioned"
|
||||
routeLease = "lease"
|
||||
)
|
||||
|
||||
// The lease write's subjects (ADR 0582 clauses 4, 6): the seat mesh-grants' events, and ASSIGNMENTS deleted.
|
||||
var (
|
||||
leaseEventPrefix = "mesh.seat." + catalogue.GrantsSeat + ".event."
|
||||
assignmentsGone = "$JS.EVENT.ADVISORY.STREAM.DELETED." + buscompose.AssignmentsStream
|
||||
)
|
||||
|
||||
// The bus module's events of the connections the bus closed (S9).
|
||||
@@ -69,8 +61,6 @@ func routeOf(subject string) string {
|
||||
return routeTools
|
||||
case subject == CheckedSubject:
|
||||
return routeChecked
|
||||
case strings.HasPrefix(subject, leaseEventPrefix) || subject == assignmentsGone:
|
||||
return routeLease
|
||||
case strings.HasPrefix(subject, "$JS.EVENT.ADVISORY."):
|
||||
return routeAdvisory
|
||||
case strings.HasPrefix(subject, "mesh.events.dead."):
|
||||
@@ -106,7 +96,6 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
|
||||
}
|
||||
if dead.isTheControllersNotices(a.ID) {
|
||||
dead.check() // the lease write's step removed it: the watchman may take the notices now
|
||||
w.RemovalGone(watch.RemovalConsumer)
|
||||
return
|
||||
}
|
||||
w.Advised(a)
|
||||
@@ -121,22 +110,6 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
|
||||
kind = watch.KindConnectionClosed
|
||||
}
|
||||
w.Closed(kind, c)
|
||||
case routeLease:
|
||||
var body struct {
|
||||
Altered string `json:"altered"`
|
||||
Failed string `json:"failed"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &body)
|
||||
switch m.Subject {
|
||||
case assignmentsGone:
|
||||
w.RemovalGone(watch.RemovalStream)
|
||||
case leaseEventPrefix + catalogue.GrantsEventLeaseAltered:
|
||||
w.LeaseAltered(body.Altered)
|
||||
case leaseEventPrefix + catalogue.GrantsEventLeaseHanded:
|
||||
w.LeaseHanded()
|
||||
case leaseEventPrefix + catalogue.GrantsEventRemovalFailed:
|
||||
w.RemovalFailed(body.Failed)
|
||||
}
|
||||
case routeProv:
|
||||
var r watch.ProvReport
|
||||
if err := json.Unmarshal(m.Data, &r); err != nil {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -59,23 +59,6 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
}
|
||||
w.ModuleVerdict("node-tools", value(""), Put)
|
||||
}, map[string]string{}},
|
||||
{"774/S22/AC1, AC2: lease-altered and removal-failed are findings; lease-handed and the removal read gone clear them", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.LeaseAltered("the key holder names controller@a pid 9, not mesh-grants by lease-write-1")
|
||||
if f := o.findings[FindingKey(ScopeBus, "mesh-controller_lease", KindLeaseAltered)]; !strings.Contains(f.Summary, "controller@a pid 9") {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
w.LeaseHanded()
|
||||
w.RemovalFailed("the stream ASSIGNMENTS: ASSIGNMENTS has 1 consumers")
|
||||
k := FindingKey(ScopeBus, "lease-removal", KindRemovalFailed)
|
||||
if f := o.findings[k]; !strings.Contains(f.Summary, "ASSIGNMENTS has 1 consumers") {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
w.RemovalGone(RemovalConsumer) // not what it waits on
|
||||
if _, ok := o.findings[k]; !ok {
|
||||
t.Fatal("cleared by the consumer, while the stream it names stands")
|
||||
}
|
||||
w.RemovalGone(RemovalStream)
|
||||
}, map[string]string{}},
|
||||
{"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.Step("go", "n1.ask1", runningStep("ask1", t0), Put)
|
||||
expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second)
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// **The lease write's findings** (novox/hq ADR 0582 clauses 4 and 6, issue 774): mesh-grants says lease-altered when the
|
||||
// lease key or its bucket's expiry differs from what it wrote, and removal-failed when the day-later removal of the
|
||||
// controller's consumer of DEAD_LETTER_NOTICES or the stream ASSIGNMENTS failed; the watchman raises each as a finding.
|
||||
// lease-altered clears when lease-handed is said again (the operator asked the write again); removal-failed clears when
|
||||
// what it names reads gone: the consumer's deletion advisory and ASSIGNMENTS' deletion advisory, each heard.
|
||||
|
||||
// The two kinds.
|
||||
const (
|
||||
KindLeaseAltered = "lease-altered"
|
||||
KindRemovalFailed = "removal-failed"
|
||||
)
|
||||
|
||||
// The finding's IDs: the lease's bucket, and the removal.
|
||||
const (
|
||||
leaseID = "mesh-controller_lease"
|
||||
removalID = "lease-removal"
|
||||
)
|
||||
|
||||
// What a failed removal still waits on.
|
||||
const (
|
||||
RemovalConsumer = "DEAD_LETTER_NOTICES.controller"
|
||||
RemovalStream = "ASSIGNMENTS"
|
||||
)
|
||||
|
||||
// LeaseAltered raises the finding that the lease differs from what mesh-grants wrote, naming what changed.
|
||||
func (w *Watchman) LeaseAltered(what string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.find(Finding{Scope: ScopeBus, ID: leaseID, Kind: KindLeaseAltered, Severity: Warning, Source: "mesh-grants",
|
||||
Summary: "the controller's lease differs from what mesh-grants wrote: " + what,
|
||||
Said: what,
|
||||
Needs: "a look at who changed the lease; mesh-grants writes nothing until lease-write is asked again.",
|
||||
Resolved: "lease-write is asked and warranted again, and the lease reads as written"})
|
||||
}
|
||||
|
||||
// LeaseHanded clears the lease-altered finding: the lease reads as written again.
|
||||
func (w *Watchman) LeaseHanded() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.unfind(FindingKey(ScopeBus, leaseID, KindLeaseAltered))
|
||||
}
|
||||
|
||||
// RemovalFailed raises the finding that the day-later removal failed, naming what failed; it waits on each of the
|
||||
// consumer and the stream the failure names.
|
||||
func (w *Watchman) RemovalFailed(failed string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.removalWaits == nil {
|
||||
w.removalWaits = map[string]bool{}
|
||||
}
|
||||
for _, what := range []string{RemovalConsumer, RemovalStream} {
|
||||
if strings.Contains(failed, strings.SplitN(what, ".", 2)[0]) {
|
||||
w.removalWaits[what] = true
|
||||
}
|
||||
}
|
||||
if len(w.removalWaits) == 0 { // names neither: waits on both
|
||||
w.removalWaits[RemovalConsumer], w.removalWaits[RemovalStream] = true, true
|
||||
}
|
||||
w.find(Finding{Scope: ScopeBus, ID: removalID, Kind: KindRemovalFailed, Severity: Warning, Source: "mesh-grants",
|
||||
Summary: "the day-later removal of the controller's consumer and stream failed: " + failed,
|
||||
Said: failed,
|
||||
Needs: "a look at why; mesh-grants tries again in a day, and grants_removal_due starts it sooner.",
|
||||
Resolved: "the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS read gone"})
|
||||
}
|
||||
|
||||
// RemovalGone hears one of the two removed (RemovalConsumer or RemovalStream), and clears the finding once neither waits.
|
||||
func (w *Watchman) RemovalGone(what string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
delete(w.removalWaits, what)
|
||||
if len(w.removalWaits) == 0 {
|
||||
w.unfind(FindingKey(ScopeBus, removalID, KindRemovalFailed))
|
||||
}
|
||||
}
|
||||
@@ -86,8 +86,6 @@ type Watchman struct {
|
||||
recount func()
|
||||
// waiting is, by `<consumer>.<node>`, each of its provisions in waits, with its words.
|
||||
waiting map[string]map[string]string
|
||||
// removalWaits is what a failed removal of the lease write still waits on to read gone (lease.go).
|
||||
removalWaits map[string]bool
|
||||
// health is each node's own health report, and tunnels each node's tunnel key from its record.
|
||||
health map[string]delivery.NodeHealth
|
||||
tunnels map[string]string
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The three bus adapters for mesh-sdk go/asker on the part's own credential, as mesh-delivery-teller has them (copied: the
|
||||
@@ -82,16 +81,3 @@ func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c a
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// seatSaying says the seat mesh-grants' events on its seat subject into EVENTS, as the module that holds it (novox/hq
|
||||
// ADR 0582 clauses 3, 4, 6): lease-handed, lease-altered and removal-failed.
|
||||
func seatSaying(ctx context.Context, js jetstream.JetStream) func(event string, body any) error {
|
||||
return func(event string, body any) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(ctx, "mesh.seat."+catalogue.GrantsSeat+".event."+event, data)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,297 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
|
||||
// standIns are the seats and tools the lease write asks, standing in on the real bus's side: node-service-manager on
|
||||
// the controller's node (a unit's ActiveState, as the systemd module answers it), the bus module's expiry tool acting on
|
||||
// the real lease bucket's stream (as nats-tools' SetBucketExpiry does), and the checks answering green.
|
||||
type standIns struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
mu sync.Mutex
|
||||
active string
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (s *standIns) Call(key string, body any) (json.RawMessage, error) {
|
||||
s.mu.Lock()
|
||||
s.calls = append(s.calls, key)
|
||||
s.mu.Unlock()
|
||||
base, _, _ := strings.Cut(key, "@")
|
||||
args, _ := body.(map[string]any)
|
||||
var out any
|
||||
switch base {
|
||||
case grants.CallServiceStop:
|
||||
s.active, out = "inactive", map[string]any{"ok": true}
|
||||
case grants.CallServiceStart:
|
||||
s.active, out = "active", map[string]any{"ok": true}
|
||||
case grants.CallServiceStatus:
|
||||
out = map[string]any{"ActiveState": s.active}
|
||||
case grants.CallBucketExpiry:
|
||||
now := fmt.Sprint(args["max_age"])
|
||||
d, err := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st, err := s.js.Stream(s.ctx, "KV_"+fmt.Sprint(args["bucket"]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg := st.CachedInfo().Config
|
||||
cfg.MaxAge = d
|
||||
if d > 0 && (cfg.Duplicates == 0 || cfg.Duplicates > d) {
|
||||
cfg.Duplicates = d
|
||||
}
|
||||
if _, err := s.js.UpdateStream(s.ctx, cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = map[string]any{"bucket": args["bucket"], "now": now, "changed": true}
|
||||
case grants.CallIssue:
|
||||
out = map[string]any{"status": "resolved"}
|
||||
case grants.CallNodeTrusts:
|
||||
out = map[string]any{"every": true, "not_yet": []string{}}
|
||||
case grants.CallConfigCompare:
|
||||
out = map[string]any{"source": "indexer", "count": 0}
|
||||
case grants.CallUpgrade:
|
||||
out = map[string]any{"empty": true, "splits": []any{}}
|
||||
case grants.CallVaultStatus:
|
||||
out = map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}
|
||||
case grants.CallStreams:
|
||||
out = map[string]any{"detail": []any{}}
|
||||
case grants.CallUsers:
|
||||
out = map[string]any{"users": []any{}}
|
||||
case grants.CallRecordWarrant, grants.CallRecordHandAct:
|
||||
out = map[string]any{}
|
||||
default:
|
||||
return nil, fmt.Errorf("no stand-in for %s", key)
|
||||
}
|
||||
return json.Marshal(out)
|
||||
}
|
||||
|
||||
// memStore is the part's own state, in memory, with its time to live per key recorded.
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
}
|
||||
|
||||
func (m *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
v, ok := m.vals[state+"/"+key]
|
||||
return v, m.revs[state+"/"+key], ok, nil
|
||||
}
|
||||
|
||||
func (m *memStore) Put(state, key string, value []byte) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.vals[state+"/"+key] = append([]byte(nil), value...)
|
||||
m.revs[state+"/"+key]++
|
||||
return m.revs[state+"/"+key], nil
|
||||
}
|
||||
|
||||
func (m *memStore) PutTTL(state, key string, value []byte, _ time.Duration) (uint64, error) {
|
||||
return m.Put(state, key, value)
|
||||
}
|
||||
|
||||
func (m *memStore) Keys(state string) ([]string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []string
|
||||
for k := range m.vals {
|
||||
if s, key, _ := strings.Cut(k, "/"); s == state {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (m *memStore) Delete(state, key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.vals, state+"/"+key)
|
||||
return nil
|
||||
}
|
||||
|
||||
type memObjects struct{}
|
||||
|
||||
func (memObjects) PutObject(string, []byte) error { return nil }
|
||||
func (memObjects) DeleteObject(string) error { return nil }
|
||||
|
||||
// yesAsker answers every ask the part raises with the operator's yes, as the router's warrant hands it.
|
||||
type yesAsker struct{ asked []string }
|
||||
|
||||
func (a *yesAsker) Ask(q asks.Ask) error { a.asked = append(a.asked, q.ID); return nil }
|
||||
func (a *yesAsker) Where(string) (string, *asks.Warrant, error) { return "open", nil, nil }
|
||||
func (a *yesAsker) Cancel(string) error { return nil }
|
||||
func (a *yesAsker) Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error) {
|
||||
return asker.Acted, act(asks.Option{ID: w.Option}, w)
|
||||
}
|
||||
|
||||
// Issue 774, 774/S23 (novox/hq ADR 0582 clauses 3, 5, "How it is checked"): mesh-grants' lease write and hand-back
|
||||
// against the real bus, with a stand-in controller holding the lease in a bucket with an expiry (2 s standing in for 15
|
||||
// s, so the test waits past it in seconds), and stand-in node-service-manager and bus tool seats.
|
||||
func TestTheLeaseWriteOnTheBus(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "lease")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const ttl = 2 * time.Second
|
||||
was := grants.LeaseExpiry
|
||||
grants.LeaseExpiry = ttl
|
||||
t.Cleanup(func() { grants.LeaseExpiry = was })
|
||||
lease, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buscompose.LeaseBucket, History: 1, TTL: ttl})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
controller, _ := json.Marshal(buscompose.LeaseHolder{Instance: "controller@a pid 1", Host: "a", Epoch: 60})
|
||||
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket := func(state string) jetstream.KeyValue {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state), History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
put := func(kv jetstream.KeyValue, key string, v any) {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
nats := `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},` +
|
||||
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`
|
||||
digest := delivery.ManifestDigest([]byte(nats))
|
||||
put(bucket(grants.ReadManifests), digest, delivery.Manifest{V: delivery.Version, Module: "nats", JSON: nats})
|
||||
put(bucket(grants.ReadModules), "nats", delivery.Module{V: delivery.Version, FP2: "src2:" + strings.Repeat("ab", 32), Manifest: digest,
|
||||
Repository: "novox/mesh-catalog", Commit: "c"})
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}}})
|
||||
if _, err := bucket(grants.ReadPlacement).Put(ctx, "placement.a", pl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(bucket(grants.ReadTakeover), health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "taken"})
|
||||
put(bucket(grants.ReadAccounts), buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1, "at": "2026-11-01T00:00:00Z"})
|
||||
busKey, _ := seal.GenerateKeyPair()
|
||||
have := bucket(grants.ReadHave)
|
||||
loads := func(source, digest string) {
|
||||
put(have, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
||||
UserList: &delivery.UserListWritten{Source: source, Revision: 1, Digest: digest, Users: 1, Key: busKey.PublicKey}})
|
||||
}
|
||||
loads("a.nats.src2", "sha256:aa")
|
||||
|
||||
seats := &standIns{ctx: ctx, js: js, active: "active"}
|
||||
store := &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
p := &grants.Part{Read: newBusReader(ctx, js), Call: seats, Store: store, Lease: &leaseHand{ctx: ctx, js: js},
|
||||
Asker: &yesAsker{}, Objects: memObjects{}, Nonce: func() string { return "0001" }}
|
||||
act := func(ask func(string, string) (any, error)) {
|
||||
t.Helper()
|
||||
if _, err := ask("operator@shanks", "issue 774's integration test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose() // the process raises the ask
|
||||
r, _ := p.LeaseState()
|
||||
if r.Open == nil {
|
||||
t.Fatalf("no act open: %+v", r)
|
||||
}
|
||||
if err := p.Warranted(asks.Option{ID: "yes"}, asks.Warrant{Ask: r.Open.Ask, Option: "yes", Outcome: asks.OutcomeChosen}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, _ = p.LeaseState()
|
||||
if r.Open == nil || r.Open.List == "" {
|
||||
t.Fatalf("the act is not waiting on its list: %+v", r)
|
||||
}
|
||||
loads(delivery.GrantsUserListSource, r.Open.List) // the bus node loads it
|
||||
p.Compose()
|
||||
if r, _ := p.LeaseState(); r.Open != nil {
|
||||
t.Fatalf("the act did not finish: %+v", r.Open)
|
||||
}
|
||||
}
|
||||
expiry := func() time.Duration {
|
||||
st, err := lease.Status(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return st.TTL()
|
||||
}
|
||||
|
||||
// 774/S23/AC1.
|
||||
act(p.LeaseWrite)
|
||||
e, err := lease.Get(ctx, buscompose.ControllerLeaseKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if away, says := buscompose.LeaseHandedAway(e.Value(), true, nil); !away || !strings.Contains(string(e.Value()), `"handed-to":"mesh-grants"`) {
|
||||
t.Fatalf("the lease after the write: %s (%s)", e.Value(), says)
|
||||
}
|
||||
if expiry() != 0 || seats.active != "inactive" {
|
||||
t.Fatalf("the lease's expiry is %s, the controller %s", expiry(), seats.active)
|
||||
}
|
||||
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); !errors.Is(err, jetstream.ErrKeyExists) {
|
||||
t.Fatalf("the controller's retake: %v, want refused", err)
|
||||
}
|
||||
time.Sleep(ttl + time.Second)
|
||||
if e, err := lease.Get(ctx, buscompose.ControllerLeaseKey); err != nil {
|
||||
t.Fatalf("the lease is gone past the old expiry: %v", err)
|
||||
} else if away, _ := buscompose.LeaseHandedAway(e.Value(), true, nil); !away {
|
||||
t.Fatalf("the lease past the old expiry: %s", e.Value())
|
||||
}
|
||||
|
||||
// 774/S23/AC2.
|
||||
act(p.LeaseHandBack)
|
||||
if expiry() != ttl || seats.active != "active" {
|
||||
t.Fatalf("after the hand-back the expiry is %s, the controller %s", expiry(), seats.active)
|
||||
}
|
||||
if _, err := lease.Create(ctx, buscompose.ControllerLeaseKey, controller); err != nil {
|
||||
t.Fatalf("the controller cannot take the lease back: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// leaseHand is the process's hand on the controller's lease (novox/hq ADR 0582 clause 3): its key holder, which the
|
||||
// lease write's cut grants the part alone, and the bucket's expiry, read from the bucket's own stream.
|
||||
type leaseHand struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
|
||||
mu sync.Mutex
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (l *leaseHand) bucket() (jetstream.KeyValue, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.kv != nil {
|
||||
return l.kv, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
|
||||
defer cancel()
|
||||
kv, err := l.js.KeyValue(ctx, buscompose.LeaseBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the lease's bucket cannot be opened: %w", err)
|
||||
}
|
||||
l.kv = kv
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
func (l *leaseHand) PutHolder(value []byte) error {
|
||||
kv, err := l.bucket()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
|
||||
defer cancel()
|
||||
_, err = kv.Put(ctx, buscompose.ControllerLeaseKey, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (l *leaseHand) DeleteHolder() error {
|
||||
kv, err := l.bucket()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
|
||||
defer cancel()
|
||||
return kv.Delete(ctx, buscompose.ControllerLeaseKey)
|
||||
}
|
||||
|
||||
func (l *leaseHand) Expiry() (time.Duration, error) {
|
||||
kv, err := l.bucket()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(l.ctx, requestWithin)
|
||||
defer cancel()
|
||||
st, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return st.TTL(), nil
|
||||
}
|
||||
|
||||
// PutTTL writes a key of the part's own state with its own time to live (the state lease).
|
||||
func (s ownStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
|
||||
return s.c.State(state).RenewTTL(key, json.RawMessage(value), ttl)
|
||||
}
|
||||
@@ -16,8 +16,10 @@
|
||||
// It holds no timer, and never exits for a refused read, write or publish: a refusal is said and tried again at the next
|
||||
// event.
|
||||
//
|
||||
// **The seat mesh-grants** (novox/hq ADR 0582 clause 1, issue 774): the part holds it for the lease write, serving
|
||||
// lease-write and lease-hand-back beside its own grants_* tools, and says lease-handed, lease-altered and removal-failed.
|
||||
// **No seat in this milestone.** ADR 0547 clause 5 gives the part the seat `mesh-grants`; a `mesh-*` seat must be in
|
||||
// mesh-sdk's own seat list, which the running controller's composer does not know, so the part serves module tools
|
||||
// (grants_*) and claims no seat until take-over. The composer grants the membership publish by the module's name,
|
||||
// mesh-grants.
|
||||
//
|
||||
// Started by the tool runner, with no credential of its own, it serves those tools (a process never serves, ADR 0436).
|
||||
package main
|
||||
@@ -119,8 +121,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
client := &asker.Client{Module: account.Module, State: conn.State(grants.StateTakeover), Publish: publishVia(ctx, js),
|
||||
Record: recordVia(ctx, nc, account.Module), Subscribe: decidedVia(nc, account.Module, logger.Printf)}
|
||||
f.Part.Asker, f.Part.Say = client, sayVia(ctx, js, account.Module)
|
||||
// The lease write's hand on the lease, and its seat's events (ADR 0582 clauses 3, 4, 6).
|
||||
f.Part.Lease, f.Part.SaySeat = &leaseHand{ctx: ctx, js: js}, seatSaying(ctx, js)
|
||||
f.Part.Forget = func(id string) error { return conn.State(grants.StateTakeover).Delete("ask." + id) }
|
||||
if err := client.Listen(f.Part.Warranted, f.Part.Said); err != nil {
|
||||
logger.Printf("the operator's answers cannot be heard yet, so take-over and hand-back wait: %v", err)
|
||||
|
||||
@@ -3,10 +3,8 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
@@ -46,10 +44,6 @@ func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(st
|
||||
|
||||
func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) }
|
||||
|
||||
func (runtimeStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
|
||||
return stdio.State(state).PutTTL(key, json.RawMessage(value), ttl)
|
||||
}
|
||||
|
||||
// runtimeBus calls tools and says events through the runtime.
|
||||
type runtimeBus struct{}
|
||||
|
||||
@@ -177,40 +171,9 @@ func toolsOf(p *grants.Part) []stdio.Tool {
|
||||
"with the tool that answers each. Asks the controller nothing; writes nothing. Read-only.",
|
||||
Input: schema(nil),
|
||||
Run: func(map[string]any) (any, error) { return p.LeaseCheck() }},
|
||||
// The seat's two verbs (ADR 0582 clauses 1, 5): asked of the operator by the part's process.
|
||||
{Name: grants.SeatTool(grants.ActLeaseWrite), Description: seatVerb(grants.ActLeaseWrite).Description,
|
||||
Input: seatVerb(grants.ActLeaseWrite).Input,
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.LeaseWrite(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
}},
|
||||
{Name: grants.SeatTool(grants.ActLeaseHandBack), Description: seatVerb(grants.ActLeaseHandBack).Description,
|
||||
Input: seatVerb(grants.ActLeaseHandBack).Input,
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.LeaseHandBack(stdio.CallerOf(a), grants.Answer(a, "why"))
|
||||
}},
|
||||
{Name: grants.ToolRemovalDue, Description: "Write the lease write's removal-due again with a short time to live, so " +
|
||||
"its expiry marker starts the day-later removal of the controller's consumer of DEAD_LETTER_NOTICES and the " +
|
||||
"stream ASSIGNMENTS (novox/hq ADR 0582 clause 6): the recovery for a marker missed while the part was down, the " +
|
||||
"removal still started by the marker and nothing else. Refused unless the lease is handed to mesh-grants and the " +
|
||||
"removal has not run. Recorded as a hand act.",
|
||||
Input: schema(map[string]string{"why": "why, kept with the hand act", "ttl": "the time to live, e.g. 10s (1s to 24h)"}, "why", "ttl"),
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return p.RemovalDue(stdio.CallerOf(a), grants.Answer(a, "why"), grants.Answer(a, "ttl"))
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// seatVerb is the seat mesh-grants' verb as mesh-sdk declares it.
|
||||
func seatVerb(name string) catalogue.Verb {
|
||||
seat, _ := catalogue.SeatNamed(catalogue.GrantsSeat)
|
||||
for _, v := range seat.Serves {
|
||||
if v.Name == name {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return catalogue.Verb{Name: name}
|
||||
}
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops.
|
||||
func serveTools() error {
|
||||
p := &grants.Part{Read: runtimeReader{}, Call: runtimeBus{}, Store: runtimeStore{}, Emit: runtimeBus{}}
|
||||
|
||||
@@ -3,21 +3,14 @@ module mesh-grants
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
@@ -20,11 +18,8 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
|
||||
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
|
||||
@@ -319,38 +319,8 @@ func (b *fakeBus) refuse(module, raw, problem string) {
|
||||
Problems: []string{problem}})
|
||||
}
|
||||
|
||||
// assign places module on node in the seat's placement, the one place the part reads it (ADR 0549 §5; issue 774).
|
||||
func (b *fakeBus) assign(node, module string) {
|
||||
var pl delivery.Placement
|
||||
if raw, ok := b.states[grants.ReadPlacement]["placement."+node]; ok {
|
||||
pl, _ = delivery.DecodePlacement(raw)
|
||||
}
|
||||
pl.Node = node
|
||||
for _, m := range pl.Modules {
|
||||
if m.Module == module {
|
||||
return
|
||||
}
|
||||
}
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: module})
|
||||
raw, err := delivery.EncodePlacement(pl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if b.states[grants.ReadPlacement] == nil {
|
||||
b.states[grants.ReadPlacement] = map[string][]byte{}
|
||||
}
|
||||
b.states[grants.ReadPlacement]["placement."+node] = raw
|
||||
}
|
||||
|
||||
// unassign takes module off node in the seat's placement.
|
||||
func (b *fakeBus) unassign(node, module string) {
|
||||
pl, err := delivery.DecodePlacement(b.states[grants.ReadPlacement]["placement."+node])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
pl.Modules = slices.DeleteFunc(pl.Modules, func(m delivery.PlacedModule) bool { return m.Module == module })
|
||||
raw, _ := delivery.EncodePlacement(pl)
|
||||
b.states[grants.ReadPlacement]["placement."+node] = raw
|
||||
b.put(grants.ReadAssignments, node+"."+module, map[string]any{"node": node, "module": module, "assigned": "2026-10-25T20:00:00Z"})
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -380,6 +350,7 @@ func fixture() *fakeBus {
|
||||
b.assign("a", "nats")
|
||||
b.assign("a", "mod-x")
|
||||
b.assign("b", "mod-y")
|
||||
b.put(grants.ReadAssignments, "b.gone", map[string]any{"node": "b", "module": "gone", "assigned": "2026-10-01T00:00:00Z", "ended": "2026-10-02T00:00:00Z"})
|
||||
b.put(grants.ReadHave, "a.node", map[string]any{"v": 1})
|
||||
b.put(grants.ReadHave, "b.node", map[string]any{"v": 1})
|
||||
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
||||
@@ -427,18 +398,8 @@ func TestGrants(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk cannot decode the manifest: %v", err)
|
||||
}
|
||||
// The part holds the seat mesh-grants for the lease write (ADR 0582 clause 1, issue 774, 774/S14), as mesh-sdk's seat
|
||||
// list declares it.
|
||||
if seat, ok := catalogue.SeatNamed(catalogue.GrantsSeat); m.Module != "mesh-grants" || len(m.Claims) != 1 || !ok ||
|
||||
catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("module %q claims %v: the part holds mesh-grants alone (%v)", m.Module, m.Claims, catalogue.CanHold(m, seat))
|
||||
}
|
||||
var keyTTL bool
|
||||
for _, s := range m.State {
|
||||
keyTTL = keyTTL || (s.Name == grants.StateLease && s.KeyTTL)
|
||||
}
|
||||
if !keyTTL {
|
||||
t.Errorf("the state %s gives no time to live per key, which removal-due needs", grants.StateLease)
|
||||
if m.Module != "mesh-grants" || len(m.Claims) != 0 {
|
||||
t.Errorf("module %q claims %v: the part claims no seat until mesh-sdk's seat list names mesh-grants", m.Module, m.Claims)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
what string
|
||||
@@ -488,9 +449,8 @@ func TestGrants(t *testing.T) {
|
||||
if path != publishOnly && path != asksOnly && (strings.Contains(string(src), ".Publish(") || strings.Contains(string(src), "mesh.assignment")) {
|
||||
t.Errorf("%s publishes: only publish.go publishes memberships, and asking.go the operator's asks", path)
|
||||
}
|
||||
// And the seat mesh-grants' events (ADR 0582 clauses 3, 4, 6; issue 774).
|
||||
if path == asksOnly && (strings.Count(string(src), ".Publish(") != 3 || strings.Contains(string(src), "mesh.assignment")) {
|
||||
t.Errorf("asking.go publishes other than an ask, a cancel and a condition, and the seat's events, through js.Publish thrice")
|
||||
if path == asksOnly && (strings.Count(string(src), ".Publish(") != 2 || strings.Contains(string(src), "mesh.assignment")) {
|
||||
t.Errorf("asking.go publishes other than an ask, a cancel and a condition, through js.Publish twice")
|
||||
}
|
||||
if path == publishOnly && strings.Count(string(src), ".Publish(") != 1 {
|
||||
t.Errorf("publish.go publishes other than one membership")
|
||||
@@ -541,8 +501,8 @@ func TestGrants(t *testing.T) {
|
||||
key, _ := delivery.BuilderBuildKey("mod-x", fp)
|
||||
b.put(grants.ReadBuilds, key, map[string]any{"v": delivery.Version, "state": "done", "manifest": digest})
|
||||
}, []string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"},
|
||||
{"placement refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadPlacement] = true },
|
||||
[]string{"controller", "node.a", "node.b"}, grants.ReadPlacement},
|
||||
{"placement refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
|
||||
[]string{"controller", "node.a", "node.b"}, grants.ReadAssignments},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
b := fixture()
|
||||
@@ -1130,10 +1090,8 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("read of the keys per module: placed %v, gaps %v", got.Placed, got.Gaps)
|
||||
}
|
||||
b.refused[grants.ReadPlacement] = true
|
||||
// The placement alone places (issue 774, 774/S29): not read, nothing is placed, and it is a failed read.
|
||||
if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadPlacement && g.Failed }) ||
|
||||
len(read.Assigned) != 0 {
|
||||
t.Errorf("placement not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
|
||||
if read := grants.ReadInputs(b, &grants.Manifests{}); slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadPlacement && g.Failed }) {
|
||||
t.Errorf("placement not on the bus is a failed read: %v", read.Gaps)
|
||||
}
|
||||
// Issue 1945, the user list of 2026-10-27 09:21: a placement's holds are the holders on record, not every holder.
|
||||
// A mesh seat no placement holds keeps the module claiming it as its holder, wherever placement puts it.
|
||||
@@ -1405,7 +1363,7 @@ func TestGrants(t *testing.T) {
|
||||
// settled is the placement of the lists agreeing, every hash held, and the loaded list as edit leaves it.
|
||||
settled := func(edit func(modX *grants.User)) func() {
|
||||
return func() {
|
||||
b.states[grants.ReadPlacement] = nil
|
||||
b.states[grants.ReadAssignments] = nil
|
||||
b.assign("a", "nats")
|
||||
b.assign("a", "mod-x")
|
||||
b.assign("b", "mod-y")
|
||||
@@ -1616,7 +1574,7 @@ func TestGrants(t *testing.T) {
|
||||
b.loaded[i].Subscribe = append(slices.Clone(b.loaded[i].Subscribe), seatGrant)
|
||||
}
|
||||
}
|
||||
b.assign("a", "mod-y")
|
||||
b.put(grants.ReadAssignments, "a.mod-y", map[string]any{"node": "a", "module": "mod-y", "assigned": "2026-10-27T08:20:00Z"})
|
||||
b.put(grants.ReadAccounts, "a.mod-y", map[string]any{"user": "a.mod-y", "hash": "$2a$10$x", "generation": 1})
|
||||
said = nil
|
||||
res := p.Compose()
|
||||
@@ -1638,7 +1596,7 @@ func TestGrants(t *testing.T) {
|
||||
b.refuseUsers = true
|
||||
b.mu.Unlock()
|
||||
b.mu.Lock()
|
||||
b.unassign("a", "mod-y")
|
||||
delete(b.states[grants.ReadAssignments], "a.mod-y")
|
||||
b.mu.Unlock()
|
||||
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "the grants of the list in force cannot be read") {
|
||||
t.Errorf("a pass whose loaded list cannot be read = %+v", res.Sealed)
|
||||
@@ -2076,11 +2034,11 @@ func TestGrants(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { f.Run(ctx); close(done) }()
|
||||
waitOpened(t, w, grants.ReadPlacement)
|
||||
waitOpened(t, w, grants.ReadAssignments)
|
||||
if r := f.Refused(); r[grants.ReadModules] == "" {
|
||||
waitFor(t, func() bool { return f.Refused()[grants.ReadModules] != "" })
|
||||
}
|
||||
w.change(grants.ReadPlacement, "placement.a")
|
||||
w.change(grants.ReadAssignments, "a.mod-x")
|
||||
waitOpened(t, w, grants.ReadModules)
|
||||
waitFor(t, func() bool { return len(f.Refused()) == 0 })
|
||||
select {
|
||||
@@ -2273,64 +2231,83 @@ func TestGrants(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// ADR 0571 §13's test for this repository (issue 774, 774/S24/AC1), replacing the guard of ADR 0549 §6 and its list
|
||||
// ../assignments-readers: at phase write no module.json here depends on mesh-controller beyond the lease and the three
|
||||
// kept to the write (mesh-sdk catalogue.ManifestRefusals, the judge grants_lease_check runs).
|
||||
t.Run("no module of this repository depends on the controller beyond what the lease write keeps", func(t *testing.T) {
|
||||
// The guard of ADR 0549 §6 for this repository, as mesh-catalog's checks/placement is for the catalogue: no module.json
|
||||
// here reads the controller's assignments unless ../assignments-readers names it with why, and that list only shrinks.
|
||||
t.Run("assignments readers: only the named modules read the controller's assignments, and the list only shrinks", func(t *testing.T) {
|
||||
raw, err := os.ReadFile("../assignments-readers")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowed := map[string]bool{}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if f := strings.Fields(line); len(f) > 1 && !strings.HasPrefix(f[0], "#") {
|
||||
allowed[f[0]] = true
|
||||
} else if len(f) == 1 {
|
||||
t.Errorf("../assignments-readers names %s with no why", f[0])
|
||||
}
|
||||
}
|
||||
paths, _ := filepath.Glob("../*/module.json")
|
||||
manifests := map[string]catalogue.Manifest{}
|
||||
reads := map[string]bool{}
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Reads []string `json:"reads"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatalf("%s: %v", path, err)
|
||||
}
|
||||
manifests[m.Module] = m
|
||||
if slices.Contains(m.Reads, grants.ReadAssignments) {
|
||||
reads[m.Module] = true
|
||||
if !allowed[m.Module] {
|
||||
t.Errorf("%s reads %s: read where modules run from the seat mesh-placement's placement (ADR 0549 §3); the lease write "+
|
||||
"is refused while a module reads the assignments (ADR 0549 §6)", m.Module, grants.ReadAssignments)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(manifests) < 8 {
|
||||
t.Fatalf("read %d manifests, want every module of the repository", len(manifests))
|
||||
}
|
||||
for _, why := range catalogue.ManifestRefusals(manifests, catalogue.PhaseWrite) {
|
||||
t.Error(why)
|
||||
for m := range allowed {
|
||||
if !reads[m] {
|
||||
t.Errorf("../assignments-readers names %s, which no longer reads %s: take it off the list", m, grants.ReadAssignments)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 774/S29/AC1 (ADR 0549 §5, ADR 0571 §10): the memberships composed from the placement alone, with no read of the
|
||||
// controller, are the ones the union of placement and assignments gave on the same input: the fixture's (two nodes,
|
||||
// the bus and mod-x on a, mod-y on b, as placed on the live mesh's own shape of nats beside its modules), whose
|
||||
// assignments named the same modules. Neither the controller's assignments nor its condition events are read.
|
||||
t.Run("placement alone places, with the memberships the union gave", func(t *testing.T) {
|
||||
// 774 (the L3 review, ADR 0549 §3, §6): once the controller's lease is handed away the assignments are frozen and place
|
||||
// nothing; the seat's placement alone does. A module left only in the stale assignments loses its account and its
|
||||
// grant names no account of it; before the lease write the two are still unioned.
|
||||
t.Run("placement alone once the lease is handed away: a module only in the stale assignments has no account or grant of it", func(t *testing.T) {
|
||||
b := fixture()
|
||||
b.loaded = shadowOf(t, b)
|
||||
p := partOn(b)
|
||||
p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"}
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "a", Modules: []delivery.PlacedModule{{Module: "nats"}, {Module: "mod-x"}}})
|
||||
b.states[grants.ReadPlacement] = map[string][]byte{"placement.a": pl}
|
||||
for _, u := range []string{"a.mod-x", "a.nats", "b.mod-y"} {
|
||||
b.put(grants.ReadAccounts, u, map[string]any{"user": u, "hash": "$2a$10$x", "generation": 1})
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("ab", 32)
|
||||
key, _ := delivery.PartGrantKey("mod-y", fp)
|
||||
grantOf := func() []string {
|
||||
g, _ := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key])
|
||||
return g.Accounts
|
||||
}
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "novox/1/2026-10-28", "epoch": 60})
|
||||
p.Compose()
|
||||
want := map[string]string{"a.nats": "", "a.mod-x": "", "b.mod-y": ""}
|
||||
for key := range want {
|
||||
raw, _, found, _ := b.ownGet(grants.StateMemberships, key)
|
||||
if !found {
|
||||
t.Fatalf("no membership of %s from the placement", key)
|
||||
}
|
||||
want[key] = string(raw)
|
||||
if b.own[grants.StateAccounts]["b.mod-y"] == nil || !slices.Equal(grantOf(), []string{"b.mod-y"}) {
|
||||
t.Fatalf("under the controller's lease the assignments no longer count: account %v, grant %v",
|
||||
b.own[grants.StateAccounts]["b.mod-y"] != nil, grantOf())
|
||||
}
|
||||
if got := slices.Sorted(maps.Keys(b.own[grants.StateMemberships])); !slices.Equal(got, []string{"a.mod-x", "a.nats", "b.mod-y"}) {
|
||||
t.Errorf("memberships %v", got)
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
|
||||
in := grants.ReadInputs(b, &grants.Manifests{})
|
||||
if len(in.Assigned["b"]) != 0 || !slices.Equal(in.Assigned["a"], []string{"mod-x", "nats"}) || in.BusNode != "a" {
|
||||
t.Errorf("after the lease write the stale assignments still place: %v, bus %q", in.Assigned, in.BusNode)
|
||||
}
|
||||
for _, r := range grants.Reads {
|
||||
if strings.HasPrefix(r, "mesh-controller.") && r != grants.ReadLease {
|
||||
t.Errorf("the part reads %s", r)
|
||||
}
|
||||
}
|
||||
for _, c := range grants.Consumes {
|
||||
if strings.HasPrefix(c, "mesh-controller.") {
|
||||
t.Errorf("the part consumes %s", c)
|
||||
}
|
||||
}
|
||||
for _, c := range b.calls {
|
||||
if strings.Contains(c, "mesh-controller") {
|
||||
t.Errorf("the part called %s", c)
|
||||
}
|
||||
res := p.Compose()
|
||||
if res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-y"}) || slices.Contains(grantOf(), "b.mod-y") {
|
||||
t.Fatalf("a module only in the stale assignments kept its account or grant: %+v, grant %v", res.Accounts, grantOf())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2372,7 +2349,7 @@ func TestGrants(t *testing.T) {
|
||||
t.Errorf("the wanted accounts are read by %v", readers)
|
||||
}
|
||||
// A read that fails retires nothing; once read, the user placement dropped is retired.
|
||||
b.unassign("b", "mod-y")
|
||||
delete(b.states[grants.ReadAssignments], "b.mod-y")
|
||||
b.refused[grants.ReadResolved] = true
|
||||
if res := p.Compose(); res.Accounts == nil || len(res.Accounts.Deleted) != 0 || b.own[grants.StateAccounts]["b.mod-y"] == nil {
|
||||
t.Fatalf("a pass whose read failed deleted %+v", res.Accounts)
|
||||
|
||||
@@ -20,13 +20,7 @@ type Watcher interface {
|
||||
// Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved
|
||||
// config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
|
||||
// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
|
||||
//
|
||||
// And its own state lease, whose key removal-due's expiry marker starts the day-later removal (ADR 0582 clause 6).
|
||||
var Followed = []string{ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover,
|
||||
OwnLease}
|
||||
|
||||
// OwnLease is the part's own state lease as the follower watches it, `<module>.<state>`.
|
||||
const OwnLease = Module + "." + StateLease
|
||||
var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPlacement, ReadLease, ReadTakeover}
|
||||
|
||||
// Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
|
||||
// open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
|
||||
@@ -127,23 +121,14 @@ func (f *Follower) heard(state, key string, value []byte) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
if state == OwnLease {
|
||||
// The marker of removal-due (its value gone: expired, or deleted by a hand-back, which the removal tells apart).
|
||||
if key == KeyRemovalDue && value == nil && f.Part != nil {
|
||||
f.Part.RemovalDueHeard()
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if state == ReadLease {
|
||||
// Renewed every few seconds by its holder: only the write that hands it away, or back, composes again (ADR 0561 §6).
|
||||
// Once handed away nobody renews it, so any change is heard: it may be the lease altered (ADR 0582 clause 4).
|
||||
on, _ := buscompose.LeaseHandedAway(value, len(value) > 0, nil)
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
was, seen := f.leaseOn, f.leaseSeen
|
||||
f.leaseOn, f.leaseSeen = on, true
|
||||
return !seen || was != on || on
|
||||
return !seen || was != on
|
||||
}
|
||||
if state != ReadHave {
|
||||
return true
|
||||
@@ -184,8 +169,7 @@ func (f *Follower) open() {
|
||||
}
|
||||
if f.Hear != nil && !f.heard_ {
|
||||
err := f.Hear(func(event string, body []byte) {
|
||||
// An account the vault minted, or a bucket's expiry set (the lease's, altered: ADR 0582 clause 4).
|
||||
if IsAccountIssued(event) || IsBucketExpirySet(event) {
|
||||
if IsAccountIssued(event) {
|
||||
f.Wake()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -24,12 +24,13 @@ import (
|
||||
|
||||
// The states of other parts the grants part reads, as its manifest's reads name them (`<module>.<state>`).
|
||||
const (
|
||||
ReadModules = "mesh-delivery-indexer.modules"
|
||||
ReadManifests = "mesh-delivery-indexer.manifests"
|
||||
ReadHave = "node-engine.have"
|
||||
ReadResolved = "mesh-config.resolved"
|
||||
ReadWants = "mesh-delivery-assigner.wants"
|
||||
ReadBuilds = "mesh-delivery-builder.builds"
|
||||
ReadAssignments = "mesh-controller.assignments"
|
||||
ReadModules = "mesh-delivery-indexer.modules"
|
||||
ReadManifests = "mesh-delivery-indexer.manifests"
|
||||
ReadHave = "node-engine.have"
|
||||
ReadResolved = "mesh-config.resolved"
|
||||
ReadWants = "mesh-delivery-assigner.wants"
|
||||
ReadBuilds = "mesh-delivery-builder.builds"
|
||||
// ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9).
|
||||
ReadAccounts = "mesh-vault.bus-accounts"
|
||||
// ReadPlacement is the seat mesh-placement's placement (ADR 0567 §1, §9): each node's modules and the seats
|
||||
@@ -37,9 +38,10 @@ const (
|
||||
ReadPlacement = delivery.PlacementRead
|
||||
)
|
||||
|
||||
// Reads is every state the part reads, in the manifest's order. Not the controller's assignments: the seat's placement
|
||||
// alone places modules (novox/hq ADR 0549 §5, ADR 0571 §10; issue 774).
|
||||
var Reads = []string{ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
|
||||
// Reads is every state the part reads, in the manifest's order. ReadAssignments stays (and in Followed) only while the
|
||||
// controller holds the lease, when the assignments are unioned with the placement (ReadInputs); after the lease write
|
||||
// they place nothing, and the first change after it takes them out (../assignments-readers names the part until then).
|
||||
var Reads = []string{ReadAssignments, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
|
||||
ReadAccounts, ReadPlacement, ReadLease, ReadTakeover}
|
||||
|
||||
// The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
|
||||
@@ -50,7 +52,7 @@ const (
|
||||
)
|
||||
|
||||
// Invokes is every tool the part calls.
|
||||
var Invokes = append([]string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant}, LeaseCalls...)
|
||||
var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree, CallRecordWarrant}
|
||||
|
||||
// Reader reads another part's state, named `<module>.<state>`. A refusal (a read not granted yet, a bucket not there) is
|
||||
// an error the part names as a gap and composes past: it never stops for one.
|
||||
@@ -81,6 +83,18 @@ var standingGaps = []Gap{
|
||||
{Input: "announced states", Why: "the controller does not announce its states-announced readable, so every read is composed from `reads`, as when announcements cannot be read"},
|
||||
}
|
||||
|
||||
// assignment is a key of the controller's assignments as composing needs it (mesh-controller internal/chain Assignment):
|
||||
// a value that names no time assigned (a module pulled in), has ended, or waits in a state, is no assignment.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Assigned time.Time `json:"assigned,omitzero"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
State string `json:"state,omitempty"`
|
||||
}
|
||||
|
||||
func (a assignment) stands() bool { return !a.Assigned.IsZero() && a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// resolved is a key of mesh-config's resolved values, `<node>.<module>.<key>`.
|
||||
type resolved struct {
|
||||
Node string `json:"node"`
|
||||
@@ -91,9 +105,10 @@ type resolved struct {
|
||||
|
||||
// Inputs is what the shadow composes from, read from the bus.
|
||||
type Inputs struct {
|
||||
// Nodes is every machine seen: in a node agent's have or the placement.
|
||||
// Nodes is every machine seen: in a node agent's have or a standing assignment.
|
||||
Nodes []string
|
||||
// Assigned is what the seat's placement puts on each machine, sorted (issue 774: never the controller's assignments).
|
||||
// Assigned is each machine's standing assignments and what the seat's placement puts there, sorted: while both
|
||||
// stand (ADR 0549 §5) a module either names keeps its membership.
|
||||
Assigned map[string][]string
|
||||
// Placed is each node's placement as the seat mesh-placement holds it (ADR 0567 §1); empty before any is read.
|
||||
// Where it is, a seat held once for the mesh is held where placement says, not wherever a module claiming it runs.
|
||||
@@ -233,11 +248,26 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
NoFingerprint: map[string]string{}, Running: map[string]catalogue.Manifest{},
|
||||
Values: map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
|
||||
nodes := map[string]bool{}
|
||||
// **The seat's placement alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's
|
||||
// assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed
|
||||
// read: nothing is placed, so after take-over the list composed past it is not sealed.
|
||||
// **After the lease write the assignments place nothing** (novox/hq ADR 0549 §3, §6): they are frozen then, so a
|
||||
// module the catalogue takes off a node would keep its account, grant, membership and user-list entry. Once the lease
|
||||
// is handed away and the placement is read, the placement alone places modules, as the assigner's Placed does with
|
||||
// the same lease reading (buscompose.LeaseHandedAway); before, or with no placement on the bus, the two are unioned.
|
||||
byAssignment, busByAssignment := map[string][]string{}, ""
|
||||
for _, raw := range each(r, ReadAssignments, &in.Gaps) {
|
||||
var a assignment
|
||||
if json.Unmarshal(raw, &a) != nil || !a.stands() || a.Node == "" || a.Module == "" {
|
||||
continue
|
||||
}
|
||||
byAssignment[a.Node] = append(byAssignment[a.Node], a.Module)
|
||||
if a.Module == BusModule {
|
||||
busByAssignment = a.Node
|
||||
}
|
||||
}
|
||||
// The seat's placement (ADR 0567 §9): what it places is composed beside the assignments, and the seats held once
|
||||
// for the mesh are held where it says. Not on the bus yet is a standing gap, never a failed read, so the list is still
|
||||
// composed and sealed from the assignments alone.
|
||||
if keys, err := r.Keys(ReadPlacement); err != nil {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true})
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadPlacement, Why: "not read yet: the assignments alone place modules (" + err.Error() + ")"})
|
||||
} else {
|
||||
// Each node's keys in either form, assembled by the seat's one reader (delivery.PlacementsOf; research 072
|
||||
// row 37): its node key and a key per module, or its legacy key `placement.<node>`.
|
||||
@@ -264,6 +294,17 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
in.Placed[node] = p
|
||||
}
|
||||
}
|
||||
away, _ := leaseHandedAway(r)
|
||||
if away && len(in.Placed) > 0 {
|
||||
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "not used: the controller's lease is handed away, " +
|
||||
"so the seat's placement alone places modules (ADR 0549 §3, §6)"})
|
||||
} else {
|
||||
for node, ms := range byAssignment {
|
||||
nodes[node] = true
|
||||
in.Assigned[node] = append(in.Assigned[node], ms...)
|
||||
}
|
||||
in.BusNode = busByAssignment
|
||||
}
|
||||
for node, p := range in.Placed {
|
||||
nodes[node] = true
|
||||
for _, m := range p.Assigned() {
|
||||
@@ -287,7 +328,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
|
||||
nodes[node] = true
|
||||
}
|
||||
}
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "machines", Why: "read from the node agents' haves and the placement: a machine with neither is not seen"})
|
||||
in.Gaps = append(in.Gaps, Gap{Input: "machines", Why: "read from the node agents' haves and the assignments: a machine with neither is not seen"})
|
||||
for n := range nodes {
|
||||
in.Nodes = append(in.Nodes, n)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package grants
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -26,40 +27,231 @@ const ToolLeaseCheck = "grants_lease_check"
|
||||
// Controller is the module and the seat the check is about.
|
||||
const Controller = "mesh-controller"
|
||||
|
||||
// **The judge is mesh-sdk's** (catalogue.Dependencies, catalogue.Judge; issue 774): the same judgement the three
|
||||
// repositories' manifest tests run through catalogue.ManifestRefusals (ADR 0571 §13), re-exported here by its names.
|
||||
// The kinds of a dependency on the controller, as a manifest declares them.
|
||||
const (
|
||||
DependsRead = catalogue.DependsRead
|
||||
DependsInvoke = catalogue.DependsInvoke
|
||||
DependsConsume = catalogue.DependsConsume
|
||||
DependsUses = catalogue.DependsUses
|
||||
DependsWaits = catalogue.DependsWaits
|
||||
PhaseBefore = catalogue.PhaseBefore
|
||||
PhaseWrite = catalogue.PhaseWrite
|
||||
DependsRead = "read"
|
||||
DependsInvoke = "invoke"
|
||||
DependsConsume = "consume"
|
||||
DependsUses = "uses"
|
||||
DependsWaits = "waits"
|
||||
)
|
||||
|
||||
type (
|
||||
// Dependency, Kept, Verdict and Coverage are mesh-sdk's.
|
||||
Dependency = catalogue.Dependency
|
||||
Kept = catalogue.KeptDependency
|
||||
Verdict = catalogue.Verdict
|
||||
Coverage = catalogue.Coverage
|
||||
// The phases of the lease write's check (ADR 0571 §13): before the lease write's own delivery the whole named list is
|
||||
// allowed; that delivery drops every entry but the three kept to the write itself, and its check runs with those alone.
|
||||
const (
|
||||
PhaseBefore = "before-delivery"
|
||||
PhaseWrite = "write"
|
||||
)
|
||||
|
||||
// KeepList and EndedBy are mesh-sdk's named list and the records that ended what it no longer keeps.
|
||||
var (
|
||||
KeepList = catalogue.KeepList
|
||||
EndedBy = catalogue.EndedBy
|
||||
)
|
||||
|
||||
// Dependencies are every declaration on the controller in the manifests given (catalogue.Dependencies).
|
||||
func Dependencies(manifests map[string]catalogue.Manifest) []Dependency {
|
||||
return catalogue.Dependencies(manifests)
|
||||
// Dependency is one declaration of a manifest other than the controller's on the controller: a read of one of its
|
||||
// states, an invoke of one of its verbs or tools, a consume or a wait on one of its events, or a send to its seat.
|
||||
type Dependency struct {
|
||||
Module string `json:"module"`
|
||||
// By is the account declaring it when it is not the module's own: `process <id>` or `tools <binary>`.
|
||||
By string `json:"by,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
What string `json:"what"`
|
||||
// Kept is the record that keeps it until the lease write, from the named list; empty when none does.
|
||||
Kept string `json:"kept,omitempty"`
|
||||
// ToTheWrite says it is one of the three kept to the write itself, which the lease write's own delivery keeps.
|
||||
ToTheWrite bool `json:"to_the_write,omitempty"`
|
||||
}
|
||||
|
||||
// Judge runs the lease write's check in one phase (catalogue.Judge).
|
||||
// Kept is one entry of the named, temporary list ADR 0571 §13 allows: a read, invoke or consume another accepted record
|
||||
// keeps until the lease write, with that record. ToTheWrite marks the three kept to the write itself.
|
||||
type Kept struct {
|
||||
Module, Kind, What, Record string
|
||||
ToTheWrite bool
|
||||
}
|
||||
|
||||
// KeepList is the named list. Each entry names the clause that keeps it; a dependency not on it fails the check. The
|
||||
// third entry kept to the write, the bus node agent's following of the controller's user list (ADR 0571 §8, ADR 0547
|
||||
// §8), is no manifest's declaration but the node-engine's own, so no manifest matches it; it stays named here so the
|
||||
// list reads as the record has it.
|
||||
var KeepList = []Kept{
|
||||
// Kept to the write itself (ADR 0571 §4, §8, §13).
|
||||
{Module: "mesh-delivery-builder", Kind: DependsConsume, What: "mesh-controller.artifacts-gone", Record: "ADR 0571 §4, §13", ToTheWrite: true},
|
||||
{Module: "artifact-store-tools", Kind: DependsInvoke, What: "seat:mesh-controller.collect", Record: "ADR 0571 §4, §13", ToTheWrite: true},
|
||||
{Module: "node-engine", Kind: DependsRead, What: "the controller's user list of the bus module's composition", Record: "ADR 0571 §8, §13; ADR 0547 §8", ToTheWrite: true},
|
||||
// Kept until the lease write's own delivery.
|
||||
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.placement-export", Record: "ADR 0567 §7"},
|
||||
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.assign", Record: "ADR 0567 §8"},
|
||||
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.unassign", Record: "ADR 0567 §8"},
|
||||
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.pin", Record: "ADR 0567 §8"},
|
||||
{Module: "mesh-delivery-assigner", Kind: DependsInvoke, What: "seat:mesh-controller.unpin", Record: "ADR 0567 §8"},
|
||||
{Module: "mesh-delivery-assigner", Kind: DependsRead, What: "mesh-controller.assignments", Record: "ADR 0567 §8"},
|
||||
{Module: "mesh-delivery-assigner", Kind: DependsRead, What: "mesh-controller.composed", Record: "ADR 0571 §8"},
|
||||
{Module: "mesh-grants", Kind: DependsRead, What: "mesh-controller.assignments", Record: "ADR 0549 §5"},
|
||||
{Module: "mesh-grants", Kind: DependsConsume, What: "mesh-controller.condition-raised", Record: "ADR 0564 §9"},
|
||||
{Module: "mesh-grants", Kind: DependsConsume, What: "mesh-controller.condition-cleared", Record: "ADR 0564 §9"},
|
||||
{Module: "messenger", Kind: DependsConsume, What: "mesh-controller.condition-raised", Record: "ADR 0564 §9"},
|
||||
{Module: "messenger", Kind: DependsConsume, What: "mesh-controller.condition-cleared", Record: "ADR 0564 §9"},
|
||||
{Module: "messenger", Kind: DependsConsume, What: "mesh-controller.condition-changed", Record: "ADR 0564 §9"},
|
||||
{Module: "messenger", Kind: DependsInvoke, What: "seat:mesh-controller.conditions", Record: "ADR 0564 §9"},
|
||||
{Module: "mesh-watcher", Kind: DependsConsume, What: "mesh-controller.doctor-heartbeat", Record: "ADR 0564 §9"},
|
||||
{Module: "mesh-conditions", Kind: DependsRead, What: "mesh-controller.conditions", Record: "ADR 0564 §9"},
|
||||
{Module: "mesh-config", Kind: DependsRead, What: "mesh-controller.declared-config", Record: "ADR 0565 §4"},
|
||||
{Module: "mesh-config", Kind: DependsRead, What: "mesh-controller.predefined", Record: "ADR 0565 §4"},
|
||||
{Module: "mesh-healers", Kind: DependsRead, What: "mesh-controller.hand-acts", Record: "ADR 0565 §11"},
|
||||
{Module: "mesh-hand-acts", Kind: DependsRead, What: "mesh-controller.hand-acts", Record: "ADR 0565 §11"},
|
||||
}
|
||||
|
||||
// EndedBy names, for a declaration no entry keeps, the record that already ended it, so a refusal says where it moved.
|
||||
// The controller's `grants` is one: ADR 0571 §9 has no manifest read it from L7's delivery, which changes ADR 0567 §10
|
||||
// and ADR 0547 §5's shadow compare, so neither the assigner's nor the grants part's read of it is kept.
|
||||
var EndedBy = map[string]string{
|
||||
DependsRead + " mesh-controller.grants": "ADR 0571 §9: no manifest reads the controller's grants; the grants part's key is the one read",
|
||||
DependsInvoke + " seat:mesh-controller.hand-act": "ADR 0565 §11: every recorder calls mesh-hand-acts",
|
||||
DependsInvoke + " seat:mesh-controller.warranted": "ADR 0565 §11: every recorder calls mesh-hand-acts",
|
||||
DependsInvoke + " seat:mesh-controller.root-free": "ADR 0571 §11: each machine's engine answers root-free (seat:node-engine.root-free@machine)",
|
||||
DependsInvoke + " seat:mesh-controller.held": "ADR 0571 §2: held is the seat mesh-artifact-sweep's",
|
||||
DependsRead + " mesh-controller.declared-packages": "ADR 0565 §6: each node agent writes its own node's packages",
|
||||
}
|
||||
|
||||
// keptFor is the list's entry for one dependency, or nil.
|
||||
func keptFor(d Dependency) *Kept {
|
||||
for i, k := range KeepList {
|
||||
if k.Module == d.Module && k.Kind == d.Kind && k.What == d.What {
|
||||
return &KeepList[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// onController says whether a name `<owner>.<rest>` is the controller's.
|
||||
func onController(name string) bool { return strings.HasPrefix(name, Controller+".") }
|
||||
|
||||
// invokesController says whether an invoke names a verb of the controller's seat or a tool of its module. `*` names none:
|
||||
// it is the console's grant to every tool, judged by no record as a dependency.
|
||||
func invokesController(invoke string) bool {
|
||||
if seat, _, ok := catalogue.SeatInvoke(invoke); ok {
|
||||
return seat == Controller
|
||||
}
|
||||
return onController(invoke)
|
||||
}
|
||||
|
||||
// Dependencies are every declaration on the controller in the manifests given, by module, each with the list's entry
|
||||
// that keeps it. The controller's own manifest is not judged, nor a read of its lease key: it names the lease's holder,
|
||||
// whoever that is, and parts read it to learn that the write happened (ADR 0571 §13).
|
||||
func Dependencies(manifests map[string]catalogue.Manifest) []Dependency {
|
||||
var out []Dependency
|
||||
add := func(module, by, kind, what string) {
|
||||
d := Dependency{Module: module, By: by, Kind: kind, What: what}
|
||||
if k := keptFor(d); k != nil {
|
||||
d.Kept, d.ToTheWrite = k.Record, k.ToTheWrite
|
||||
}
|
||||
if !slices.Contains(out, d) {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
for name, m := range manifests {
|
||||
if name == Controller || m.Module == Controller {
|
||||
continue
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if onController(r) && r != buscompose.ControllerLease {
|
||||
add(name, "", DependsRead, r)
|
||||
}
|
||||
}
|
||||
for _, i := range m.Invokes {
|
||||
if invokesController(i) {
|
||||
add(name, "", DependsInvoke, i)
|
||||
}
|
||||
}
|
||||
for _, c := range m.Consumes {
|
||||
if onController(c) {
|
||||
add(name, "", DependsConsume, c)
|
||||
}
|
||||
}
|
||||
for _, w := range m.Waits {
|
||||
if onController(w) {
|
||||
add(name, "", DependsWaits, w)
|
||||
}
|
||||
}
|
||||
for _, u := range m.Uses {
|
||||
if u == Controller {
|
||||
add(name, "", DependsUses, u)
|
||||
}
|
||||
}
|
||||
for _, a := range m.BusAccounts() {
|
||||
by := "process " + a.ID
|
||||
if a.Tools {
|
||||
by = "tools " + a.ID
|
||||
}
|
||||
for _, i := range a.Bus.Invokes {
|
||||
if invokesController(i) {
|
||||
add(name, by, DependsInvoke, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := out[i], out[j]
|
||||
if a.Module != b.Module {
|
||||
return a.Module < b.Module
|
||||
}
|
||||
if a.Kind != b.Kind {
|
||||
return a.Kind < b.Kind
|
||||
}
|
||||
if a.What != b.What {
|
||||
return a.What < b.What
|
||||
}
|
||||
return a.By < b.By
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// Verdict is the lease write's check in one phase.
|
||||
type Verdict struct {
|
||||
Phase string `json:"phase"`
|
||||
Passes bool `json:"passes"`
|
||||
Reasons []string `json:"reasons"`
|
||||
}
|
||||
|
||||
// Coverage is how much of the indexer the dependencies were derived from: the manifests read, the modules the indexer
|
||||
// holds (-1 when its keys could not be listed, which Unread then names), and each manifest that could not be read.
|
||||
type Coverage struct {
|
||||
Read, Indexed int
|
||||
Unread []Gap
|
||||
}
|
||||
|
||||
// Judge runs the lease write's check over the dependencies (ADR 0571 §13) in one phase: before the lease write's own
|
||||
// delivery, every dependency must be on the named list; at the write, only the three kept to the write itself may stay.
|
||||
// It passes only over every module the indexer holds: no manifest read, fewer read than the indexer holds, or one that
|
||||
// could not be read fails it, since what was not read could hide a dependency. A lease already handed away leaves
|
||||
// nothing to write.
|
||||
func Judge(deps []Dependency, phase string, cov Coverage, handedAway bool, leaseSays string) Verdict {
|
||||
return catalogue.Judge(deps, phase, cov, handedAway, leaseSays)
|
||||
v := Verdict{Phase: phase, Reasons: []string{}}
|
||||
if handedAway {
|
||||
v.Reasons = append(v.Reasons, "there is nothing to write: "+leaseSays)
|
||||
}
|
||||
switch {
|
||||
case cov.Read == 0:
|
||||
v.Reasons = append(v.Reasons, fmt.Sprintf("no manifest was read from the indexer (it holds %d modules): with nothing read, nothing on the controller can be judged absent", max(cov.Indexed, 0)))
|
||||
case cov.Indexed >= 0 && cov.Read < cov.Indexed:
|
||||
v.Reasons = append(v.Reasons, fmt.Sprintf("only %d of the %d modules the indexer holds had their manifest read, so what the others declare on the controller is not known", cov.Read, cov.Indexed))
|
||||
}
|
||||
for _, d := range deps {
|
||||
who := d.Module
|
||||
if d.By != "" {
|
||||
who += " (" + d.By + ")"
|
||||
}
|
||||
switch {
|
||||
case d.Kept == "":
|
||||
why := fmt.Sprintf("%s declares %s %s, which no record keeps until the lease write (ADR 0571 §13)", who, d.Kind, d.What)
|
||||
if ended := EndedBy[d.Kind+" "+d.What]; ended != "" {
|
||||
why += "; ended by " + ended
|
||||
}
|
||||
v.Reasons = append(v.Reasons, why)
|
||||
case phase == PhaseWrite && !d.ToTheWrite:
|
||||
v.Reasons = append(v.Reasons, fmt.Sprintf("%s declares %s %s, kept by %s only until the lease write's own delivery, which drops it", who, d.Kind, d.What, d.Kept))
|
||||
}
|
||||
}
|
||||
for _, g := range cov.Unread {
|
||||
v.Reasons = append(v.Reasons, fmt.Sprintf("%s could not be read (%s), so what it declares on the controller is not known", g.Input, g.Why))
|
||||
}
|
||||
v.Passes = len(v.Reasons) == 0
|
||||
return v
|
||||
}
|
||||
|
||||
// LeaseNow is who holds the lease and what follows from it.
|
||||
@@ -81,6 +273,7 @@ var (
|
||||
"mesh-healers writes each act it would take in `would`, and acts on none (ADR 0564 §8)",
|
||||
"the artifact sweep lets go of nothing; the controller's take-in sweeps and says `artifacts-gone` (ADR 0571 §4)",
|
||||
"the watchman only lists dead letters; the controller keeps DEAD_LETTERS through its consumer of DEAD_LETTER_NOTICES (ADR 0564 §5)",
|
||||
"placement proposals take only assignments, pins and config values, and the assigner keeps the controller's assignments in step (ADR 0567 §8, §9)",
|
||||
"the controller keeps composing for the bus node, whose agent follows its user list, so the hand-backs of config, conditions, placement and the user list still answer (ADR 0547 §8, ADR 0564 §9, ADR 0565 §4, ADR 0567 §11)",
|
||||
}
|
||||
DoesHandedAway = []string{
|
||||
@@ -220,6 +413,8 @@ type LeaseAnswer struct {
|
||||
Streams []StreamUse `json:"streams,omitempty"`
|
||||
Idle []string `json:"streams_idle,omitempty"`
|
||||
StreamsError string `json:"streams_unread,omitempty"`
|
||||
// Assignments is how many of the controller's assignments stand, which it publishes memberships and grants for.
|
||||
Assignments int `json:"assignments_standing"`
|
||||
// Check is the dry run at the write (the three alone), Before the same over the whole list before the write's
|
||||
// delivery.
|
||||
Check Verdict `json:"check"`
|
||||
@@ -255,16 +450,20 @@ func (p *Part) LeaseCheck() (any, error) {
|
||||
out.Unkept[d.Module] = append(out.Unkept[d.Module], d.Kind+" "+d.What)
|
||||
}
|
||||
}
|
||||
var gaps []Gap
|
||||
for _, raw := range each(p.Read, ReadAssignments, &gaps) {
|
||||
var a assignment
|
||||
if json.Unmarshal(raw, &a) == nil && a.stands() {
|
||||
out.Assignments++
|
||||
}
|
||||
}
|
||||
if used, idle, err := Duties(p.Call); err != nil {
|
||||
out.StreamsError = err.Error()
|
||||
} else {
|
||||
out.Streams, out.Idle = used, idle
|
||||
}
|
||||
// Only the indexer's own modules count as read: a manifest the mesh provides itself must not stand in for one missing.
|
||||
cov := Coverage{Indexed: -1}
|
||||
for _, g := range out.Unread {
|
||||
cov.Unread = append(cov.Unread, catalogue.Unread{Input: g.Input, Why: g.Why})
|
||||
}
|
||||
cov := Coverage{Indexed: -1, Unread: out.Unread}
|
||||
if keys, err := p.Read.Keys(ReadModules); err == nil {
|
||||
cov.Indexed = len(keys)
|
||||
for _, k := range keys {
|
||||
|
||||
@@ -1,945 +0,0 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// **The lease write, a verb of the seat mesh-grants** (novox/hq ADR 0582, issue 774). `lease-write` takes a why and is
|
||||
// refused, naming the check, until every precondition of clause 2 reads green; otherwise the operator is asked once, on
|
||||
// a channel that proves who answered, with the checks shown and their digest bound to the yes (clause 1). On the
|
||||
// warrant the checks are read again, and then, each step read back before the next (clause 3): the controller's service
|
||||
// is stopped on its node and read back stopped; the bus's user list is composed without the controller's write on the
|
||||
// lease (buscompose.Records.LeaseCut) and waited on until the bus node's have says it loaded it; the lease bucket's
|
||||
// expiry is removed through the bus module's tool; the key `holder` names mesh-grants with the operator, the why, the
|
||||
// ask and the time; `lease-handed` is said, the act recorded through mesh-hand-acts.warranted, and `removal-due` is
|
||||
// written in the state `lease` with a day to live. A lease altered after it is said as `lease-altered`, once, and
|
||||
// nothing is written again until the operator asks (clause 4). `lease-hand-back` reverses it for a day (clause 5). The
|
||||
// expiry marker of `removal-due`, and nothing else, starts the day-later removal of the controller's consumer of
|
||||
// DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (clause 6). Every step waits on an event (a warrant, the bus node's
|
||||
// have, a marker), never on a timer.
|
||||
|
||||
// The verbs, events and states of the lease write.
|
||||
const (
|
||||
ActLeaseWrite = catalogue.GrantsVerbLeaseWrite
|
||||
ActLeaseHandBack = catalogue.GrantsVerbLeaseHandBack
|
||||
EventLeaseHanded = catalogue.GrantsEventLeaseHanded
|
||||
// EventLeaseAltered and EventRemovalFailed are the seat's other two events.
|
||||
EventLeaseAltered = catalogue.GrantsEventLeaseAltered
|
||||
EventRemovalFailed = catalogue.GrantsEventRemovalFailed
|
||||
// StateLease is the part's state with a time to live per key, holding KeyRemovalDue alone.
|
||||
StateLease = "lease"
|
||||
KeyRemovalDue = "removal-due"
|
||||
// KeyLease is the lease write's record in StateTakeover: the cut, the write, the removal and every act.
|
||||
KeyLease = "lease"
|
||||
// ToolRemovalDue is the tool that writes removal-due again with a short time to live, the recovery for a marker
|
||||
// missed while the part was down.
|
||||
ToolRemovalDue = "grants_removal_due"
|
||||
)
|
||||
|
||||
// RemovalAfter is how long after the write the controller's consumer and stream are removed (ADR 0582 clause 6).
|
||||
const RemovalAfter = 24 * time.Hour
|
||||
|
||||
// The calls the lease write makes, as the manifest's invokes name them.
|
||||
const (
|
||||
// CallServiceStop, CallServiceStart and CallServiceStatus are the node seat's verbs, asked of the controller's node.
|
||||
CallServiceStop = "seat:" + catalogue.ServiceManagerSeat + ".stop"
|
||||
CallServiceStart = "seat:" + catalogue.ServiceManagerSeat + ".start"
|
||||
CallServiceStatus = "seat:" + catalogue.ServiceManagerSeat + ".status"
|
||||
// CallBucketExpiry sets and restores a bucket's expiry, recording the act itself; CallConsumerRemove and
|
||||
// CallStreamRemove are the removal's two tools.
|
||||
CallBucketExpiry = "nats.nats_bucket_expiry"
|
||||
CallConsumerRemove = "nats.nats_consumer_remove"
|
||||
CallStreamRemove = "nats.nats_stream_remove"
|
||||
// CallRecordHandAct records the removals as hand acts (ADR 0582 clause 6, ADR 0565 §11).
|
||||
CallRecordHandAct = "seat:" + handacts.Seat + ".hand-act"
|
||||
// The checks' calls (clause 2).
|
||||
CallNodeTrusts = "mesh-delivery-view.node_trusts"
|
||||
CallConfigCompare = "seat:mesh-config.compare"
|
||||
CallUpgrade = "seat:" + catalogue.WantsSeat + ".upgrade"
|
||||
CallVaultStatus = "mesh-vault.vault_status"
|
||||
CallIssue = "seat:issue-tracker.get"
|
||||
)
|
||||
|
||||
// LeaseCalls are every call the lease write makes.
|
||||
var LeaseCalls = []string{CallServiceStop, CallServiceStart, CallServiceStatus, CallBucketExpiry, CallConsumerRemove,
|
||||
CallStreamRemove, CallRecordHandAct, CallNodeTrusts, CallConfigCompare, CallUpgrade, CallVaultStatus, CallIssue}
|
||||
|
||||
// ControllerUnit is the controller's service on its node.
|
||||
const ControllerUnit = Controller + ".service"
|
||||
|
||||
// LeaseExpiry is the lease bucket's expiry a hand-back restores (buscompose.LeaseTTL).
|
||||
var LeaseExpiry = buscompose.LeaseTTL
|
||||
|
||||
// RemovedTools are the bus module's tools the lease write and its removal need: the check reads them in its manifest.
|
||||
var RemovedTools = []string{"nats_bucket_expiry", "nats_consumer_remove", "nats_stream_remove"}
|
||||
|
||||
// EarlierSteps are the issues of design 47's earlier steps the write waits on (ADR 0582 clause 2: issues 1952-1955 delivered).
|
||||
var EarlierSteps = []uint64{1952, 1953, 1954, 1955}
|
||||
|
||||
// LeaseBus is the process's hand on the controller's lease: the key `holder` put and deleted, and the bucket's expiry
|
||||
// read. Nil in the tools, which ask the process.
|
||||
type LeaseBus interface {
|
||||
PutHolder(value []byte) error
|
||||
DeleteHolder() error
|
||||
Expiry() (time.Duration, error)
|
||||
}
|
||||
|
||||
// TTLStore is a Store that writes a key with its own time to live (the state lease).
|
||||
type TTLStore interface {
|
||||
PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error)
|
||||
}
|
||||
|
||||
// SeatSayer says one of the seat's events, `mesh.seat.mesh-grants.event.<event>`.
|
||||
type SeatSayer func(event string, body any) error
|
||||
|
||||
// Check is one precondition as read: its name, whether it reads green, and what it says.
|
||||
type Check struct {
|
||||
Name string `json:"name"`
|
||||
Green bool `json:"green"`
|
||||
Says string `json:"says"`
|
||||
}
|
||||
|
||||
// checksDigest is the digest of the checks as shown: what the yes binds.
|
||||
func checksDigest(cs []Check) string {
|
||||
raw, _ := json.Marshal(cs)
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// LeaseAct is one lease-write or lease-hand-back: asked, answered and each step it reached.
|
||||
type LeaseAct struct {
|
||||
Act string `json:"act"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
Ask string `json:"ask"`
|
||||
// Checks are the preconditions as the ask showed them, and Digest their digest the yes binds.
|
||||
Checks []Check `json:"checks,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Raised bool `json:"raised,omitempty"`
|
||||
// Step is where it stands: asked; then on the warrant stopped, cut, loaded, expiry, written, done for a write and
|
||||
// restored, uncut, done for a hand-back; or declined, expired, refused, failed.
|
||||
Step string `json:"step"`
|
||||
Said string `json:"said,omitempty"`
|
||||
// List is the digest of the list composed at the cut (or without it, for a hand-back), awaited loaded.
|
||||
List string `json:"list,omitempty"`
|
||||
Done time.Time `json:"done,omitzero"`
|
||||
}
|
||||
|
||||
// The steps an act reaches.
|
||||
const (
|
||||
StepAsked = "asked"
|
||||
StepStopped = "stopped"
|
||||
StepCut = "cut"
|
||||
StepWritten = "written"
|
||||
StepRestored = "restored"
|
||||
StepUncut = "uncut"
|
||||
StepDone = "done"
|
||||
StepDeclined = "declined"
|
||||
StepExpired = "expired"
|
||||
StepRefused = "refused"
|
||||
StepFailed = "failed"
|
||||
)
|
||||
|
||||
// LeaseRecord is the key `lease` of StateTakeover.
|
||||
type LeaseRecord struct {
|
||||
V int `json:"v"`
|
||||
// Cut composes the list with buscompose.Records.LeaseCut: from the write's second step until a hand-back's.
|
||||
Cut bool `json:"cut"`
|
||||
// Node is the controller's node, from the lease's holder at the write.
|
||||
Node string `json:"node,omitempty"`
|
||||
// Handed is when the key holder was written, zero when not (or handed back); Ask the write's ask.
|
||||
Handed time.Time `json:"handed,omitzero"`
|
||||
Ask string `json:"ask,omitempty"`
|
||||
// Removed is when the day-later removal was done; zero until then.
|
||||
Removed time.Time `json:"removed,omitzero"`
|
||||
// Altered is what lease-altered said, empty while the lease reads as written.
|
||||
Altered string `json:"altered,omitempty"`
|
||||
// Open is the act asked or under way.
|
||||
Open *LeaseAct `json:"open,omitempty"`
|
||||
// Acts are the acts ended, newest last.
|
||||
Acts []LeaseAct `json:"acts"`
|
||||
}
|
||||
|
||||
func (p *Part) readLease() (LeaseRecord, error) {
|
||||
r := LeaseRecord{V: 1, Acts: []LeaseAct{}}
|
||||
raw, _, found, err := p.Store.Get(StateTakeover, KeyLease)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("the lease write's record cannot be read: %w", err)
|
||||
}
|
||||
if found {
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return r, fmt.Errorf("the lease write's record does not read: %w", err)
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (p *Part) writeLease(r LeaseRecord) error {
|
||||
if len(r.Acts) > keptActs {
|
||||
r.Acts = r.Acts[len(r.Acts)-keptActs:]
|
||||
}
|
||||
if r.Acts == nil {
|
||||
r.Acts = []LeaseAct{}
|
||||
}
|
||||
data, _ := json.Marshal(r)
|
||||
if _, err := p.Store.Put(StateTakeover, KeyLease, data); err != nil {
|
||||
return fmt.Errorf("the lease write's record cannot be written: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// leaseCut says whether the list is composed with the cut.
|
||||
func (p *Part) leaseCut() bool {
|
||||
r, err := p.readLease()
|
||||
return err == nil && r.Cut
|
||||
}
|
||||
|
||||
// leaseHanded says whether the key holder names mesh-grants by the part's own write.
|
||||
func (p *Part) leaseHanded() bool {
|
||||
r, err := p.readLease()
|
||||
return err == nil && !r.Handed.IsZero()
|
||||
}
|
||||
|
||||
// end closes the open act with its step and why.
|
||||
func (r *LeaseRecord) end(step, said string, at time.Time) {
|
||||
if r.Open == nil {
|
||||
return
|
||||
}
|
||||
a := *r.Open
|
||||
a.Step, a.Said, a.Done = step, said, at
|
||||
r.Acts = append(r.Acts, a)
|
||||
r.Open = nil
|
||||
}
|
||||
|
||||
// LeaseCheckNames are the checks LeaseChecks reads, in its order.
|
||||
var LeaseCheckNames = []string{"lease-check", "earlier-steps", "node-trusts", "config-compare", "upgrade-compare",
|
||||
"keeper-take-over", "vault-minting", "bus-tools"}
|
||||
|
||||
// LeaseChecks reads every precondition of the write (ADR 0582 clause 2), each named, in one order.
|
||||
func (p *Part) LeaseChecks() []Check {
|
||||
var out []Check
|
||||
add := func(name string, green bool, says string) {
|
||||
out = append(out, Check{Name: name, Green: green, Says: says})
|
||||
}
|
||||
// The lease check at the write, with the three kept to it alone (ADR 0571 §13).
|
||||
if raw, err := p.LeaseCheck(); err != nil {
|
||||
add("lease-check", false, "the lease check could not run: "+err.Error())
|
||||
} else if a, ok := raw.(LeaseAnswer); !ok {
|
||||
add("lease-check", false, "the lease check answered nothing it reads")
|
||||
} else {
|
||||
add("lease-check", a.Check.Passes, strings.Join(a.Check.Reasons, "; "))
|
||||
}
|
||||
// Design 47's earlier steps delivered.
|
||||
var open []string
|
||||
for _, n := range EarlierSteps {
|
||||
raw, err := p.Call.Call(CallIssue, map[string]any{"number": n, "brief": true})
|
||||
var is struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
open = append(open, fmt.Sprintf("issue %d cannot be read (%v)", n, err))
|
||||
case json.Unmarshal(unwrap(raw), &is) != nil || is.Status != "resolved":
|
||||
open = append(open, fmt.Sprintf("issue %d is %q, not resolved", n, is.Status))
|
||||
}
|
||||
}
|
||||
add("earlier-steps", len(open) == 0, said(open, "issues 1952 to 1955 are resolved"))
|
||||
// Every node trusts the vault's key.
|
||||
if raw, err := p.Call.Call(CallNodeTrusts, map[string]any{}); err != nil {
|
||||
add("node-trusts", false, "node_trusts cannot be read: "+err.Error())
|
||||
} else {
|
||||
var t struct {
|
||||
Every bool `json:"every"`
|
||||
NotYet []string `json:"not_yet"`
|
||||
}
|
||||
_ = json.Unmarshal(unwrap(raw), &t)
|
||||
add("node-trusts", t.Every, said(prefixed("not yet: ", t.NotYet), "every node's record trusts the vault's key"))
|
||||
}
|
||||
// mesh-config's compare empty and its take-over done.
|
||||
if raw, err := p.Call.Call(CallConfigCompare, map[string]any{}); err != nil {
|
||||
add("config-compare", false, "mesh-config's compare cannot be read: "+err.Error())
|
||||
} else {
|
||||
var c struct {
|
||||
Source string `json:"source"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
_ = json.Unmarshal(unwrap(raw), &c)
|
||||
add("config-compare", c.Count == 0 && c.Source == "indexer",
|
||||
fmt.Sprintf("mesh-config resolves from %q with %d differences", c.Source, c.Count))
|
||||
}
|
||||
// The upgrade compare empty.
|
||||
if raw, err := p.Call.Call(CallUpgrade, map[string]any{"compare": "true"}); err != nil {
|
||||
add("upgrade-compare", false, "mesh-wants' upgrade compare cannot be read: "+err.Error())
|
||||
} else {
|
||||
var u struct {
|
||||
Empty bool `json:"empty"`
|
||||
Splits []json.RawMessage `json:"splits"`
|
||||
}
|
||||
_ = json.Unmarshal(unwrap(raw), &u)
|
||||
add("upgrade-compare", u.Empty, fmt.Sprintf("%d policy splits", len(u.Splits)))
|
||||
}
|
||||
// The condition keeper's take-over done.
|
||||
raw, found, err := p.Read.Get(ReadTakeover, health.TakeoverKey)
|
||||
t, terr := health.DecodeTakeover(raw, found)
|
||||
switch {
|
||||
case err != nil:
|
||||
add("keeper-take-over", false, "the keeper's take-over cannot be read: "+err.Error())
|
||||
case terr != nil:
|
||||
add("keeper-take-over", false, "the keeper's take-over does not read: "+terr.Error())
|
||||
default:
|
||||
add("keeper-take-over", t.Follow != health.FollowController, "conditions follow "+t.Follow)
|
||||
}
|
||||
// The vault minting with nothing held back.
|
||||
if raw, err := p.Call.Call(CallVaultStatus, map[string]any{}); err != nil {
|
||||
add("vault-minting", false, "vault_status cannot be read: "+err.Error())
|
||||
} else {
|
||||
var v struct {
|
||||
Ready bool `json:"ready"`
|
||||
Minting struct {
|
||||
On bool `json:"on"`
|
||||
HeldBack map[string]string `json:"held_back"`
|
||||
} `json:"minting"`
|
||||
}
|
||||
_ = json.Unmarshal(unwrap(raw), &v)
|
||||
add("vault-minting", v.Ready && v.Minting.On && len(v.Minting.HeldBack) == 0,
|
||||
fmt.Sprintf("ready %v, minting %v, %d held back", v.Ready, v.Minting.On, len(v.Minting.HeldBack)))
|
||||
}
|
||||
// The bus module's tools for the expiry and the removal live: its manifest at the indexer's trunk lists them.
|
||||
in := ReadInputs(p.Read, &p.cache)
|
||||
var missing []string
|
||||
for _, tool := range RemovedTools {
|
||||
if !slices.Contains(in.Manifests[BusModule].Tools, tool) {
|
||||
missing = append(missing, tool)
|
||||
}
|
||||
}
|
||||
add("bus-tools", len(missing) == 0, said(prefixed("the bus module lists no ", missing), "the bus module lists its three tools"))
|
||||
return out
|
||||
}
|
||||
|
||||
func said(problems []string, ok string) string {
|
||||
if len(problems) == 0 {
|
||||
return ok
|
||||
}
|
||||
return strings.Join(problems, "; ")
|
||||
}
|
||||
|
||||
func prefixed(prefix string, items []string) []string {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []string{prefix + strings.Join(items, ", ")}
|
||||
}
|
||||
|
||||
// notGreen is every check not green, `<name>: <says>`.
|
||||
func notGreen(cs []Check) []string {
|
||||
var out []string
|
||||
for _, c := range cs {
|
||||
if !c.Green {
|
||||
out = append(out, c.Name+": "+c.Says)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changedChecks names each check whose reading differs from the one shown.
|
||||
func changedChecks(shown, now []Check) []string {
|
||||
was := map[string]Check{}
|
||||
for _, c := range shown {
|
||||
was[c.Name] = c
|
||||
}
|
||||
var out []string
|
||||
for _, c := range now {
|
||||
if w, ok := was[c.Name]; !ok || w != c {
|
||||
out = append(out, c.Name+" (now: "+c.Says+")")
|
||||
}
|
||||
delete(was, c.Name)
|
||||
}
|
||||
for name := range was {
|
||||
out = append(out, name+" (no longer read)")
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// LeaseWrite is the verb lease-write: refused, naming each check not green, or asked of the operator by the part's process.
|
||||
func (p *Part) LeaseWrite(caller, why string) (any, error) {
|
||||
return p.leaseRequest(ActLeaseWrite, caller, why)
|
||||
}
|
||||
|
||||
// LeaseHandBack is the verb lease-hand-back.
|
||||
func (p *Part) LeaseHandBack(caller, why string) (any, error) {
|
||||
return p.leaseRequest(ActLeaseHandBack, caller, why)
|
||||
}
|
||||
|
||||
func (p *Part) leaseRequest(act, caller, why string) (any, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("%s needs a why, shown to the operator and kept", act)
|
||||
}
|
||||
r, err := p.readLease()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Open != nil {
|
||||
return nil, fmt.Errorf("%s is already under way: the ask %s stands at %s", r.Open.Act, r.Open.Ask, r.Open.Step)
|
||||
}
|
||||
var checks []Check
|
||||
switch act {
|
||||
case ActLeaseWrite:
|
||||
if !r.Handed.IsZero() && r.Altered == "" {
|
||||
return nil, errors.New("the lease is handed to mesh-grants already: nothing to write")
|
||||
}
|
||||
checks = p.LeaseChecks()
|
||||
if bad := notGreen(checks); len(bad) > 0 {
|
||||
return nil, fmt.Errorf("lease-write is refused until every check reads green (ADR 0582 clause 2): %s", strings.Join(bad, "; "))
|
||||
}
|
||||
case ActLeaseHandBack:
|
||||
if refused := handBackRefused(r); refused != "" {
|
||||
return nil, errors.New(refused)
|
||||
}
|
||||
}
|
||||
a := LeaseAct{Act: act, By: caller, Why: why, At: p.now(), Checks: checks, Digest: checksDigest(checks), Step: StepAsked}
|
||||
a.Ask = act + "-" + p.now().Format("20060102t150405") + "-" + p.nonce()
|
||||
r.Open = &a
|
||||
if err := p.writeLease(r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, _ = p.AskCompose(caller)
|
||||
return map[string]any{"asked": a.Ask, "checks": checks, "says": "the part's process asks the operator on their channel; " +
|
||||
"only the operator's yes runs it, after the checks are read again. grants_lease_check shows the lease, and " +
|
||||
"the record is the key lease of the state takeover"}, nil
|
||||
}
|
||||
|
||||
// handBackRefused is why a hand-back cannot be asked, or empty.
|
||||
func handBackRefused(r LeaseRecord) string {
|
||||
switch {
|
||||
case !r.Removed.IsZero():
|
||||
return "lease-hand-back is refused: the day-later removal ran, so the controller would start without its " +
|
||||
"consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS (ADR 0582 clause 5)"
|
||||
case r.Handed.IsZero() && !r.Cut:
|
||||
return "lease-hand-back is refused: the lease is not handed to mesh-grants, so there is nothing to hand back"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// EventBucketExpirySet is the bus module's event of a bucket's expiry set, which the part consumes: a lease bucket's
|
||||
// expiry changed is heard, never polled.
|
||||
const EventBucketExpirySet = BusModule + ".bucket-expiry-set"
|
||||
|
||||
// IsBucketExpirySet says an event heard is the bus module's bucket-expiry-set.
|
||||
func IsBucketExpirySet(event string) bool {
|
||||
return event == EventBucketExpirySet || event == "mesh.mod."+BusModule+".event.bucket-expiry-set"
|
||||
}
|
||||
|
||||
// isLeaseAsk says an ask is the lease write's.
|
||||
func isLeaseAsk(id string) bool {
|
||||
return strings.HasPrefix(id, ActLeaseWrite+"-") || strings.HasPrefix(id, ActLeaseHandBack+"-")
|
||||
}
|
||||
|
||||
// leaseAskOf is the ask of an act, in the words the operator reads: what it does, the checks it showed, and the one
|
||||
// option whose yes binds the act, the holder, the ask and the checks' digest (ADR 0582 clause 1).
|
||||
func (p *Part) leaseAskOf(a LeaseAct) (asks.Ask, error) {
|
||||
act := asks.Act{"verb": a.Act, "module": Module, "ask": a.Ask, "holder": Module, "checks": a.Digest}
|
||||
bind, err := asks.ActDigest(act)
|
||||
if err != nil {
|
||||
return asks.Ask{}, err
|
||||
}
|
||||
var lines []string
|
||||
for _, c := range a.Checks {
|
||||
lines = append(lines, "✓ "+c.Name+": "+c.Says)
|
||||
}
|
||||
headline, does, stays := "Hand the controller's lease to mesh-grants?",
|
||||
"The controller is stopped and the lease is handed to mesh-grants for good; hand-back stays open for a day.",
|
||||
"The controller keeps its lease."
|
||||
expl := "Every check of the lease write reads green. Your yes stops the controller, takes from it the right to write " +
|
||||
"its lease, removes the lease's expiry and names mesh-grants as its holder. A controller started later fails on the lease."
|
||||
if a.Act == ActLeaseHandBack {
|
||||
headline, does, stays = "Give the controller its lease back?",
|
||||
"The lease's expiry is restored, mesh-grants lets go of it, the controller gets its rights back and is started.",
|
||||
"The lease stays with mesh-grants."
|
||||
expl = "The lease was handed to mesh-grants less than a day ago. Your yes gives it back and starts the controller."
|
||||
}
|
||||
return asks.Ask{ID: a.Ask, Headline: headline, Explanation: expl, Who: asks.Operator,
|
||||
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
|
||||
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
|
||||
Expires: a.At.Add(asks.ApproveLasts - time.Minute), OnExpiry: stays + " Nothing changes.", About: "mesh-grants-" + a.Act,
|
||||
Details: "Asked by " + a.By + ": " + a.Why + "\n" + strings.Join(lines, "\n")}, nil
|
||||
}
|
||||
|
||||
// raiseLease asks the operator the open act once; closes one past its expiry; catches up a warrant the process missed.
|
||||
func (p *Part) raiseLease() error {
|
||||
if p.Asker == nil {
|
||||
return nil
|
||||
}
|
||||
r, err := p.readLease()
|
||||
if err != nil || r.Open == nil || r.Open.Step != StepAsked {
|
||||
return err
|
||||
}
|
||||
a := r.Open
|
||||
if !a.Raised {
|
||||
ask, err := p.leaseAskOf(*a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.Asker.Ask(ask); err != nil {
|
||||
if p.Forget != nil {
|
||||
_ = p.Forget(a.Ask)
|
||||
}
|
||||
_ = p.Asker.Cancel(a.Ask)
|
||||
r.end(StepRefused, "the operator could not be asked: "+err.Error(), p.now())
|
||||
_ = p.writeLease(r)
|
||||
return fmt.Errorf("the operator could not be asked the %s: %v", a.Act, err)
|
||||
}
|
||||
a.Raised = true
|
||||
return p.writeLease(r)
|
||||
}
|
||||
_, w, err := p.Asker.Where(a.Ask)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the router's record of the ask %s cannot be read: %v", a.Ask, err)
|
||||
}
|
||||
if w != nil {
|
||||
h, herr := p.Asker.Take(*w, p.Warranted)
|
||||
if h == "" && herr != nil {
|
||||
return fmt.Errorf("the operator's answer to %s could not be taken yet: %v", a.Ask, herr)
|
||||
}
|
||||
p.Said(h, *w, herr)
|
||||
return nil
|
||||
}
|
||||
if p.now().After(a.At.Add(asks.ApproveLasts + askMargin)) {
|
||||
r.end(StepExpired, "the ask expired: nothing was written", p.now())
|
||||
return p.writeLease(r)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// leaseWarranted is the operator's answer to a lease act: a no or a refusal writes nothing; a yes runs the act.
|
||||
func (p *Part) leaseWarranted(o asks.Option, w asks.Warrant) error {
|
||||
r, err := p.readLease()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.Open == nil || r.Open.Ask != w.Ask || r.Open.Step != StepAsked {
|
||||
return fmt.Errorf("the warrant answers the ask %s, which is not the lease act asked: nothing is written", w.Ask)
|
||||
}
|
||||
if o.ID != "yes" {
|
||||
r.end(StepDeclined, "the operator said no: nothing was written", p.now())
|
||||
return p.writeLease(r)
|
||||
}
|
||||
a := r.Open
|
||||
if a.Act == ActLeaseWrite {
|
||||
// Read again on the warrant (ADR 0582 clauses 1, 2): the yes binds the checks shown, so any check that reads
|
||||
// otherwise now (one no longer green among them) refuses it, named.
|
||||
if now := p.LeaseChecks(); checksDigest(now) != a.Digest {
|
||||
r.end(StepRefused, "refused on the warrant, the checks changed since the ask: "+strings.Join(changedChecks(a.Checks, now), "; "), p.now())
|
||||
p.sayLeaseRefused(a.Act, r.Acts[len(r.Acts)-1].Said)
|
||||
return p.writeLease(r)
|
||||
}
|
||||
} else if refused := handBackRefused(r); refused != "" {
|
||||
r.end(StepRefused, refused, p.now())
|
||||
return p.writeLease(r)
|
||||
}
|
||||
a.Step = "warranted"
|
||||
if err := p.writeLease(r); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.advanceLease()
|
||||
}
|
||||
|
||||
// sayLeaseRefused tells the operator a yes was refused, as the take-over's refusal is said.
|
||||
func (p *Part) sayLeaseRefused(act, why string) {
|
||||
if p.Say == nil {
|
||||
return
|
||||
}
|
||||
_ = p.Say(asks.Condition{Key: "mesh-grants." + act + "-refused", Severity: asks.SeverityWarning,
|
||||
Subject: asks.Scope{Scope: "module", ID: Module}, State: asks.StateRaised, Summary: act + " refused: " + why,
|
||||
Headline: "The lease write was refused", Needs: "a look at why",
|
||||
Explanation: asks.NeedsYou + "a look at why. You said yes and it was refused: " + why, Resolved: "the act is asked again and goes through"})
|
||||
}
|
||||
|
||||
// advanceLease takes the open act as far as it can now: each step read back before the next, and a step that waits (the
|
||||
// list loaded) left for the pass that hears the bus node's have. A step that fails ends the act there, saying why, and
|
||||
// writes nothing more.
|
||||
func (p *Part) advanceLease() error {
|
||||
r, err := p.readLease()
|
||||
if err != nil || r.Open == nil || r.Open.Step == StepAsked {
|
||||
return err
|
||||
}
|
||||
a := r.Open
|
||||
fail := func(why string) error {
|
||||
r.end(StepFailed, why, p.now())
|
||||
p.sayLeaseRefused(a.Act, why)
|
||||
return p.writeLease(r)
|
||||
}
|
||||
if a.Act == ActLeaseHandBack {
|
||||
return p.advanceHandBack(&r, fail)
|
||||
}
|
||||
switch a.Step {
|
||||
case "warranted":
|
||||
// 1. Stop the controller's service on its node, read back stopped.
|
||||
node, why := p.controllerNode()
|
||||
if node == "" {
|
||||
return fail(why)
|
||||
}
|
||||
if stopped, why := p.service(CallServiceStop, node); !stopped {
|
||||
return fail("the controller's service on " + node + " did not read back stopped (" + why + "): nothing more is written")
|
||||
}
|
||||
r.Node, a.Step = node, StepStopped
|
||||
fallthrough
|
||||
case StepStopped:
|
||||
// 2. Compose the list without the controller's write on the lease.
|
||||
r.Cut, a.Step = true, StepCut
|
||||
if err := p.writeLease(r); err != nil {
|
||||
return err
|
||||
}
|
||||
res := p.composeQuiet()
|
||||
g, _, _ := p.Sealed()
|
||||
if res.Sealed == nil || g.Digest == "" {
|
||||
return fail("the list composed with the cut could not be written sealed: " + strings.Join(res.Errors, "; "))
|
||||
}
|
||||
if r, err = p.readLease(); err != nil || r.Open == nil {
|
||||
return err
|
||||
}
|
||||
a = r.Open
|
||||
a.List = g.Digest
|
||||
if err := p.writeLease(r); err != nil {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case StepCut:
|
||||
// 3. Wait until the bus node's have says it loaded that list.
|
||||
if !p.loaded(a.List) {
|
||||
return nil
|
||||
}
|
||||
// 4. Remove the lease bucket's expiry.
|
||||
if ok, why := p.setExpiry("none", *a); !ok {
|
||||
return fail("the lease's expiry was not removed (" + why + "): the holder is not written")
|
||||
}
|
||||
// 5. Create the key holder naming mesh-grants.
|
||||
h := buscompose.LeaseHolder{HandedTo: Module, HandedBy: "operator", Why: a.Why, Ask: a.Ask, At: p.now(), Host: r.Node}
|
||||
raw, _ := json.Marshal(h)
|
||||
if p.Lease == nil {
|
||||
return fail("the part holds no hand on the lease here")
|
||||
}
|
||||
if err := p.Lease.PutHolder(raw); err != nil {
|
||||
return fail("the key holder could not be written: " + err.Error())
|
||||
}
|
||||
r.Handed, r.Ask, r.Altered, a.Step = p.now(), a.Ask, "", StepWritten
|
||||
if err := p.writeLease(r); err != nil {
|
||||
return err
|
||||
}
|
||||
// 6. Say lease-handed, record the act, and write removal-due with a day to live.
|
||||
p.saySeat(EventLeaseHanded, map[string]any{"holder": Module, "ask": a.Ask, "why": a.Why, "node": r.Node, "at": r.Handed})
|
||||
p.recordWarrant(a.Ask)
|
||||
if err := p.putRemovalDue(RemovalAfter); err != nil {
|
||||
r.end(StepDone, "the lease is handed; removal-due could not be written ("+err.Error()+"): grants_removal_due writes it", p.now())
|
||||
} else {
|
||||
r.end(StepDone, "the lease is handed to mesh-grants; the removal is due in a day", p.now())
|
||||
}
|
||||
return p.writeLease(r)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// advanceHandBack restores the expiry, deletes holder and removal-due, composes without the cut, and once that list is
|
||||
// loaded starts the controller (ADR 0582 clause 5).
|
||||
func (p *Part) advanceHandBack(r *LeaseRecord, fail func(string) error) error {
|
||||
a := r.Open
|
||||
switch a.Step {
|
||||
case "warranted":
|
||||
if ok, why := p.setExpiry(LeaseExpiry.String(), *a); !ok {
|
||||
return fail("the lease's expiry was not restored (" + why + "): the lease stays with mesh-grants")
|
||||
}
|
||||
if p.Lease == nil {
|
||||
return fail("the part holds no hand on the lease here")
|
||||
}
|
||||
// The record first: the delete of removal-due is then heard as no removal (Handed is zero).
|
||||
r.Handed, a.Step = time.Time{}, StepRestored
|
||||
if err := p.writeLease(*r); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.Lease.DeleteHolder(); err != nil {
|
||||
return fail("the key holder could not be deleted: " + err.Error())
|
||||
}
|
||||
if err := p.Store.Delete(StateLease, KeyRemovalDue); err != nil {
|
||||
return fail("removal-due could not be deleted: " + err.Error())
|
||||
}
|
||||
r.Cut, a.Step = false, StepUncut
|
||||
if err := p.writeLease(*r); err != nil {
|
||||
return err
|
||||
}
|
||||
p.composeQuiet()
|
||||
g, _, _ := p.Sealed()
|
||||
fresh, err := p.readLease()
|
||||
if err != nil || fresh.Open == nil {
|
||||
return err
|
||||
}
|
||||
*r = fresh
|
||||
a = r.Open
|
||||
a.List = g.Digest
|
||||
if err := p.writeLease(*r); err != nil {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case StepUncut:
|
||||
if !p.loaded(a.List) {
|
||||
return nil
|
||||
}
|
||||
if started, why := p.service(CallServiceStart, r.Node); !started {
|
||||
return fail("the controller's service on " + r.Node + " did not read back started (" + why + ")")
|
||||
}
|
||||
p.recordWarrant(a.Ask)
|
||||
r.end(StepDone, "the lease is given back; the controller is started", p.now())
|
||||
return p.writeLease(*r)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// controllerNode is the node the controller holds its lease on, from the lease's holder.
|
||||
func (p *Part) controllerNode() (string, string) {
|
||||
raw, found, err := p.Read.Get(ReadLease, LeaseKey)
|
||||
if err != nil || !found {
|
||||
return "", "the lease cannot be read, so the controller's node is not known: nothing is stopped"
|
||||
}
|
||||
var h buscompose.LeaseHolder
|
||||
if json.Unmarshal(raw, &h) != nil || h.Host == "" || h.HandedTo != "" {
|
||||
return "", "the lease names no controller's host, so the controller's node is not known: nothing is stopped"
|
||||
}
|
||||
return h.Host, ""
|
||||
}
|
||||
|
||||
// service asks the node seat's verb of the controller's unit and reads it back with status: stopped for a stop, started
|
||||
// for a start.
|
||||
func (p *Part) service(verb, node string) (bool, string) {
|
||||
unit := map[string]any{"unit": ControllerUnit}
|
||||
if _, err := p.Call.Call(verb+"@"+node, unit); err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
raw, err := p.Call.Call(CallServiceStatus+"@"+node, unit)
|
||||
if err != nil {
|
||||
return false, "its status cannot be read: " + err.Error()
|
||||
}
|
||||
var st struct {
|
||||
Active string `json:"ActiveState"`
|
||||
}
|
||||
_ = json.Unmarshal(unwrap(raw), &st)
|
||||
if verb == CallServiceStart {
|
||||
return st.Active == "active", "it reads " + st.Active
|
||||
}
|
||||
return st.Active == "inactive" || st.Active == "failed", "it reads " + st.Active
|
||||
}
|
||||
|
||||
// loaded says the bus node's have names the list of that digest loaded from the grants part's key.
|
||||
func (p *Part) loaded(digest string) bool {
|
||||
l := ReadInputs(p.Read, &p.cache).Loaded
|
||||
return digest != "" && l != nil && l.Source == delivery.GrantsUserListSource && l.Digest == digest
|
||||
}
|
||||
|
||||
// setExpiry asks the bus module's tool to set the lease bucket's expiry, and reads it back from the bucket.
|
||||
func (p *Part) setExpiry(maxAge string, a LeaseAct) (bool, string) {
|
||||
raw, err := p.Call.Call(CallBucketExpiry, map[string]any{"bucket": buscompose.LeaseBucket, "max_age": maxAge,
|
||||
"why": a.Act + ": " + a.Why, "warrant": a.Ask})
|
||||
if err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
var set struct {
|
||||
Now string `json:"now"`
|
||||
}
|
||||
_ = json.Unmarshal(unwrap(raw), &set)
|
||||
if set.Now != maxAge {
|
||||
return false, "the tool answered " + set.Now
|
||||
}
|
||||
if p.Lease != nil {
|
||||
got, err := p.Lease.Expiry()
|
||||
want, _ := time.ParseDuration(strings.Replace(maxAge, "none", "0s", 1))
|
||||
if err != nil || got != want {
|
||||
return false, fmt.Sprintf("the bucket reads back %s (%v)", got, err)
|
||||
}
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// putRemovalDue writes removal-due with its time to live.
|
||||
func (p *Part) putRemovalDue(ttl time.Duration) error {
|
||||
s, ok := p.Store.(TTLStore)
|
||||
if !ok {
|
||||
return errors.New("the part's store writes no time to live here")
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"due": p.now().Add(ttl), "written": p.now()})
|
||||
_, err := s.PutTTL(StateLease, KeyRemovalDue, raw, ttl)
|
||||
return err
|
||||
}
|
||||
|
||||
// saySeat says one of the seat's events; a failure is logged, never the end of the act.
|
||||
func (p *Part) saySeat(event string, body any) {
|
||||
if p.SaySeat == nil {
|
||||
return
|
||||
}
|
||||
if err := p.SaySeat(event, body); err != nil && p.Logf != nil {
|
||||
p.Logf("the event %s could not be said: %v", event, err)
|
||||
}
|
||||
}
|
||||
|
||||
// leaseStep is the pass's part in the lease write: the open act asked or advanced, the lease watched once handed, and
|
||||
// the removal run when its marker was heard.
|
||||
func (p *Part) leaseStep(res *Result) {
|
||||
for _, step := range []func() error{p.raiseLease, p.advanceLease, p.watchLease} {
|
||||
if err := step(); err != nil {
|
||||
res.Errors = append(res.Errors, err.Error())
|
||||
}
|
||||
}
|
||||
if p.removalHeard.Swap(false) {
|
||||
if err := p.removal(); err != nil {
|
||||
res.Errors = append(res.Errors, err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RemovalDueHeard is the follower's: the marker of removal-due was heard (its delete, by expiry or by a hand-back).
|
||||
func (p *Part) RemovalDueHeard() { p.removalHeard.Store(true) }
|
||||
|
||||
// watchLease says lease-altered once when the key or the bucket's expiry differs from what the part wrote (ADR 0582
|
||||
// clause 4); a hand-back under way is the part's own change and is not watched.
|
||||
func (p *Part) watchLease() error {
|
||||
r, err := p.readLease()
|
||||
if err != nil || r.Handed.IsZero() || r.Altered != "" || (r.Open != nil && r.Open.Act == ActLeaseHandBack && r.Open.Step != StepAsked) {
|
||||
return err
|
||||
}
|
||||
var what []string
|
||||
raw, found, err := p.Read.Get(ReadLease, LeaseKey)
|
||||
var h buscompose.LeaseHolder
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil // not read is no change: the next pass reads it again
|
||||
case !found:
|
||||
what = append(what, "the key holder is gone")
|
||||
case json.Unmarshal(raw, &h) != nil || h.HandedTo != Module || h.Ask != r.Ask:
|
||||
what = append(what, "the key holder names "+firstOf(h.HandedTo, h.Instance, "something else")+", not mesh-grants by "+r.Ask)
|
||||
}
|
||||
if p.Lease != nil {
|
||||
if d, err := p.Lease.Expiry(); err == nil && d != 0 {
|
||||
what = append(what, "the lease's bucket expires after "+d.String())
|
||||
}
|
||||
}
|
||||
if len(what) == 0 {
|
||||
return nil
|
||||
}
|
||||
r.Altered = strings.Join(what, "; ")
|
||||
p.saySeat(EventLeaseAltered, map[string]any{"altered": r.Altered, "ask": r.Ask, "at": p.now()})
|
||||
return p.writeLease(r)
|
||||
}
|
||||
|
||||
func firstOf(s ...string) string {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return x
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// removal is the day-later removal, started by the marker of removal-due alone (ADR 0582 clause 6): when the lease still
|
||||
// names mesh-grants, the controller's consumer of DEAD_LETTER_NOTICES and the stream ASSIGNMENTS are removed, each read
|
||||
// back by the bus module's tool and recorded as a hand act. A failure is said as removal-failed and removal-due is
|
||||
// written again with a day.
|
||||
func (p *Part) removal() error {
|
||||
r, err := p.readLease()
|
||||
if err != nil || r.Handed.IsZero() || !r.Removed.IsZero() {
|
||||
return err // handed back (a hand-back deletes the key), or done already
|
||||
}
|
||||
if raw, found, err := p.Read.Get(ReadLease, LeaseKey); err != nil || !found {
|
||||
return p.removalFailed(r, "the lease cannot be read, so it is not known to name mesh-grants")
|
||||
} else if away, _ := buscompose.LeaseHandedAway(raw, true, nil); !away {
|
||||
return p.removalFailed(r, "the lease no longer names mesh-grants")
|
||||
}
|
||||
why := "a day after the lease write " + r.Ask + " (ADR 0582 clause 6)"
|
||||
var failed []string
|
||||
if raw, err := p.Call.Call(CallConsumerRemove, map[string]any{"stream": buscompose.DeadLetterNoticesStream,
|
||||
"consumer": buscompose.ControllerName, "why": why}); err != nil && !strings.Contains(err.Error(), "has no consumer") {
|
||||
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES: "+err.Error())
|
||||
} else if err == nil && !removed(raw) {
|
||||
failed = append(failed, "the consumer controller of DEAD_LETTER_NOTICES does not read back removed")
|
||||
} else {
|
||||
p.recordHandAct("removed the controller's consumer of DEAD_LETTER_NOTICES", why)
|
||||
}
|
||||
if raw, err := p.Call.Call(CallStreamRemove, map[string]any{"stream": buscompose.AssignmentsStream, "why": why}); err != nil {
|
||||
failed = append(failed, "the stream ASSIGNMENTS: "+err.Error())
|
||||
} else if !removed(raw) {
|
||||
failed = append(failed, "the stream ASSIGNMENTS does not read back removed")
|
||||
} else {
|
||||
p.recordHandAct("removed the stream ASSIGNMENTS", why)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return p.removalFailed(r, strings.Join(failed, "; "))
|
||||
}
|
||||
r.Removed = p.now()
|
||||
return p.writeLease(r)
|
||||
}
|
||||
|
||||
// removed reads a removal tool's answer: removed now, or not there to remove.
|
||||
func removed(raw json.RawMessage) bool {
|
||||
var a struct {
|
||||
Removed bool `json:"removed"`
|
||||
Present *bool `json:"present"`
|
||||
}
|
||||
_ = json.Unmarshal(unwrap(raw), &a)
|
||||
return a.Removed || (a.Present != nil && !*a.Present)
|
||||
}
|
||||
|
||||
func (p *Part) removalFailed(r LeaseRecord, why string) error {
|
||||
p.saySeat(EventRemovalFailed, map[string]any{"failed": why, "ask": r.Ask, "at": p.now()})
|
||||
if err := p.putRemovalDue(RemovalAfter); err != nil {
|
||||
return fmt.Errorf("the removal failed (%s) and removal-due could not be written again: %v", why, err)
|
||||
}
|
||||
return fmt.Errorf("the day-later removal failed, tried again in a day: %s", why)
|
||||
}
|
||||
|
||||
// recordHandAct records one removal in the hand-act log; said, never fatal.
|
||||
func (p *Part) recordHandAct(what, why string) {
|
||||
if p.Call == nil {
|
||||
return
|
||||
}
|
||||
if _, err := p.Call.Call(CallRecordHandAct, map[string]any{"what": what, "why": why, "cause": "lease-removal"}); err != nil && p.Logf != nil {
|
||||
p.Logf("%s, and the hand act could not be recorded: %v", what, err)
|
||||
}
|
||||
}
|
||||
|
||||
// RemovalDue is the tool grants_removal_due: removal-due written again with a short time to live, so its marker starts
|
||||
// the removal (the recovery for a marker missed while the part was down). Recorded as a hand act.
|
||||
func (p *Part) RemovalDue(caller, why, ttl string) (any, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, errors.New("grants_removal_due needs a why, kept with the hand act")
|
||||
}
|
||||
d, err := time.ParseDuration(ttl)
|
||||
if err != nil || d < time.Second || d > RemovalAfter {
|
||||
return nil, fmt.Errorf("ttl %q is no time to live between 1s and %s", ttl, RemovalAfter)
|
||||
}
|
||||
r, err := p.readLease()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Handed.IsZero() || !r.Removed.IsZero() {
|
||||
return nil, errors.New("no removal is due: the lease is not handed to mesh-grants, or the removal ran")
|
||||
}
|
||||
if err := p.putRemovalDue(d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.recordHandAct("wrote removal-due again with "+d.String()+" to live", why+" (by "+caller+")")
|
||||
return map[string]any{"written": KeyRemovalDue, "ttl": d.String(), "says": "its marker starts the removal when it expires"}, nil
|
||||
}
|
||||
|
||||
// LeaseState answers where the lease write stands, for grants_lease_check.
|
||||
func (p *Part) LeaseState() (LeaseRecord, error) { return p.readLease() }
|
||||
@@ -34,7 +34,7 @@ const (
|
||||
)
|
||||
|
||||
// States is every state the part keeps.
|
||||
var States = []string{StateAccounts, StateAssertions, StateGrants, StateLease, StateMemberships, StateTakeover, StateUserList, StateWithheld}
|
||||
var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList, StateWithheld}
|
||||
|
||||
// The events the part says.
|
||||
const (
|
||||
@@ -170,17 +170,9 @@ type Part struct {
|
||||
// Forget takes away the record the asker kept of an ask it failed to publish; Nonce is a test's id suffix.
|
||||
Forget func(id string) error
|
||||
Nonce func() string
|
||||
// Lease is the process's hand on the controller's lease (leasewrite.go); nil in the tools, which ask the process.
|
||||
Lease LeaseBus
|
||||
// SaySeat says the seat mesh-grants' events (lease-handed, lease-altered, removal-failed): the process's; nil in the
|
||||
// tools.
|
||||
SaySeat SeatSayer
|
||||
// deciding is set while a take-over is decided, or a lease act composes, so the composition it makes asks and
|
||||
// reconciles nothing.
|
||||
// deciding is set while a take-over is decided, so the composition it makes asks and reconciles nothing.
|
||||
deciding atomic.Bool
|
||||
// removalHeard is set when the marker of removal-due was heard, and taken by the next pass (leasewrite.go).
|
||||
removalHeard atomic.Bool
|
||||
Now func() time.Time
|
||||
Now func() time.Time
|
||||
|
||||
cache Manifests
|
||||
mu sync.Mutex
|
||||
@@ -257,8 +249,6 @@ func (p *Part) Compose() Result {
|
||||
// the trunk names, so the list grants a build's subjects before the want moves to it (issue 774).
|
||||
recs, _ := Records(in)
|
||||
listRecs, left := ListRecords(in)
|
||||
// From the lease write's cut, the controller's write on the lease is taken from the list (ADR 0582 clause 3).
|
||||
listRecs.LeaseCut = p.leaseCut()
|
||||
holdings := HoldingsOf(in)
|
||||
res.RootNotFree = WithRootFree(&listRecs, p.Call)
|
||||
recs.RootFree = listRecs.RootFree
|
||||
@@ -337,10 +327,6 @@ func (p *Part) Compose() Result {
|
||||
members = append(members, m.Membership)
|
||||
}
|
||||
seated := holdings.ByMemberships(members...)
|
||||
// From the lease write the memberships are written to the state alone and no longer said on their subject, which
|
||||
// nothing reads any more (novox/hq ADR 0571 clause 10, issue 774).
|
||||
notice, _ := leaseHandedAway(p.Read)
|
||||
notice = !notice
|
||||
for _, key := range sortedKeys(ms) {
|
||||
// The last membership is the state's own (ADR 0571 clause 7), never the assignments stream's, which the bus
|
||||
// module's tools remove at the lease write (research 072 row 33): a last read there would freeze every membership
|
||||
@@ -360,10 +346,8 @@ func (p *Part) Compose() Result {
|
||||
m.Published, now = "not published (tried again at the next event): its own "+StateMemberships+" cannot be written", false
|
||||
} else {
|
||||
res.MembershipsWritten++
|
||||
if notice {
|
||||
if err := p.Publish.PublishMembership(m.Node, m.Module, body); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("the membership of %s is written, its notice on its subject not said: %v", key, err))
|
||||
}
|
||||
if err := p.Publish.PublishMembership(m.Node, m.Module, body); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("the membership of %s is written, its notice on its subject not said: %v", key, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -426,18 +410,10 @@ func (p *Part) Compose() Result {
|
||||
if err := p.Reconcile(); err != nil {
|
||||
res.Errors = append(res.Errors, err.Error())
|
||||
}
|
||||
p.leaseStep(&res)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// composeQuiet composes as a step of an act: asking and reconciling nothing.
|
||||
func (p *Part) composeQuiet() Result {
|
||||
p.deciding.Store(true)
|
||||
defer p.deciding.Store(false)
|
||||
return p.Compose()
|
||||
}
|
||||
|
||||
// Last is the result of the last pass this part made, kept under the lock alone.
|
||||
func (p *Part) Last() Result {
|
||||
p.mu.Lock()
|
||||
|
||||
@@ -49,9 +49,7 @@ func HeardFrom(event string) string {
|
||||
const EventAccountIssued = "mesh-vault." + buscompose.AccountIssuedEvent
|
||||
|
||||
// Consumes is every event the part hears, as its manifest's consumes name them.
|
||||
//
|
||||
// Not the controller's condition events: the keeper's alone (novox/hq ADR 0564 §9, issue 774).
|
||||
var Consumes = []string{EventKeeperRaised, EventKeeperCleared, EventAccountIssued, EventBucketExpirySet}
|
||||
var Consumes = []string{EventConditionRaised, EventConditionCleared, EventKeeperRaised, EventKeeperCleared, EventAccountIssued}
|
||||
|
||||
// IsAccountIssued says an event heard is the vault's account-issued, by its name or its bus subject.
|
||||
func IsAccountIssued(event string) bool {
|
||||
|
||||
@@ -95,8 +95,6 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped
|
||||
cur, _, had := p.Sealed()
|
||||
next := cur
|
||||
next.Follow = followOf(p.SwitchState())
|
||||
// The bus node follows this list alone once the lease is handed (ADR 0571 clause 8, ADR 0582 clause 3).
|
||||
next.LeaseHanded = p.leaseHanded()
|
||||
key := ""
|
||||
if in.Loaded != nil {
|
||||
key = in.Loaded.Key
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -132,15 +131,8 @@ func (p *Part) SwitchState() string {
|
||||
// the refusals of Decide. Asking again while an ask is open is refused.
|
||||
func (p *Part) TakeOver(caller, why string) (any, error) { return p.request(ActTakeOver, caller, why) }
|
||||
|
||||
// HandBack is the tool: it asks the operator to switch the bus node back to the controller's list. Once the lease is
|
||||
// handed to the grants part it is answered as gone: the grants part's list is the only one (ADR 0547 §11).
|
||||
func (p *Part) HandBack(caller, why string) (any, error) {
|
||||
if p.leaseHanded() {
|
||||
return nil, errors.New("gone: the controller's lease is handed to mesh-grants, so the grants part's list is the only " +
|
||||
"one and the user list is not handed back (ADR 0547 §11); lease-hand-back gives the lease back first")
|
||||
}
|
||||
return p.request(ActHandBack, caller, why)
|
||||
}
|
||||
// HandBack is the tool: it asks the operator to switch the bus node back to the controller's list.
|
||||
func (p *Part) HandBack(caller, why string) (any, error) { return p.request(ActHandBack, caller, why) }
|
||||
|
||||
// nonce is four hex digits that keep two asks of one second apart.
|
||||
func (p *Part) nonce() string {
|
||||
@@ -348,9 +340,6 @@ func (p *Part) AskOf(r Request, st UserListState) (asks.Ask, error) {
|
||||
// Warranted is the process's act on the operator's answer (asker.Client.Listen hands it once, after checking the ask, the
|
||||
// option and its level). A no is recorded; a yes runs the switch after the refusals, and a refusal is said to the operator.
|
||||
func (p *Part) Warranted(o asks.Option, w asks.Warrant) error {
|
||||
if isLeaseAsk(w.Ask) {
|
||||
return p.leaseWarranted(o, w)
|
||||
}
|
||||
s, err := p.readSwitch()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -408,13 +397,6 @@ func (p *Part) Said(h asker.Handled, w asks.Warrant, err error) {
|
||||
if err != nil && p.Logf != nil {
|
||||
p.Logf("the answer to %s (%s) ended in an error: %v", w.Ask, h, err)
|
||||
}
|
||||
if isLeaseAsk(w.Ask) {
|
||||
if r, err := p.readLease(); err == nil && r.Open != nil && r.Open.Ask == w.Ask && r.Open.Step == StepAsked {
|
||||
r.end(string(w.Outcome), string(w.Outcome)+" ("+string(h)+"): nothing was written", p.now())
|
||||
_ = p.writeLease(r)
|
||||
}
|
||||
return
|
||||
}
|
||||
s, err := p.readSwitch()
|
||||
if err != nil || s.Open != w.Ask {
|
||||
return
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
)
|
||||
@@ -30,10 +29,7 @@ const (
|
||||
|
||||
// Tools is every tool, in the manifest's order.
|
||||
var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState,
|
||||
ToolAccounts, ToolConsumerMatch, ToolAcceptHashes, ToolLeaseCheck, ToolRemovalDue}
|
||||
|
||||
// SeatTool is the tool serving one of the seat mesh-grants' verbs, `mesh-grants.<verb>`.
|
||||
func SeatTool(verb string) string { return catalogue.GrantsSeat + "." + verb }
|
||||
ToolAccounts, ToolConsumerMatch, ToolAcceptHashes, ToolLeaseCheck}
|
||||
|
||||
// WritesGrants is what grants says of the part's own grants: written once the list the bus loaded carries a build's
|
||||
// accounts, and the one key the assigner reads (ADR 0571 clause 9, which changes ADR 0547 clause 5).
|
||||
|
||||
@@ -13,9 +13,7 @@ import (
|
||||
|
||||
// Issue 774, design 47 L3 (ADR 0547 §11, ADR 0561 §3, ADR 0571 §13): grants_lease_check answers from bus state who holds
|
||||
// the controller's lease, every manifest's dependency on the controller with what keeps it, the controller's streams in
|
||||
// use, and the lease write's check run dry, at the write (the three kept to it alone) and before its delivery. The
|
||||
// judgement of each kind of dependency is mesh-sdk's and tested there (catalogue/controller_dependence_test.go, its rows
|
||||
// moved from here, 774/S4); these rows are the tool's own: what it read, the lease, the streams.
|
||||
// use, and the lease write's check run dry, at the write (the three kept to it alone) and before its delivery.
|
||||
func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
|
||||
const mManifest = `{"module":"mod-m","version":"1","reads":["mesh-controller.declared-packages"]}`
|
||||
controllerHolds := map[string]any{"instance": "controller@novox-be pid 1 since 2026-10-30T14:52:10Z", "epoch": 60,
|
||||
@@ -41,6 +39,29 @@ func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
|
||||
map[string]string{"mod-l": `{"module":"mod-l","version":"1","reads":["mesh-controller.lease"]}`,
|
||||
"mesh-controller": `{"module":"mesh-controller","version":"1","invokes":["seat:mesh-controller.status"]}`},
|
||||
controllerHolds, want{writePasses: true, beforePasses: true}, nil},
|
||||
{"the builder's consume of artifacts-gone is kept to the write itself",
|
||||
map[string]string{"mesh-delivery-builder": `{"module":"mesh-delivery-builder","version":"1","consumes":["mesh-controller.artifacts-gone"]}`},
|
||||
controllerHolds, want{deps: []string{"mesh-delivery-builder consume mesh-controller.artifacts-gone kept"}, writePasses: true, beforePasses: true}, nil},
|
||||
{"the assigner's read of composed is kept until the write's delivery, which drops it",
|
||||
map[string]string{"mesh-delivery-assigner": `{"module":"mesh-delivery-assigner","version":"1","reads":["mesh-controller.composed","mesh-controller.lease"]}`},
|
||||
controllerHolds, want{deps: []string{"mesh-delivery-assigner read mesh-controller.composed kept"}, beforePasses: true,
|
||||
reason: "mesh-delivery-assigner declares read mesh-controller.composed, kept by ADR 0571 §8 only until the lease write's own delivery"}, nil},
|
||||
{"a read no record keeps fails both phases, named",
|
||||
map[string]string{"pacman": `{"module":"pacman","version":"1","reads":["mesh-controller.declared-packages"]}`},
|
||||
controllerHolds, want{deps: []string{"pacman read mesh-controller.declared-packages unkept"},
|
||||
reason: "pacman declares read mesh-controller.declared-packages, which no record keeps until the lease write"}, nil},
|
||||
{"a seat verb and a module tool of the controller's, a consume and a wait are each one; every tool (*) is none",
|
||||
map[string]string{"mod-i": `{"module":"mod-i","version":"1","invokes":["seat:mesh-controller.root-free","mesh-controller.status","*","nats.nats_users"],` +
|
||||
`"consumes":["mesh-controller.checked","mod-x.thing"],"uses":["mesh-controller"]}`,
|
||||
"node-tools": `{"module":"node-tools","version":"1","waits":["mesh-controller.checked"]}`},
|
||||
controllerHolds, want{deps: []string{"mod-i consume mesh-controller.checked unkept", "mod-i invoke mesh-controller.status unkept",
|
||||
"mod-i invoke seat:mesh-controller.root-free unkept", "mod-i uses mesh-controller unkept", "node-tools waits mesh-controller.checked unkept"},
|
||||
reason: "mod-i declares invoke seat:mesh-controller.root-free, which no record keeps"}, nil},
|
||||
{"a process's own invoke of the controller is its module's dependency, named by the process",
|
||||
map[string]string{"mod-p": `{"module":"mod-p","version":"1","resources":[{"id":"account","type":"user","name":"mod-p"},{"id":"worker","type":"process","run":["./w"],"user":"mod-p",` +
|
||||
`"bus":{"invokes":["seat:mesh-controller.held"]}}]}`},
|
||||
controllerHolds, want{deps: []string{"mod-p invoke seat:mesh-controller.held unkept"},
|
||||
reason: "mod-p (process worker) declares invoke seat:mesh-controller.held"}, nil},
|
||||
{"no manifest read from the indexer fails both phases, though nothing read depends on the controller",
|
||||
map[string]string{},
|
||||
controllerHolds, want{reason: "no manifest was read from the indexer (it holds 0 modules)", bothPhases: true}, nil},
|
||||
@@ -119,3 +140,19 @@ func TestTheLeaseCheckJudgesEveryDependencyOnTheController(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Every entry of the keep list names the record that keeps it, and exactly three are kept to the write itself (ADR 0571 §13).
|
||||
func TestTheKeepListNamesItsRecordsAndThreeToTheWrite(t *testing.T) {
|
||||
toTheWrite := 0
|
||||
for _, k := range grants.KeepList {
|
||||
if !strings.HasPrefix(k.Record, "ADR ") || k.Module == "" || k.What == "" {
|
||||
t.Errorf("entry %+v names no record, module or declaration", k)
|
||||
}
|
||||
if k.ToTheWrite {
|
||||
toTheWrite++
|
||||
}
|
||||
}
|
||||
if toTheWrite != 3 {
|
||||
t.Errorf("%d entries kept to the write itself, ADR 0571 §13 names three", toTheWrite)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,459 +0,0 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
|
||||
"mesh-grants/internal/grants"
|
||||
)
|
||||
|
||||
// leaseFake is the bus as the lease write meets it: fakeBus's states and tools, and the seats and tools the write asks.
|
||||
// Each answer is shaped as its writer's code answers it (issue 774): node-service-manager's status the systemd module's
|
||||
// Manager.Status (ActiveState), nats_bucket_expiry nats-tools' BucketExpirySet (now), nats_consumer_remove its
|
||||
// ConsumerRemoved (removed) and an absent consumer its error ("has no consumer"), nats_stream_remove its StreamRemoved
|
||||
// (removed, present), issue-tracker.get mesh-issues' Brief (status), node_trusts mesh-delivery-view's TrustsAnswer
|
||||
// (every), mesh-config's compare its Comparison (source, count), mesh-wants' upgrade compare the assigner's answer
|
||||
// (empty), vault_status mesh-vault's answer (ready, minting.on, minting.held_back).
|
||||
type leaseFake struct {
|
||||
*fakeBus
|
||||
answer map[string]func(map[string]any) (any, error)
|
||||
active string // the controller's ActiveState
|
||||
expiry time.Duration // the lease bucket's
|
||||
ttls map[string]time.Duration
|
||||
log []string // the write's acts on the bus, in order
|
||||
}
|
||||
|
||||
func newLeaseFake(t *testing.T) (*leaseFake, *grants.Part) {
|
||||
t.Helper()
|
||||
b := fixture()
|
||||
b.manifestOf("nats", `{"module":"nats","version":"1","claims":[{"name":"mesh-broker"}],"own-secrets":{"broker":"/var/lib/nats/broker"},`+
|
||||
`"tools":["nats_bucket_expiry","nats_consumer_remove","nats_stream_remove"]}`)
|
||||
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 1", "host": "a", "epoch": 60,
|
||||
"taken": "2026-10-30T14:52:10Z"})
|
||||
b.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowKeeper, By: "operator", Why: "the keeper took over"})
|
||||
busKey, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
||||
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
|
||||
// The controller's hash in the vault, so the list carries the controller and the cut changes it.
|
||||
b.put(grants.ReadAccounts, buscompose.ControllerName, map[string]any{"hash": "$2a$11$controller", "generation": 1,
|
||||
"at": "2026-10-30T14:52:10Z"})
|
||||
f := &leaseFake{fakeBus: b, active: "active", expiry: 15 * time.Second, ttls: map[string]time.Duration{}}
|
||||
f.answer = map[string]func(map[string]any) (any, error){
|
||||
grants.CallIssue: func(map[string]any) (any, error) { return map[string]any{"status": "resolved"}, nil },
|
||||
grants.CallNodeTrusts: func(map[string]any) (any, error) { return map[string]any{"every": true, "not_yet": []string{}}, nil },
|
||||
grants.CallConfigCompare: func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 0}, nil },
|
||||
grants.CallUpgrade: func(map[string]any) (any, error) { return map[string]any{"empty": true, "splits": []any{}}, nil },
|
||||
grants.CallVaultStatus: func(map[string]any) (any, error) {
|
||||
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{}}}, nil
|
||||
},
|
||||
grants.CallServiceStop: func(map[string]any) (any, error) { f.active = "inactive"; return map[string]any{"ok": true}, nil },
|
||||
grants.CallServiceStart: func(map[string]any) (any, error) { f.active = "active"; return map[string]any{"ok": true}, nil },
|
||||
grants.CallServiceStatus: func(map[string]any) (any, error) { return map[string]any{"ActiveState": f.active}, nil },
|
||||
grants.CallBucketExpiry: func(a map[string]any) (any, error) {
|
||||
now := fmt.Sprint(a["max_age"])
|
||||
d, _ := time.ParseDuration(strings.Replace(now, "none", "0s", 1))
|
||||
f.expiry = d
|
||||
f.log = append(f.log, "expiry "+now)
|
||||
return map[string]any{"bucket": a["bucket"], "now": now, "changed": true}, nil
|
||||
},
|
||||
grants.CallRecordWarrant: func(a map[string]any) (any, error) { f.log = append(f.log, "warranted"); return map[string]any{}, nil },
|
||||
grants.CallRecordHandAct: func(a map[string]any) (any, error) {
|
||||
f.log = append(f.log, "hand-act "+fmt.Sprint(a["what"]))
|
||||
return map[string]any{}, nil
|
||||
},
|
||||
grants.CallConsumerRemove: func(map[string]any) (any, error) {
|
||||
f.log = append(f.log, "consumer removed")
|
||||
return map[string]any{"removed": true}, nil
|
||||
},
|
||||
grants.CallStreamRemove: func(map[string]any) (any, error) {
|
||||
f.log = append(f.log, "stream removed")
|
||||
return map[string]any{"removed": true, "present": true}, nil
|
||||
},
|
||||
}
|
||||
p := partOn(b)
|
||||
p.Call, p.Store, p.Lease, p.Asker, p.Objects = f, ttlStore{ownStore{b}, f}, f, &fakeAsker{wheres: map[string]*asks.Warrant{}}, &fakeObjects{}
|
||||
p.SaySeat = func(event string, _ any) error { f.log = append(f.log, "event "+event); return nil }
|
||||
p.Nonce = func() string { return "0001" }
|
||||
return f, p
|
||||
}
|
||||
|
||||
func (f *leaseFake) Call(key string, body any) (json.RawMessage, error) {
|
||||
base, _, _ := strings.Cut(key, "@")
|
||||
if answer, ok := f.answer[base]; ok {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, key)
|
||||
f.mu.Unlock()
|
||||
args, _ := body.(map[string]any)
|
||||
out, err := answer(args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(out)
|
||||
}
|
||||
return f.fakeBus.Call(key, body)
|
||||
}
|
||||
|
||||
func (f *leaseFake) PutHolder(value []byte) error {
|
||||
f.log = append(f.log, "holder")
|
||||
f.states[grants.ReadLease][grants.LeaseKey] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *leaseFake) DeleteHolder() error {
|
||||
f.log = append(f.log, "holder deleted")
|
||||
delete(f.states[grants.ReadLease], grants.LeaseKey)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *leaseFake) Expiry() (time.Duration, error) { return f.expiry, nil }
|
||||
|
||||
// loads says the bus node loaded the list the part last sealed.
|
||||
func (f *leaseFake) loads(p *grants.Part) {
|
||||
g, _, _ := p.Sealed()
|
||||
f.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
|
||||
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Revision: g.Revision, Digest: g.Digest,
|
||||
Users: g.Users, Key: keyOf(f)}})
|
||||
}
|
||||
|
||||
func keyOf(f *leaseFake) string {
|
||||
var h delivery.Have
|
||||
_ = json.Unmarshal(f.states[grants.ReadHave]["a.nats"], &h)
|
||||
return h.UserList.Key
|
||||
}
|
||||
|
||||
// ttlStore is the part's own store with a time to live per key, recorded.
|
||||
type ttlStore struct {
|
||||
ownStore
|
||||
f *leaseFake
|
||||
}
|
||||
|
||||
func (s ttlStore) PutTTL(state, key string, value []byte, ttl time.Duration) (uint64, error) {
|
||||
s.f.ttls[state+"."+key] = ttl
|
||||
s.f.log = append(s.f.log, key+" "+ttl.String())
|
||||
return s.Put(state, key, value)
|
||||
}
|
||||
|
||||
// record is the part's lease record now.
|
||||
func record(t *testing.T, p *grants.Part) grants.LeaseRecord {
|
||||
t.Helper()
|
||||
r, err := p.LeaseState()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// warrant answers the open act's ask with option, as the router's warrant hands it.
|
||||
func warrant(t *testing.T, p *grants.Part, option string) error {
|
||||
t.Helper()
|
||||
r := record(t, p)
|
||||
if r.Open == nil {
|
||||
t.Fatal("no act is open")
|
||||
}
|
||||
return p.Warranted(asks.Option{ID: option}, asks.Warrant{Ask: r.Open.Ask, Option: option, Outcome: asks.OutcomeChosen})
|
||||
}
|
||||
|
||||
// Issue 774 (novox/hq ADR 0582): the lease write, from its ask through the day-later removal, against stand-in seats.
|
||||
func TestTheLeaseWrite(t *testing.T) {
|
||||
// 774/S14/AC1: any one precondition not green refuses the verb naming it; all green asks.
|
||||
t.Run("preconditions", func(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
break_ func(f *leaseFake)
|
||||
}{
|
||||
{"lease-check", func(f *leaseFake) {
|
||||
f.manifestOf("mod-u", `{"module":"mod-u","version":"1","reads":["mesh-controller.conditions"]}`)
|
||||
}},
|
||||
{"earlier-steps", func(f *leaseFake) {
|
||||
f.answer[grants.CallIssue] = func(a map[string]any) (any, error) {
|
||||
if fmt.Sprint(a["number"]) == "1954" {
|
||||
return map[string]any{"status": "fixing"}, nil
|
||||
}
|
||||
return map[string]any{"status": "resolved"}, nil
|
||||
}
|
||||
}},
|
||||
{"node-trusts", func(f *leaseFake) {
|
||||
f.answer[grants.CallNodeTrusts] = func(map[string]any) (any, error) {
|
||||
return map[string]any{"every": false, "not_yet": []string{"ace"}}, nil
|
||||
}
|
||||
}},
|
||||
{"config-compare", func(f *leaseFake) {
|
||||
f.answer[grants.CallConfigCompare] = func(map[string]any) (any, error) { return map[string]any{"source": "indexer", "count": 2}, nil }
|
||||
}},
|
||||
{"upgrade-compare", func(f *leaseFake) {
|
||||
f.answer[grants.CallUpgrade] = func(map[string]any) (any, error) { return map[string]any{"empty": false, "splits": []any{1}}, nil }
|
||||
}},
|
||||
{"keeper-take-over", func(f *leaseFake) {
|
||||
f.put(grants.ReadTakeover, health.TakeoverKey, health.Takeover{V: 1, Follow: health.FollowController})
|
||||
}},
|
||||
{"vault-minting", func(f *leaseFake) {
|
||||
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
|
||||
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
|
||||
}
|
||||
}},
|
||||
{"bus-tools", func(f *leaseFake) { f.manifestOf("nats", natsManifest) }},
|
||||
{"", func(*leaseFake) {}},
|
||||
} {
|
||||
f, p := newLeaseFake(t)
|
||||
row.break_(f)
|
||||
_, err := p.LeaseWrite("operator@shanks", "the controller goes")
|
||||
switch {
|
||||
case row.name == "" && err != nil:
|
||||
t.Errorf("all green: refused: %v", err)
|
||||
case row.name != "" && (err == nil || !strings.Contains(err.Error(), row.name+": ")):
|
||||
t.Errorf("%s not green: %v, want a refusal naming it", row.name, err)
|
||||
case row.name != "":
|
||||
for _, other := range grants.LeaseCheckNames {
|
||||
if other != row.name && strings.Contains(err.Error(), other+": ") {
|
||||
t.Errorf("%s not green: the refusal names %s too: %v", row.name, other, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if r := record(t, p); (r.Open != nil) != (row.name == "") {
|
||||
t.Errorf("%q: an act open %v", row.name, r.Open)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 774/S15/AC1, AC2: a no, an expiry, or a warrant whose checks changed writes nothing, naming the changed check.
|
||||
t.Run("the ask and the warrant", func(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name, step, said string
|
||||
answer func(t *testing.T, f *leaseFake, p *grants.Part)
|
||||
}{
|
||||
{"declined", grants.StepDeclined, "said no", func(t *testing.T, _ *leaseFake, p *grants.Part) {
|
||||
if err := warrant(t, p, "no"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{"expired", "expired", "expired", func(t *testing.T, _ *leaseFake, p *grants.Part) {
|
||||
r := record(t, p)
|
||||
p.Said("", asks.Warrant{Ask: r.Open.Ask, Outcome: asks.OutcomeExpired}, nil)
|
||||
}},
|
||||
{"checks changed", grants.StepRefused, "the checks changed since the ask: vault-minting (now: ready true, minting true, 1 held back)",
|
||||
func(t *testing.T, f *leaseFake, p *grants.Part) {
|
||||
// The vault holds one account back between the ask and the warrant.
|
||||
f.answer[grants.CallVaultStatus] = func(map[string]any) (any, error) {
|
||||
return map[string]any{"ready": true, "minting": map[string]any{"on": true, "held_back": map[string]string{"a.x": "why"}}}, nil
|
||||
}
|
||||
if err := warrant(t, p, "yes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
} {
|
||||
f, p := newLeaseFake(t)
|
||||
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose() // the process asks it
|
||||
if a := p.Asker.(*fakeAsker); len(a.asks) != 1 || len(a.asks[0].Options) != 2 || a.asks[0].Options[0].Binds == "" {
|
||||
t.Fatalf("%s: asked %+v, want one ask whose yes binds the act", row.name, a.asks)
|
||||
}
|
||||
row.answer(t, f, p)
|
||||
r := record(t, p)
|
||||
if r.Open != nil || len(r.Acts) != 1 || r.Acts[0].Step != row.step || !strings.Contains(r.Acts[0].Said, row.said) {
|
||||
t.Errorf("%s: open %v, acts %+v; want one act %s saying %q", row.name, r.Open, r.Acts, row.step, row.said)
|
||||
}
|
||||
if f.count(grants.CallServiceStop+"@a") != 0 || len(f.log) != 0 || r.Cut {
|
||||
t.Errorf("%s: something was written: stops %d, %v, cut %v", row.name, f.count(grants.CallServiceStop+"@a"), f.log, r.Cut)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 774/S16/AC1, AC2; 774/S17/AC1, AC2: the write's six steps, in order, each read back before the next.
|
||||
t.Run("the write", func(t *testing.T) {
|
||||
f, p := newLeaseFake(t)
|
||||
f.answer[grants.CallServiceStop] = func(map[string]any) (any, error) { return map[string]any{"ok": true}, nil } // reads back active
|
||||
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose()
|
||||
if err := warrant(t, p, "yes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r := record(t, p); r.Open != nil || r.Cut || r.Acts[0].Step != grants.StepFailed || !strings.Contains(r.Acts[0].Said, "did not read back stopped") || len(f.log) != 0 {
|
||||
t.Fatalf("a stop not read back: %+v, %v; want the act failed there, nothing written", r, f.log)
|
||||
}
|
||||
|
||||
f, p = newLeaseFake(t)
|
||||
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose()
|
||||
before, _, _ := p.Sealed()
|
||||
if err := warrant(t, p, "yes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := record(t, p)
|
||||
g, _, _ := p.Sealed()
|
||||
if f.active != "inactive" || !r.Cut || r.Open == nil || r.Open.Step != grants.StepCut || r.Open.List != g.Digest || g.Digest == before.Digest {
|
||||
t.Fatalf("after the warrant: active %s, %+v, sealed %s (was %s); want stopped, cut and waiting on the cut list", f.active, r, g.Digest, before.Digest)
|
||||
}
|
||||
if len(f.log) != 0 {
|
||||
t.Fatalf("the expiry or the key was touched before the cut list loaded: %v", f.log)
|
||||
}
|
||||
// The cut list takes the controller's write on the lease and gives mesh-grants the key holder.
|
||||
var st grants.UserListState
|
||||
raw, _, _, _ := ownStore{f.fakeBus}.Get(grants.StateUserList, grants.KeyList)
|
||||
_ = json.Unmarshal(raw, &st)
|
||||
p.Compose() // a pass with the bus node not loaded yet waits
|
||||
if len(f.log) != 0 {
|
||||
t.Fatalf("a pass before the load acted: %v", f.log)
|
||||
}
|
||||
f.loads(p)
|
||||
p.Compose()
|
||||
want := []string{"expiry none", "holder", "event lease-handed", "warranted", "removal-due 24h0m0s"}
|
||||
if !slices.Equal(f.log, want) {
|
||||
t.Errorf("the write's acts %v, want %v in that order", f.log, want)
|
||||
}
|
||||
var h buscompose.LeaseHolder
|
||||
_ = json.Unmarshal(f.states[grants.ReadLease][grants.LeaseKey], &h)
|
||||
r = record(t, p)
|
||||
if h.HandedTo != "mesh-grants" || h.HandedBy != "operator" || h.Why != "the controller goes" || h.Ask != r.Ask || h.At.IsZero() ||
|
||||
r.Open != nil || r.Handed.IsZero() || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
|
||||
t.Errorf("holder %+v, record %+v", h, r)
|
||||
}
|
||||
p.Compose() // the pass the lease key's change wakes
|
||||
if g, _, _ := p.Sealed(); !g.LeaseHanded {
|
||||
t.Error("the user-list key does not say the lease is handed")
|
||||
}
|
||||
if _, err := p.HandBack("operator@shanks", "back"); err == nil || !strings.HasPrefix(err.Error(), "gone") {
|
||||
t.Errorf("the user list's hand-back once the lease is handed: %v, want gone", err)
|
||||
}
|
||||
if _, err := p.LeaseWrite("operator@shanks", "again"); err == nil {
|
||||
t.Error("a second lease-write with the lease handed was asked")
|
||||
}
|
||||
|
||||
// 774/S18/AC1: the key or the expiry changed by another hand is said once, and nothing is written.
|
||||
f.log = nil
|
||||
f.put(grants.ReadLease, grants.LeaseKey, map[string]any{"instance": "controller@a pid 9", "host": "a", "epoch": 61})
|
||||
p.Compose()
|
||||
p.Compose()
|
||||
f.expiry = 15 * time.Second
|
||||
p.Compose()
|
||||
if !slices.Equal(f.log, []string{"event lease-altered"}) || !strings.Contains(record(t, p).Altered, "controller@a pid 9") {
|
||||
t.Errorf("an altered lease said %v (%q), want lease-altered once naming it", f.log, record(t, p).Altered)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/S19/AC1, AC2: hand-back restores the expiry, deletes holder and removal-due, and starts the controller only once
|
||||
// the list with its rights is loaded; once the removal ran it is refused.
|
||||
t.Run("hand-back", func(t *testing.T) {
|
||||
f, p := handed(t)
|
||||
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose()
|
||||
f.log = nil
|
||||
if err := warrant(t, p, "yes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := record(t, p)
|
||||
if f.expiry != 15*time.Second || f.states[grants.ReadLease][grants.LeaseKey] != nil || r.Cut || f.active != "inactive" {
|
||||
t.Fatalf("after the warrant: expiry %s, holder %s, cut %v, controller %s", f.expiry, f.states[grants.ReadLease][grants.LeaseKey], r.Cut, f.active)
|
||||
}
|
||||
if _, _, found, _ := f.ownGet(grants.StateLease, grants.KeyRemovalDue); found {
|
||||
t.Error("removal-due is not deleted")
|
||||
}
|
||||
f.loads(p)
|
||||
p.Compose()
|
||||
if r := record(t, p); f.active != "active" || r.Open != nil || r.Acts[len(r.Acts)-1].Step != grants.StepDone {
|
||||
t.Errorf("once loaded: controller %s, %+v", f.active, r)
|
||||
}
|
||||
// AC2: once the removal ran, refused naming why.
|
||||
f, p = handed(t)
|
||||
p.RemovalDueHeard()
|
||||
p.Compose()
|
||||
if _, err := p.LeaseHandBack("operator@shanks", "too late"); err == nil || !strings.Contains(err.Error(), "without its consumer") {
|
||||
t.Errorf("hand-back after the removal: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/S20/AC1, AC2, AC3: the marker of removal-due starts the removal; a failure says removal-failed and writes the key
|
||||
// again with a day; a hand-back's delete removes nothing.
|
||||
t.Run("the day-later removal", func(t *testing.T) {
|
||||
f, p := handed(t)
|
||||
f.log = nil
|
||||
p.RemovalDueHeard()
|
||||
p.Compose()
|
||||
want := []string{"consumer removed", "hand-act removed the controller's consumer of DEAD_LETTER_NOTICES", "stream removed",
|
||||
"hand-act removed the stream ASSIGNMENTS"}
|
||||
if !slices.Equal(f.log, want) || record(t, p).Removed.IsZero() {
|
||||
t.Errorf("the removal %v (removed %v), want %v", f.log, record(t, p).Removed, want)
|
||||
}
|
||||
|
||||
f, p = handed(t)
|
||||
f.log = nil
|
||||
f.answer[grants.CallStreamRemove] = func(map[string]any) (any, error) {
|
||||
return nil, errors.New("ASSIGNMENTS has 1 consumers: a reader is attached, so it was not removed")
|
||||
}
|
||||
p.RemovalDueHeard()
|
||||
p.Compose()
|
||||
if !slices.Contains(f.log, "event removal-failed") || f.ttls[grants.StateLease+"."+grants.KeyRemovalDue] != grants.RemovalAfter ||
|
||||
!record(t, p).Removed.IsZero() {
|
||||
t.Errorf("a failed removal: %v, removal-due %s", f.log, f.ttls[grants.StateLease+"."+grants.KeyRemovalDue])
|
||||
}
|
||||
|
||||
f, p = handed(t)
|
||||
if _, err := p.LeaseHandBack("operator@shanks", "it goes back"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose()
|
||||
if err := warrant(t, p, "yes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.log = nil
|
||||
p.RemovalDueHeard() // the hand-back's own delete of removal-due, heard
|
||||
p.Compose()
|
||||
if slices.ContainsFunc(f.log, func(s string) bool { return strings.Contains(s, "removed") }) {
|
||||
t.Errorf("a hand-back's delete of removal-due removed %v", f.log)
|
||||
}
|
||||
})
|
||||
|
||||
// 774/S21/AC1: once the lease is handed away a membership is written to the state and not published on its subject.
|
||||
t.Run("memberships after the write", func(t *testing.T) {
|
||||
f, p := handed(t)
|
||||
f.publishes = nil
|
||||
f.assign("b", "mod-x")
|
||||
p.Compose()
|
||||
if len(f.publishes) != 0 {
|
||||
t.Errorf("published on mesh.assignment after the lease write: %v", f.publishes)
|
||||
}
|
||||
if _, _, found, _ := f.ownGet(grants.StateMemberships, "b.mod-x"); !found {
|
||||
t.Error("the membership is not written to the state")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// handed is a part whose lease write is done: the lease names mesh-grants and removal-due is written.
|
||||
func handed(t *testing.T) (*leaseFake, *grants.Part) {
|
||||
t.Helper()
|
||||
f, p := newLeaseFake(t)
|
||||
if _, err := p.LeaseWrite("operator@shanks", "the controller goes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.Compose()
|
||||
if err := warrant(t, p, "yes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.loads(p)
|
||||
p.Compose()
|
||||
if r := record(t, p); r.Handed.IsZero() {
|
||||
t.Fatalf("the write did not finish: %+v, %v", r, f.log)
|
||||
}
|
||||
return f, p
|
||||
}
|
||||
@@ -2,18 +2,9 @@
|
||||
"module": "mesh-grants",
|
||||
"version": "1",
|
||||
"slug": "grants",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-grants",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"lease-write",
|
||||
"lease-hand-back"
|
||||
]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
@@ -25,10 +16,11 @@
|
||||
"mesh-conditions.takeover"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-controller.condition-raised",
|
||||
"mesh-controller.condition-cleared",
|
||||
"mesh-conditions.condition-raised",
|
||||
"mesh-conditions.condition-cleared",
|
||||
"mesh-vault.account-issued",
|
||||
"nats.bucket-expiry-set"
|
||||
"mesh-vault.account-issued"
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
@@ -38,19 +30,7 @@
|
||||
"nats.nats_user_can",
|
||||
"nats.nats_users",
|
||||
"seat:node-engine.root-free",
|
||||
"seat:mesh-hand-acts.warranted",
|
||||
"seat:node-service-manager.stop",
|
||||
"seat:node-service-manager.start",
|
||||
"seat:node-service-manager.status",
|
||||
"nats.nats_bucket_expiry",
|
||||
"nats.nats_consumer_remove",
|
||||
"nats.nats_stream_remove",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"mesh-delivery-view.node_trusts",
|
||||
"seat:mesh-config.compare",
|
||||
"seat:mesh-wants.upgrade",
|
||||
"mesh-vault.vault_status",
|
||||
"seat:issue-tracker.get"
|
||||
"seat:mesh-hand-acts.warranted"
|
||||
],
|
||||
"emits": [
|
||||
"asserted",
|
||||
@@ -71,8 +51,7 @@
|
||||
"grants_accounts",
|
||||
"grants_consumer_match",
|
||||
"grants_accept_hashes",
|
||||
"grants_lease_check",
|
||||
"grants_removal_due"
|
||||
"grants_lease_check"
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
@@ -87,10 +66,6 @@
|
||||
"name": "grants",
|
||||
"history": 5
|
||||
},
|
||||
{
|
||||
"name": "lease",
|
||||
"key-ttl": true
|
||||
},
|
||||
{
|
||||
"name": "memberships",
|
||||
"history": 5
|
||||
|
||||
Reference in New Issue
Block a user