Compare commits
87
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b55a735a97 | ||
|
|
9a469222f8 | ||
|
|
8fdd92f532 | ||
|
|
650a1d0c58 | ||
|
|
511e4b7414 | ||
|
|
4dd7dcfc14 | ||
|
|
d26781b403 | ||
|
|
e2cb398186 | ||
|
|
6a4d88265d | ||
|
|
655692c7ba | ||
|
|
8f9766e46c | ||
|
|
83c1b3ce38 | ||
|
|
000a24d261 | ||
|
|
3228d911cb | ||
|
|
7e358b0f1d | ||
|
|
8d4c13427a | ||
|
|
2049b1a370 | ||
|
|
43d4f511c5 | ||
|
|
f7f3518a04 | ||
|
|
d4aa1fe11b | ||
|
|
14046f9b14 | ||
|
|
be88162cde | ||
|
|
a192052d82 | ||
|
|
1dae2de835 | ||
|
|
0490043990 | ||
|
|
e094e362f4 | ||
|
|
8850a5673e | ||
|
|
cc9c681fa3 | ||
|
|
8f87dbe94d | ||
|
|
7cee3842b8 | ||
|
|
f1c3343929 | ||
|
|
63c8687166 | ||
|
|
b13f65674b | ||
|
|
e6ee939bd4 | ||
|
|
bde60130a8 | ||
|
|
795724f5ca | ||
|
|
44fee9eb0e | ||
|
|
cbcbb9644b | ||
|
|
d1a256b429 | ||
|
|
40bbc7e0d3 | ||
|
|
70f13a3775 | ||
|
|
189e64c486 | ||
|
|
1a1d3a1ecc | ||
|
|
288364f28d | ||
|
|
447bcf3a6b | ||
|
|
07d26adcd3 | ||
|
|
02214cf633 | ||
|
|
f904ff425e | ||
|
|
89370b6f5c | ||
|
|
4edd0e526c | ||
|
|
5d5ee69775 | ||
|
|
9125fcc712 | ||
|
|
17ef1d577d | ||
|
|
da27cd1593 | ||
|
|
aa6a4e7a59 | ||
|
|
8bf45fbafe | ||
|
|
e18be5cabe | ||
|
|
a78fa3db9d | ||
|
|
b81d7475e8 | ||
|
|
7ec6767418 | ||
|
|
30ec4329e0 | ||
|
|
3bae0d8fa2 | ||
|
|
7bd18306d4 | ||
|
|
1302eb9e1f | ||
|
|
f2ed0938ce | ||
|
|
4a84370b2f | ||
|
|
7d1b8864cc | ||
|
|
b18c62f671 | ||
|
|
cc38ca3f8d | ||
|
|
d739516056 | ||
|
|
a6c225eb45 | ||
|
|
5085b681e9 | ||
|
|
9e407f8f76 | ||
|
|
8545e7fc9f | ||
|
|
55a506975b | ||
|
|
b7dfe4c637 | ||
|
|
c4b05a3e93 | ||
|
|
38f2401f17 | ||
|
|
c8b041ab37 | ||
|
|
f52f47e1ff | ||
|
|
213ebe7cfd | ||
|
|
90e278cb97 | ||
|
|
58748c2be1 | ||
|
|
8e9cc58301 | ||
|
|
fb96216e46 | ||
|
|
5059fef3e6 | ||
|
|
564a04b45c |
@@ -1,7 +0,0 @@
|
||||
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
|
||||
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
|
||||
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
|
||||
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
|
||||
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
|
||||
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
|
||||
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
|
||||
+9
-10
@@ -14,7 +14,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// holder is a test holder of the Go build seat: it takes every ask from the seat's check worker, as mesh-build-go does,
|
||||
@@ -92,7 +91,7 @@ func (b *bed) hold(t *testing.T) *holder {
|
||||
|
||||
// 1622/AC1-AC3 and 1623 on the bus: started whole, the assigner composes a group holding a mesh-sdk pull request by
|
||||
// asking the Go build seat for the composer at the member's head, and for the composer at the mesh-sdk the running
|
||||
// controller pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
|
||||
// node-engine pins; it waits naming the asks while they run, writes the verdict from their answers naming the mesh-sdk the
|
||||
// head's composer was built with, and asks nothing more on a later pass of the same heads.
|
||||
func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
@@ -103,10 +102,10 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
b.place(t, "g14", "web")
|
||||
running := strings.Repeat("3", 40)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
|
||||
b.built(t, "mesh-controller", "novox/mesh-controller", running)
|
||||
b.place(t, "novox", "mesh-host")
|
||||
b.built(t, "mesh-host", "novox/mesh-host", running)
|
||||
sdkHead := strings.Repeat("4", 40)
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
@@ -122,8 +121,8 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
key, _ := delivery.CompositionKey("feat/x", hash)
|
||||
// No holder takes the asks yet: the composition waits, naming them.
|
||||
waits := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
|
||||
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-controller at "+running) {
|
||||
t.Fatalf("the waiting composition says %q, want the head's and the running controller's asks", waits.Words)
|
||||
if !strings.Contains(waits.Words, "mesh-sdk "+sdkHead) || !strings.Contains(waits.Words, "mesh-host at "+running) {
|
||||
t.Fatalf("the waiting composition says %q, want the head's and the running node-engine's asks", waits.Words)
|
||||
}
|
||||
h := b.hold(t)
|
||||
v := b.verdict(t, key, func(v delivery.CompositionVerdict) bool { return v.State != delivery.CompositionWaits })
|
||||
@@ -132,7 +131,7 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
}
|
||||
asks := h.taken()
|
||||
if len(asks) != 2 {
|
||||
t.Fatalf("the seat was asked %d composes, want two (the head, the running controller)", len(asks))
|
||||
t.Fatalf("the seat was asked %d composes, want two (the head, the running node-engine)", len(asks))
|
||||
}
|
||||
byRepo := map[string]delivery.StepAsk{}
|
||||
for _, a := range asks {
|
||||
@@ -144,8 +143,8 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
|
||||
if a := byRepo["novox/mesh-sdk"]; a.Compose == nil || a.Compose.Fetch["commit"] != sdkHead || a.Compose.Dir != "go" {
|
||||
t.Errorf("the head's ask = %+v", a.Compose)
|
||||
}
|
||||
if a := byRepo["novox/mesh-controller"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
|
||||
t.Errorf("the running controller's ask = %+v", a.Compose)
|
||||
if a := byRepo["novox/mesh-host"]; a.Compose == nil || a.Compose.Fetch["commit"] != running || a.Compose.Dir != delivery.RootPath {
|
||||
t.Errorf("the running node-engine's ask = %+v", a.Compose)
|
||||
}
|
||||
// A later pass of the same heads (woken by any change) asks nothing more.
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
|
||||
@@ -26,8 +26,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/judge"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -40,6 +40,8 @@ type bed struct {
|
||||
ctx context.Context
|
||||
dir string
|
||||
buckets map[string]jetstream.KeyValue
|
||||
// placed is, by node, the modules the bed's placement names on it (place).
|
||||
placed map[string][]string
|
||||
}
|
||||
|
||||
func startBed(t *testing.T, leaveOut string) *bed {
|
||||
@@ -235,8 +237,7 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
}
|
||||
|
||||
// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
|
||||
// it from the manifest) it is read with no restart: a module only the placement names is
|
||||
// said as placed by one source only.
|
||||
// it from the manifest) it is read with no restart: a node it names gets its own want, the node composing its own.
|
||||
pb := startBed(t, delivery.PlacementBucket)
|
||||
pdone := pb.start(t)
|
||||
select {
|
||||
@@ -253,13 +254,8 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
|
||||
if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the placement state made after the start was not read")
|
||||
}
|
||||
if nw, err := delivery.DecodeNodeWant(pb.get(t, delivery.AssignerWantsBucket, "want.n1.node")); err != nil || !nw.Self {
|
||||
t.Fatalf("the placement state made after the start was not read: n1's own want = %+v, %v", nw, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,6 +267,29 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// place adds module to the seat's placement of node, written as the seat writes it: one node key and a key per module
|
||||
// (research 072 row 37). The placement alone names where modules run (ADR 0567 §8).
|
||||
func (b *bed) place(t *testing.T, node, module string) {
|
||||
t.Helper()
|
||||
if b.placed == nil {
|
||||
b.placed = map[string][]string{}
|
||||
}
|
||||
if !slices.Contains(b.placed[node], module) {
|
||||
b.placed[node] = append(b.placed[node], module)
|
||||
}
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range b.placed[node] {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
b.putJSON(t, delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// built appends the end of a done trunk build of module at commit to the builder's log, as the builder does.
|
||||
func (b *bed) built(t *testing.T, module, repository, commit string) {
|
||||
t.Helper()
|
||||
@@ -284,24 +303,20 @@ func (b *bed) built(t *testing.T, module, repository, commit string) {
|
||||
}
|
||||
}
|
||||
|
||||
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
|
||||
// move to it, in mesh-controller_composed.
|
||||
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
|
||||
// compose says the node's own composition is the digest d (a letter repeated), as its have names it (`<node>.node`): a
|
||||
// placed node composes its own (ADR 0571 §6), and the controller's composition is read no more. It answers the digest.
|
||||
func (b *bed) compose(t *testing.T, node string, d byte) string {
|
||||
t.Helper()
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat(string(d), 64)
|
||||
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
|
||||
if judgement != delivery.JudgedNeither {
|
||||
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
|
||||
}
|
||||
v, err := delivery.EncodeComposedDigest(cd)
|
||||
v, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
b.putJSON(t, delivery.HaveBucket, key, v)
|
||||
return digest
|
||||
}
|
||||
|
||||
@@ -329,8 +344,8 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.other", []byte(`{"node":"ace","module":"other","build":"build-2"}`))
|
||||
b.place(t, "g14", "web")
|
||||
b.place(t, "ace", "other")
|
||||
heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
// 1927/AC1: the candidate carries the fingerprint set the SDK computes, as the indexer writes it.
|
||||
@@ -405,20 +420,20 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
for _, fp := range []string{fNew, fOld} {
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
gkey, _ := delivery.PartGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.compose(t, node, 'c')
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
||||
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
|
||||
b.place(t, node, "web")
|
||||
}
|
||||
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
b.compose(t, "shanks", 'a')
|
||||
b.start(t)
|
||||
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
@@ -426,18 +441,18 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if err != nil || want.FP != fNew {
|
||||
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||
}
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "" || !w.Self || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs, the node composing it, held back from F", w)
|
||||
}
|
||||
var neither asks.Condition
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
|
||||
if strings.HasSuffix(k, ".neither") && c.State == asks.StateRaised {
|
||||
neither = c
|
||||
}
|
||||
}
|
||||
}
|
||||
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
|
||||
if neither.Subject.Machine != "g14" || !strings.HasPrefix(neither.Subject.ID, "g14.web") || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
|
||||
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
|
||||
}
|
||||
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
|
||||
@@ -451,8 +466,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
// there by itself: the condition is cleared.
|
||||
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F with its composition", w)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "" || !w.Self || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F, the node composing it", w)
|
||||
}
|
||||
cleared := false
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
|
||||
@@ -503,7 +518,7 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.place(t, "ace", "web")
|
||||
b.start(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
|
||||
@@ -518,36 +533,34 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key.
|
||||
// conditionsSaid reads what the assigner said of its conditions, by finding key: each a finding in its own bucket, which
|
||||
// the condition keeper carries (novox/hq ADR 0590 §3, issue 1126), raised while it stands and cleared once deleted, each
|
||||
// with the words its last finding said and the subject the finding names (`<module>[.<fingerprint>]`).
|
||||
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
|
||||
t.Helper()
|
||||
stream, err := b.js.Stream(b.ctx, "CONDITIONS")
|
||||
w, err := b.buckets[assign.FindingsBucket].WatchAll(b.ctx, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer w.Stop()
|
||||
out := map[string]asks.Condition{}
|
||||
cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
batch, err := cons.FetchNoWait(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
return out
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
c, err := asks.DecodeCondition(m.Data())
|
||||
c := out[e.Key()]
|
||||
c.Key, c.State = e.Key(), asks.StateCleared
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
f, err := health.DecodeFinding(e.Value())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[c.Key] = c
|
||||
}
|
||||
if n == 0 {
|
||||
return out
|
||||
c = asks.Condition{Key: e.Key(), State: asks.StateRaised, Severity: string(f.Severity), Explanation: f.Explanation,
|
||||
Resolved: f.Resolved, Subject: asks.Scope{Scope: string(f.Subject.Scope), ID: f.Subject.ID, Machine: f.Subject.Machine}}
|
||||
}
|
||||
out[e.Key()] = c
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
|
||||
@@ -566,7 +579,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
wantKey, _ := delivery.WantKey("ace", "mail")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
|
||||
b.place(t, "ace", "mail")
|
||||
b.start(t)
|
||||
|
||||
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
@@ -580,7 +593,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
key := ""
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
|
||||
if c.State == asks.StateRaised && strings.HasPrefix(c.Subject.ID, "mail.") && strings.HasSuffix(k, ".held") {
|
||||
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
|
||||
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
|
||||
}
|
||||
@@ -615,11 +628,11 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
}
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
gkey, _ := delivery.PartGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
|
||||
b.compose(t, "ace", 'c')
|
||||
b.place(t, "ace", name)
|
||||
}
|
||||
count := func() int {
|
||||
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
|
||||
@@ -649,58 +662,6 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
|
||||
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
|
||||
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
|
||||
func TestAWantWaitsForItsComposition(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
time.Sleep(time.Second)
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
t.Fatalf("a want was written before the composition existed: %s", e.Value())
|
||||
}
|
||||
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
|
||||
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
|
||||
}
|
||||
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
|
||||
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
|
||||
}
|
||||
// the node's own want
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
|
||||
}
|
||||
ownKey, _ := delivery.ComposedNodeKey("ace")
|
||||
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
|
||||
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the node's own want = %+v, %v", w, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
|
||||
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
t.Helper()
|
||||
@@ -710,12 +671,12 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
gkey, _ := delivery.PartGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
|
||||
b.place(t, "ace", name)
|
||||
}
|
||||
|
||||
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
|
||||
@@ -778,8 +739,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", 'b')
|
||||
b.compose(t, "ace", 'a')
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
@@ -794,13 +755,13 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b.get(t, delivery.AssignerAsksBucket, hold)
|
||||
b.answer(t, hold, "release")
|
||||
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
|
||||
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
if w.Composed != "" || !w.Self || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v, want F by the rules, the node composing it", w)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
cleared := false
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
|
||||
cleared = cleared || (strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held"))
|
||||
}
|
||||
if cleared {
|
||||
return
|
||||
@@ -825,19 +786,19 @@ func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk
|
||||
|
||||
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
|
||||
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
|
||||
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
|
||||
// value, so the node retries.
|
||||
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
|
||||
b.compose(t, "ace", 'b')
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Self: true}) // a placed node's want names the build alone
|
||||
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
|
||||
b.start(t)
|
||||
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
|
||||
a := b.askNow(t, akey, 0)
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") {
|
||||
t.Fatalf("the backup ask = %+v", a)
|
||||
}
|
||||
// an answer to another ask is no answer to this one
|
||||
@@ -848,7 +809,7 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
}
|
||||
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
|
||||
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
|
||||
if w1.FP != fNew || w1.Composed != first {
|
||||
if w1.FP != fNew || w1.Composed != "" || !w1.Self {
|
||||
t.Errorf("the want = %+v", w1)
|
||||
}
|
||||
// the node retries and the backup fails again: a new have value, asked again under the next number
|
||||
@@ -863,91 +824,10 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
if w2.Words == w1.Words {
|
||||
t.Errorf("the second answer left the want as it was: %+v", w2)
|
||||
}
|
||||
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
|
||||
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
|
||||
a3 := b.askNow(t, akey, 3)
|
||||
if !strings.Contains(a3.Ask.Explanation, second) {
|
||||
t.Errorf("the ask for the new revision = %+v", a3)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
|
||||
t.Errorf("the old answer was taken for the new revision: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
|
||||
|
||||
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
|
||||
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
|
||||
// neither is written at once.
|
||||
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, recorded)
|
||||
old := "sha256:" + strings.Repeat("a", 64)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
|
||||
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
|
||||
b.start(t)
|
||||
akey, _ := delivery.RecordAskKey("ace", "web", nw)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
|
||||
t.Fatalf("the record ask = %+v, %v", a, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the move was written before the yes: %+v", w)
|
||||
}
|
||||
b.answer(t, akey, assign.OptionYes)
|
||||
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
|
||||
// a new revision at the same fingerprint, judged to move data, waits as well
|
||||
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
|
||||
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
|
||||
b.get(t, delivery.AssignerAsksBucket, rkey)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
|
||||
t.Fatalf("the revision was written before the yes: %+v", w)
|
||||
}
|
||||
// and a revision judged neither is written at once
|
||||
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
|
||||
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
|
||||
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
|
||||
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
|
||||
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
|
||||
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
|
||||
b.start(t)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
|
||||
}
|
||||
// the controller composes the bus at the fingerprint it runs with a new user list
|
||||
key, _ := delivery.ComposedKey("ace", "bus", fOld)
|
||||
free := "sha256:" + strings.Repeat("e", 64)
|
||||
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
|
||||
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
if w.FP != fOld {
|
||||
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
|
||||
}
|
||||
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
|
||||
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
|
||||
t.Errorf("the user-list revision was asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
|
||||
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
|
||||
// cleared saying why, and the want stays at the fingerprint.
|
||||
@@ -955,7 +835,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", 'b')
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
@@ -971,7 +851,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
if strings.HasPrefix(k, "module.web.") && c.State == asks.StateCleared && strings.HasSuffix(k, ".held") {
|
||||
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
}
|
||||
@@ -988,7 +868,7 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", 'b')
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
@@ -1252,4 +1132,50 @@ func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
|
||||
if failed.Verdict == nil || !strings.Contains(strings.Join(failed.Verdict.Fails, "; "), "laptop (offline) did not answer") {
|
||||
t.Errorf("the silent node's verdict = %+v", failed.Verdict)
|
||||
}
|
||||
|
||||
// 774/S59 (ADR 0587 §7): a joining node is written on the enrolment's warrant with no node to answer and no ask raised,
|
||||
// and placement-changed carries that warrant. The assigner reads no lease (ADR 0567 §8): the enrolment's own issue waits
|
||||
// for the lease write (ADR 0587 §4).
|
||||
for len(heard) > 0 {
|
||||
<-heard
|
||||
}
|
||||
joined, err := verbs.doPropose(map[string]any{"why": "enrolment of ace", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`,
|
||||
"changes": `[{"op":"add-node","node":"ace","decided":{"address":"10.9.0.2"},"keys":{"overlay":"dHVubmVs"}}]`}, "novox.mesh-enrolment")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Proposal.ID != joined.(proposed).Proposal.ID || e.Warrant == nil || e.Warrant.Warrant != "enrol-ace-1@2026-10-28T20:00:00Z" || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" {
|
||||
t.Errorf("placement-changed for the joining node = %+v", e)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("no placement-changed heard for the joining node")
|
||||
}
|
||||
if ace, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err != nil || !strings.Contains(string(ace.Value()), "10.9.0.2") {
|
||||
t.Errorf("the joining node's key = %v, %v", ace, err)
|
||||
}
|
||||
if akey, _ := delivery.PlacementAskKey(joined.(proposed).Proposal.ID); b.hasKey(delivery.AssignerAsksBucket, akey) || b.hasKey(delivery.PlacementProposalsBucket, joined.(proposed).Proposal.ID) {
|
||||
t.Error("an ask was raised for the joining node, or its proposal stays open")
|
||||
}
|
||||
// Its enrolment ends before it confirms: a remove-node on the same warrant deletes its key at once, with no ask, and
|
||||
// placement-changed carries the warrant (the operator's decision of 2026-11-02 01:05).
|
||||
for len(heard) > 0 {
|
||||
<-heard
|
||||
}
|
||||
if _, err := verbs.doPropose(map[string]any{"why": "enrolment of ace revoked", "warrant": `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`,
|
||||
"changes": `[{"op":"remove-node","node":"ace"}]`}, "novox.mesh-enrolment"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Warrant == nil || len(e.Nodes) != 1 || e.Nodes[0].Node != "ace" {
|
||||
t.Errorf("placement-changed for the removed node = %+v", e)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("no placement-changed heard for the removed node")
|
||||
}
|
||||
if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "ace.node"); err == nil {
|
||||
t.Error("the removed node's key stays")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
@@ -58,10 +57,7 @@ func main() {
|
||||
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||
var readBuckets = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
buscompose.LeaseBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
@@ -70,7 +66,6 @@ var readBuckets = []string{
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
delivery.StepsBucket(goBuildModule),
|
||||
takeoverBucket,
|
||||
provisions.ProvisionsBucket,
|
||||
keptBucket,
|
||||
}
|
||||
@@ -79,10 +74,6 @@ var readBuckets = []string{
|
||||
// newest done trunk build of a module names the commit a reader of the manifests runs at (novox/hq ADR 0571 clause 1).
|
||||
var readLogs = []string{delivery.BuildLogRead}
|
||||
|
||||
// takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said
|
||||
// conditions go straight to the router or only as findings (issue 1952 WM6).
|
||||
var takeoverBucket = health.Bucket(health.KeeperModule, health.TakeoverState)
|
||||
|
||||
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
|
||||
// their steps in its steps state (novox/hq ADR 0482 clause 4).
|
||||
const goBuildModule = "mesh-build-go"
|
||||
@@ -127,40 +118,14 @@ type buckets map[string]jetstream.KeyValue
|
||||
// without it and takes no put further until it starts again (ADR 0546).
|
||||
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||
out := buckets{}
|
||||
// The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names
|
||||
// where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the
|
||||
// controller holds it, as before.
|
||||
away, why := false, ""
|
||||
if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err)
|
||||
} else {
|
||||
out[buscompose.LeaseBucket] = kv
|
||||
e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey)
|
||||
var raw []byte
|
||||
if gerr == nil {
|
||||
raw = e.Value()
|
||||
}
|
||||
if errors.Is(gerr, jetstream.ErrKeyNotFound) {
|
||||
gerr = nil
|
||||
}
|
||||
away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr)
|
||||
}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
if name == buscompose.LeaseBucket {
|
||||
continue
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if name == provider.AssignmentsBucket && err != nil && away {
|
||||
logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why)
|
||||
continue
|
||||
}
|
||||
if name == delivery.AssignerPutsBucket && err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if slices.Contains(seatBuckets, name) && err != nil {
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement the controller's
|
||||
// assignments alone name where modules run, as before any placement was written (ADR 0549 §5).
|
||||
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement no module is placed.
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
@@ -187,16 +152,15 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if (name == assign.FindingsBucket || name == takeoverBucket) && err != nil {
|
||||
// New with health leaving the controller (issue 1952 WM6): without them the conditions are said straight to the
|
||||
// router and kept as no finding, as before.
|
||||
if name == assign.FindingsBucket && err != nil {
|
||||
// New with health leaving the controller (issue 1952 WM6): without it the conditions are said straight to the
|
||||
// router and kept as no finding.
|
||||
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if sourceOfGrants(name) && err != nil {
|
||||
// The grants part's grants and the controller's lease choose where grants are read from (issue 774): without
|
||||
// them the controller's grants are read, as before the lease is handed away.
|
||||
logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err)
|
||||
if name == delivery.PartGrantsBucket && err != nil {
|
||||
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
|
||||
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
@@ -211,17 +175,11 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
|
||||
func startsWithout(name string) bool {
|
||||
switch name {
|
||||
case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket, takeoverBucket,
|
||||
case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
|
||||
delivery.AssignerPutsBucket:
|
||||
return true
|
||||
}
|
||||
return slices.Contains(seatBuckets, name) || sourceOfGrants(name)
|
||||
}
|
||||
|
||||
// sourceOfGrants says the bucket is the grants part's grants or the controller's lease: neither stops the assigner when it
|
||||
// is not there.
|
||||
func sourceOfGrants(name string) bool {
|
||||
return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket
|
||||
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
@@ -259,10 +217,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, buildLog, logger)
|
||||
}
|
||||
|
||||
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, and the
|
||||
// condition keeper's take-over key, watched, so a condition is said straight to the router only until the take-over. A
|
||||
// bucket it cannot open is said: without its findings the conditions are said straight, as before; without the take-over
|
||||
// key they are said straight too, the controller being followed.
|
||||
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
|
||||
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
|
||||
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
|
||||
var f assign.Findings
|
||||
if kv := kvs[assign.FindingsBucket]; kv != nil {
|
||||
@@ -276,46 +232,6 @@ func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) ass
|
||||
return keys, err
|
||||
}
|
||||
}
|
||||
var keeper atomic.Bool
|
||||
f.ToKeeper = keeper.Load
|
||||
takeover := takeoverBucket
|
||||
kv := kvs[takeover]
|
||||
if kv == nil {
|
||||
return f
|
||||
}
|
||||
w, err := kv.Watch(ctx, health.TakeoverKey)
|
||||
if err != nil {
|
||||
process.NotWatched(logf, takeover, err, "the controller is followed: the conditions are said straight to the router")
|
||||
return f
|
||||
}
|
||||
go func() {
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
logf("the watch of %s ended: the last source it named is followed until the assigner starts again", takeover)
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
var raw []byte
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
raw = e.Value()
|
||||
}
|
||||
t, err := health.DecodeTakeover(raw, raw != nil)
|
||||
if err != nil {
|
||||
logf("the take-over key cannot be read (%v): the controller is followed", err)
|
||||
}
|
||||
if was := keeper.Swap(t.Follow == health.FollowKeeper); was != keeper.Load() {
|
||||
logf("the assigner's conditions are said by %s from now (the take-over key follows %s)", t.Seat(), t.Follow)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return f
|
||||
}
|
||||
|
||||
@@ -331,10 +247,7 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
buscompose.ResolvedBucket,
|
||||
provider.AssignmentsBucket,
|
||||
delivery.PartGrantsBucket,
|
||||
buscompose.LeaseBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
@@ -417,12 +330,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
before := w.handedAway()
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
if name == buscompose.LeaseBucket && w.handedAway() == before {
|
||||
// The controller renews its lease every few seconds: only the lease's being handed away makes a pass.
|
||||
continue
|
||||
}
|
||||
wake()
|
||||
}
|
||||
}
|
||||
@@ -458,13 +366,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
return
|
||||
}
|
||||
if bus != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
started := 0
|
||||
for _, name := range watched {
|
||||
if kvs[name] == nil && (startsWithout(name) || name == provider.AssignmentsBucket) {
|
||||
if kvs[name] == nil && startsWithout(name) {
|
||||
continue // not on the bus at this start (open said so)
|
||||
}
|
||||
started++
|
||||
@@ -480,8 +388,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
return err
|
||||
}
|
||||
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then the
|
||||
// controller's assignments alone name where modules run (ADR 0549 §5).
|
||||
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
|
||||
// module is placed.
|
||||
type appeared struct {
|
||||
name string
|
||||
kv jetstream.KeyValue
|
||||
@@ -494,7 +402,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
}
|
||||
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
|
||||
for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
|
||||
for _, name := range []string{delivery.PartGrantsBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
|
||||
if kvs[name] == nil && js != nil {
|
||||
name := name
|
||||
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
|
||||
@@ -521,10 +429,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if bus != nil {
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf, w.handedAway)
|
||||
p.seat = newSeatActs(bus, wake, logger.Printf)
|
||||
}
|
||||
if js != nil {
|
||||
p.publish = streamPublisher(ctx, js, thisNode)
|
||||
|
||||
@@ -19,10 +19,8 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
@@ -74,28 +72,21 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
|
||||
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
|
||||
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
lease []byte
|
||||
said string
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
|
||||
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
|
||||
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
|
||||
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, "no put is taken further"},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, "no put is taken further"},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, ""},
|
||||
// Issue 774: the grants part's grants not there do not stop the start.
|
||||
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, "no grant is read"},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
@@ -112,12 +103,14 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
placed, err := delivery.PlacementValues(delivery.Placement{Node: "ace"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
for key, raw := range placed {
|
||||
kv := kvs[delivery.PlacementBucket].(*watchKV)
|
||||
kv.entries = append(kv.entries, kvEntry{key: key, value: raw})
|
||||
}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
|
||||
@@ -136,18 +129,13 @@ type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
lease []byte // the controller's lease, when it holds one
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
}
|
||||
kv := &memKV{values: map[string][]byte{}}
|
||||
if name == buscompose.LeaseBucket && r.lease != nil {
|
||||
kv.values[buscompose.ControllerLeaseKey] = r.lease
|
||||
}
|
||||
return kv, nil
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
|
||||
@@ -30,10 +30,8 @@ type passer struct {
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(asks.Condition) error
|
||||
said map[string]bool
|
||||
// findings is the assigner's findings bucket (its said conditions and policy-split, ADR 0564 §2, ADR 0565 §8); splits the
|
||||
// modules whose policy-split finding stands, nil until the first pass, which deletes any a previous process left.
|
||||
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
|
||||
findings assign.Findings
|
||||
splits map[string]bool
|
||||
// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
|
||||
failedProvisions map[string]bool
|
||||
// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
|
||||
@@ -59,12 +57,10 @@ type passer struct {
|
||||
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
|
||||
// §1, issue 1496): raised once per words, cleared once when it ends.
|
||||
nodeFailed map[string]string
|
||||
// callSaid is, by the key of a refused controller call's condition, what is raised (issue 774): by this process, or by
|
||||
// a previous one and taken over from the findings.
|
||||
callSaid map[string]*callSaid
|
||||
// callsSwept says the first pass took over the refused calls' conditions a previous process raised (sweepCalls).
|
||||
callsSwept bool
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its calls of the controller (nil: none is made), how it says
|
||||
// callsCleared says the first pass cleared the conditions of refused controller calls a previous process raised
|
||||
// (clearStaleCalls): the assigner makes no such call now (issue 774).
|
||||
callsCleared bool
|
||||
// The seat mesh-placement's holder (seat.go, ADR 0567): its acts (nil: none is made), how it says
|
||||
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
|
||||
seat *seatActs
|
||||
publish publisher
|
||||
@@ -92,8 +88,7 @@ func (p *passer) pass(ctx context.Context) {
|
||||
p.backups(ctx)
|
||||
}
|
||||
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
placed := p.w.placedNodes()
|
||||
@@ -109,20 +104,6 @@ func (p *passer) nodeWants(ctx context.Context) {
|
||||
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
if placed[node] {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
continue
|
||||
}
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
}
|
||||
|
||||
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
|
||||
@@ -130,7 +111,7 @@ func (p *passer) nodeWants(ctx context.Context) {
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned, _ := p.w.assigned()
|
||||
assigned := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
manifests := map[string]catalogue.Manifest{}
|
||||
for name, mod := range trunk {
|
||||
@@ -155,10 +136,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
defer func() { p.provisionsMissing(ctx, missing) }()
|
||||
defer p.unplacedFindings()
|
||||
defer p.seatHolders(ctx)
|
||||
defer p.policySplits(manifests, chosen)
|
||||
differ := p.withCalls(p.w.placementDiffers())
|
||||
p.sayCalls(differ)
|
||||
frozen := p.sayPlacement(differ)
|
||||
p.clearStaleCalls()
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
@@ -178,7 +156,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
stale := map[string]neitherOne{}
|
||||
left := map[string]string{}
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
// Where grants are read from, once for the pass: the grants part's while the lease is handed away (issue 774).
|
||||
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
|
||||
grants := p.w.grantsBucket()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
@@ -186,18 +164,10 @@ func (p *passer) wants(ctx context.Context) {
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Self: func(node string) bool { return placed[node] },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if placed[node] && p.w.handedAway() {
|
||||
return p.w.ownComposed(node) // from the lease write: the digest its own have names (ADR 0571 §6)
|
||||
}
|
||||
if placed[node] {
|
||||
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
|
||||
// names the build alone, and the node judges the move itself.
|
||||
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
|
||||
}
|
||||
return p.w.composed(node, name, fp)
|
||||
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
|
||||
},
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
key, err := delivery.PartGrantKey(name, fp)
|
||||
return err == nil && p.w.has(grants, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
@@ -210,13 +180,13 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.PartGrantKey(name, fp)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
|
||||
if raw := p.w.value(grants, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
}
|
||||
@@ -267,9 +237,6 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.recordAsk(ctx, name, fp, need)
|
||||
}
|
||||
for node, want := range out.Wants {
|
||||
if frozen[node+"."+name] {
|
||||
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
|
||||
}
|
||||
key, err := delivery.WantKey(node, name)
|
||||
if err != nil {
|
||||
p.logf("no want of %s on %s is written: %v", name, node, err)
|
||||
@@ -903,42 +870,6 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
|
||||
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
|
||||
// for them and keeps the one written before. A say that fails is made again at the next pass.
|
||||
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
|
||||
frozen := map[string]bool{}
|
||||
now := map[string]bool{}
|
||||
for _, d := range differ {
|
||||
frozen[d.Node+"."+d.Module] = true
|
||||
c := assign.PlacementCondition(d, true)
|
||||
now[c.Key] = true
|
||||
if p.said[c.Key+".raised"] {
|
||||
continue
|
||||
}
|
||||
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
|
||||
continue
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
}
|
||||
prefix := delivery.AssignerModule + ".placement."
|
||||
for key := range p.said {
|
||||
c, raised := strings.CutSuffix(key, ".raised")
|
||||
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
|
||||
continue
|
||||
}
|
||||
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
|
||||
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
|
||||
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
delete(p.said, key)
|
||||
}
|
||||
return frozen
|
||||
}
|
||||
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
@@ -991,58 +922,6 @@ func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogu
|
||||
return nil
|
||||
}
|
||||
|
||||
// policySplits writes the finding policy-split for each module whose policy at the assigner differs from one the controller
|
||||
// writes beside a composition of it, and deletes it once they agree (ADR 0565 §8): written when a split begins, never again
|
||||
// while it stands. The first pass after a start deletes every one a previous process left that no longer splits.
|
||||
func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) {
|
||||
if p.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now
|
||||
if p.now != nil {
|
||||
now = p.now
|
||||
}
|
||||
split := map[string]bool{}
|
||||
theirs := map[string][]string{}
|
||||
if !p.w.handedAway() { // the controller writes no policy from the lease write (ADR 0571 §6)
|
||||
theirs = assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket))
|
||||
}
|
||||
for _, s := range assign.Splits(manifests, chosen, theirs) {
|
||||
split[s.Module] = true
|
||||
if p.splits[s.Module] {
|
||||
continue
|
||||
}
|
||||
value, err := assign.PolicySplitFinding(s, now())
|
||||
if err == nil {
|
||||
err = p.findings.Put(assign.PolicySplitKey(s.Module), value)
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the policy split of %s is not written: %v", s.Module, err)
|
||||
delete(split, s.Module)
|
||||
}
|
||||
}
|
||||
gone := p.splits
|
||||
if gone == nil && p.findings.Keys != nil {
|
||||
gone = map[string]bool{}
|
||||
if keys, err := p.findings.Keys(); err == nil {
|
||||
for _, k := range keys {
|
||||
if m, ok := strings.CutSuffix(strings.TrimPrefix(k, "module."), "."+assign.KindPolicySplit); ok && strings.HasPrefix(k, "module.") {
|
||||
gone[m] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for m := range gone {
|
||||
if !split[m] {
|
||||
if err := p.findings.Delete(assign.PolicySplitKey(m)); err != nil {
|
||||
p.logf("the policy split of %s is not deleted: %v", m, err)
|
||||
split[m] = true // tried again at the next pass
|
||||
}
|
||||
}
|
||||
}
|
||||
p.splits = split
|
||||
}
|
||||
|
||||
// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat
|
||||
// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed
|
||||
// provision is also said as the assigner's finding `provider.<key>.provision-failed` (a waiting one is the watchman's
|
||||
|
||||
@@ -8,17 +8,14 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -52,6 +49,34 @@ func newPasser(kvs buckets, at time.Time) *passer {
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
// placeOn puts the seat's placement of node, naming the modules, into the world: one node key and a key per module
|
||||
// (research 072 row 37).
|
||||
func placeOn(t *testing.T, w *world, node string, modules ...string) {
|
||||
t.Helper()
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// composes puts the digest the node's own have names as its composition (`<node>.node` of the haves).
|
||||
func composes(t *testing.T, w *world, node, digest string) {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
w.put(delivery.HaveBucket, key, raw)
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
@@ -196,7 +221,7 @@ func putOfTheBus(t *testing.T) {
|
||||
p := newPasser(kvs, now)
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, nil)
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
|
||||
p.acts.snapshot = func(context.Context, string, string) (string, error) {
|
||||
return "novox's restore point of nats (streams) taken 22:01", nil
|
||||
}
|
||||
@@ -211,7 +236,7 @@ func putOfTheBus(t *testing.T) {
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||
placeOn(t, p.w, "novox", "nats")
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
@@ -267,11 +292,9 @@ func putOfTheBus(t *testing.T) {
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
composes(t, p.w, "novox", digest)
|
||||
gkey, _ := delivery.PartGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
world(delivery.PartGrantsBucket, gkey, g)
|
||||
got := step()
|
||||
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||
@@ -356,7 +379,7 @@ func putOfTheBus(t *testing.T) {
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
|
||||
aPlacementAloneNamesWhereModulesRun(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
@@ -530,7 +553,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
@@ -568,7 +591,7 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
placeOn(t, p.w, "shanks", "systemd-resolved")
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
@@ -632,15 +655,13 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
name string
|
||||
trunk map[string]delivery.Module
|
||||
gone []string // the keys taken away
|
||||
noDigit bool // the controller's composition is not read
|
||||
heldAt string // the node's want is at this fingerprint, whose composition is digOld
|
||||
noDigit bool // the node's own composition is not read
|
||||
}{
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
|
||||
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
|
||||
{"the trunk is not read at all", nil, nil, false, ""},
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true},
|
||||
{"the trunk is not read at all", nil, nil, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
@@ -667,21 +688,9 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
asksKV.values[key] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
}
|
||||
placeOn(t, p.w, "novox", "web") // a placed node composes its own: the digest its have names
|
||||
if !c.noDigit {
|
||||
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
|
||||
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
p.w.put(delivery.ControllerComposedBucket, ckey, v)
|
||||
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
|
||||
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
if c.heldAt != "" {
|
||||
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
|
||||
}
|
||||
p.w.put(delivery.ControllerComposedBucket, okey, old)
|
||||
}
|
||||
if c.heldAt != "" {
|
||||
wkey, _ := delivery.WantKey("novox", "web")
|
||||
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
|
||||
p.w.put(delivery.AssignerWantsBucket, wkey, w)
|
||||
composes(t, p.w, "novox", digNew)
|
||||
}
|
||||
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||
wantsKV.values[heldOld] = []byte("x")
|
||||
@@ -779,134 +788,38 @@ func mustKey(key string, err error) string {
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
|
||||
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
|
||||
if cerr != nil {
|
||||
t.Fatal(cerr)
|
||||
}
|
||||
put(delivery.ControllerComposedBucket, ckey, cd)
|
||||
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
|
||||
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, tkey, tcd)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", revised)
|
||||
for i := 0; i < 2; i++ {
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
|
||||
}
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != revised {
|
||||
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
|
||||
}
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Error("the answered record ask was taken away")
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
|
||||
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
|
||||
// assignments are the one source and nothing is raised.
|
||||
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
|
||||
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
|
||||
// module in an assignment only is no module of its node; before any placement is read nothing is placed and the check composes
|
||||
// no input; placed, the check composes over the input the nodes compose from (frombus, 1126, ADR 0547 §2).
|
||||
func aPlacementAloneNamesWhereModulesRun(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
assigned := func(node, module string) {
|
||||
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
|
||||
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
|
||||
}
|
||||
place := func(node string, modules ...string) {
|
||||
pl := delivery.Placement{Node: node}
|
||||
for _, m := range modules {
|
||||
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
// One node key and a key per module (research 072 row 37); a module taken off leaves its key, which the node
|
||||
// key no longer names.
|
||||
values, err := delivery.PlacementValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
p.w.put(delivery.PlacementBucket, key, raw)
|
||||
}
|
||||
}
|
||||
assigned("g14", "a")
|
||||
assigned("g14", "b")
|
||||
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v, said %v", got, said)
|
||||
p.w.put("mesh-controller_assignments", "g14.b", []byte(`{"node":"g14","module":"b","build":"b-1"}`)) // read no more
|
||||
if got := p.w.assigned(); len(got) != 0 {
|
||||
t.Fatalf("before any placement: assigned %v", got)
|
||||
}
|
||||
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
|
||||
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
place("g14", "a", "c")
|
||||
got, _ := p.w.assigned()
|
||||
frozen := p.sayPlacement(p.w.placementDiffers())
|
||||
p.sayPlacement(p.w.placementDiffers())
|
||||
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
|
||||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
|
||||
!strings.Contains(said[1].Explanation, "the seat mesh-placement's placement names c on g14") {
|
||||
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
|
||||
placeOn(t, p.w, "g14", "c", "a")
|
||||
placeOn(t, p.w, "n2", "d")
|
||||
if got := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "c"}) || !slices.Equal(got["n2"], []string{"d"}) || len(got) != 2 {
|
||||
t.Errorf("the placement alone: assigned %v", got)
|
||||
}
|
||||
place("g14", "a", "b", "c")
|
||||
assigned("g14", "c")
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
|
||||
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
|
||||
// wrote its record; until then it says why and composes the assignments alone.
|
||||
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
|
||||
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
|
||||
}
|
||||
placeOn(t, p.w, "g14", "a", "b", "c")
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
|
||||
p.w.put(delivery.HaveBucket, "g14.node", rec)
|
||||
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n2", TunnelKey: "k2"}})
|
||||
p.w.put(delivery.HaveBucket, "n2.node", rec2)
|
||||
m := p.w.mesh(context.Background(), none)
|
||||
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
if m.Input == nil || len(m.Input.Nodes) != 2 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
|
||||
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
|
||||
}
|
||||
|
||||
// Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules
|
||||
// run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of
|
||||
// the node. A lease that goes back to a controller brings the comparison back.
|
||||
place("n2", "d")
|
||||
assigned("g14", "e")
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`))
|
||||
got, all := p.w.assigned()
|
||||
if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 {
|
||||
t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all)
|
||||
}
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 {
|
||||
t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:])
|
||||
}
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`))
|
||||
if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] {
|
||||
t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen)
|
||||
}
|
||||
}
|
||||
|
||||
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
|
||||
@@ -921,7 +834,7 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
put(delivery.PlacementBucket, "placement.g14", pl)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
@@ -958,15 +871,10 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatal("a have waiting for a yes raised no record ask")
|
||||
}
|
||||
// Issue 774, 2026-10-31: while the controller still composes the placed node's module (its lease not handed away), its
|
||||
// digests are not the one the node's have waits on; the ask stays, and a pass that changes nothing puts nothing: on novox
|
||||
// the asks of postgres, keycloak and mongodb were taken away and made again every 8 s, and a want put unchanged each
|
||||
// pass made the node compose again each time.
|
||||
for _, fp := range []string{fpTest, fpOld} {
|
||||
ck, _ := delivery.ComposedKey("g14", "web", fp)
|
||||
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, ck, cv)
|
||||
}
|
||||
// Issue 774, 2026-10-31: the node's own composition is not the digest its have waits on; the ask stays, and a pass that
|
||||
// changes nothing puts nothing: on novox the asks of postgres, keycloak and mongodb were taken away and made again every
|
||||
// 8 s, and a want put unchanged each pass made the node compose again each time.
|
||||
composes(t, p.w, "g14", "sha256:"+strings.Repeat("c", 64))
|
||||
asked := asksKV.values[akey]
|
||||
p.wants(context.Background())
|
||||
if !bytes.Equal(asksKV.values[akey], asked) {
|
||||
@@ -1003,178 +911,37 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is the assigner's process account against a controller that refuses the calls whose verb is in refuse.
|
||||
type refusingBus struct {
|
||||
fakeBus
|
||||
refuse map[string]string
|
||||
// pins are the controller's pins its placement-export answers, by node.
|
||||
mu sync.Mutex
|
||||
pins map[string]map[string]delivery.Provider
|
||||
}
|
||||
|
||||
func (b *refusingBus) setPins(node string, pins map[string]delivery.Provider) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.pins == nil {
|
||||
b.pins = map[string]map[string]delivery.Provider{}
|
||||
// 774/S25/AC1 (ADR 0567 §8): an approved placement change is written and followed with its warrant recorded and
|
||||
// placement-changed said, and no verb of the controller is called; the refused-call conditions a previous process raised are
|
||||
// cleared once, and said again at a later pass while the findings cannot be read.
|
||||
func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 11, 2, 9, 0, 0, 0, time.UTC))
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
p.seat = newSeatActs(bus, func() {}, func(string, ...any) {})
|
||||
var said []string
|
||||
p.publish = func(event string, _ any, _ string) error { said = append(said, event); return nil }
|
||||
prop := delivery.PlacementProposal{ID: "p-one", By: "operator@laptop", Why: "move web"}
|
||||
change := delivery.ChangeOf(delivery.Placement{Node: "anchor"}, delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
p.followWrite(prop, delivery.Answer{}, []delivery.PlacementChange{change})
|
||||
if len(bus.calls) != 0 || !slices.Equal(said, []string{delivery.PlacementChangedEvent}) {
|
||||
t.Fatalf("calls = %d, said = %v", len(bus.calls), said)
|
||||
}
|
||||
b.pins[node] = pins
|
||||
}
|
||||
|
||||
func (b *refusingBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
if key == exportCall {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
node, _ := body.(map[string]any)["node"].(string)
|
||||
answer, _ := json.Marshal(map[string]delivery.Placement{node: {V: delivery.Version, Node: node, Pins: b.pins[node]}})
|
||||
return json.Marshal(map[string]any{"ok": true, "answer": json.RawMessage(answer)})
|
||||
}
|
||||
for verb, words := range b.refuse {
|
||||
if strings.HasSuffix(key, "."+verb) {
|
||||
return nil, errors.New(words)
|
||||
}
|
||||
}
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
}
|
||||
|
||||
// Issue 774: a call of the controller it refuses is a condition, never only a journal line: raised once, urgent, naming the verb,
|
||||
// the node, the module and the refusal; cleared when a later call for the pair is answered, or, for a module's place, once the
|
||||
// pair differed and the two sources agree.
|
||||
func TestARefusedControllerCallIsACondition(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
bus := &refusingBus{refuse: map[string]string{"assign": `assign takes no argument "why" — it takes "build", "module", "node"; nothing was done`}}
|
||||
woke := make(chan struct{}, 4)
|
||||
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
|
||||
assignWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
|
||||
call := func(c assign.ControllerCall) {
|
||||
p.seat.callAll([]assign.ControllerCall{c})
|
||||
<-woke
|
||||
fails := 1
|
||||
p.findings.Keys = func() ([]string, error) {
|
||||
if fails > 0 {
|
||||
fails--
|
||||
return nil, errors.New("the bucket is not there yet")
|
||||
}
|
||||
return []string{"module.anchor.web.controller-call", "module.laptop.db.controller-pin", "module.anchor.web.neither"}, nil
|
||||
}
|
||||
differ := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
|
||||
|
||||
call(assignWeb)
|
||||
p.sayCalls(differ)
|
||||
p.sayCalls(differ)
|
||||
if len(conds) != 1 || conds[0].State != asks.StateRaised || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
!strings.Contains(conds[0].Explanation, `takes no argument "why"`) || !strings.Contains(conds[0].Headline, "assign web on anchor") {
|
||||
t.Fatalf("after a refused assign, said = %+v", conds)
|
||||
}
|
||||
|
||||
bus.refuse = nil
|
||||
call(assignWeb)
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != conds[0].Key {
|
||||
t.Fatalf("after the assign is answered, said = %+v", conds)
|
||||
}
|
||||
|
||||
// Refused again, then assigned by hand: cleared once the pair, seen differing, agrees; not before it was seen differing.
|
||||
bus.refuse = map[string]string{"assign": "refused"}
|
||||
call(assignWeb)
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 3 || conds[2].State != asks.StateRaised {
|
||||
t.Fatalf("a second refusal is not raised: %+v", conds)
|
||||
}
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 3 {
|
||||
t.Fatalf("cleared before the pair was seen differing: %+v", conds)
|
||||
}
|
||||
p.sayCalls(differ)
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 4 || conds[3].State != asks.StateCleared || !strings.Contains(conds[3].Resolved, "agree") {
|
||||
t.Fatalf("the hand assign does not clear it: %+v", conds)
|
||||
}
|
||||
p.sayCalls(nil)
|
||||
if len(conds) != 4 {
|
||||
t.Fatalf("said again after the clear: %+v", conds)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774: a refused pin's condition, and one a previous process raised, clear once the pair agrees after it was seen differing:
|
||||
// the controller's pins (its placement-export, read again when its compositions change) against the placement's, the controller's
|
||||
// assignments against the placement for a module's place. A start whose findings cannot be read takes them over at a later pass.
|
||||
func TestARefusedCallClearsWhenTheSourcesAgree(t *testing.T) {
|
||||
store := delivery.Provider{Node: "anchor", Module: "store"}
|
||||
pinDB := assign.ControllerCall{Verb: "pin", Node: "laptop", Module: "db",
|
||||
Args: map[string]any{"node": "laptop", "provision": "db", "from": "anchor", "module": "store"}}
|
||||
placeWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
|
||||
webDiffers := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
|
||||
for name, c := range map[string]struct {
|
||||
call *assign.ControllerCall // refused in this process; nil: raised by a previous one
|
||||
findings []string // the findings a previous process left
|
||||
keysFail int // how many first reads of the findings fail
|
||||
differ []assign.Pair // the pairs the two sources name apart, while they differ
|
||||
agree func(*refusingBus) // the person's hand call
|
||||
key string
|
||||
}{
|
||||
"a pin refused, pinned by hand": {call: &pinDB, agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
|
||||
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
|
||||
"a place refused, assigned by hand": {call: &placeWeb, differ: webDiffers, agree: func(*refusingBus) {},
|
||||
key: "mesh-delivery-assigner.controller-call.anchor.web"},
|
||||
"a pin raised before a start": {findings: []string{"module.laptop.db.controller-pin"}, keysFail: 1,
|
||||
agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
|
||||
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
|
||||
"a place raised before a start": {findings: []string{"module.anchor.web.controller-call"}, differ: webDiffers, agree: func(*refusingBus) {},
|
||||
key: "mesh-delivery-assigner.controller-call.anchor.web"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
|
||||
var conds []asks.Condition
|
||||
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
|
||||
fails := c.keysFail
|
||||
p.findings.Keys = func() ([]string, error) {
|
||||
if fails > 0 {
|
||||
fails--
|
||||
return nil, errors.New("the bucket is not there yet")
|
||||
}
|
||||
return c.findings, nil
|
||||
}
|
||||
bus := &refusingBus{refuse: map[string]string{"pin": "refused", "assign": "refused"}}
|
||||
woke := make(chan struct{}, 16)
|
||||
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
|
||||
// The placement pins db on laptop; the controller does not.
|
||||
raw, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "laptop", Pins: map[string]delivery.Provider{"db": store}})
|
||||
key, _ := delivery.LegacyPlacementKey("laptop")
|
||||
p.w.put(delivery.PlacementBucket, key, raw)
|
||||
// pass is a pass, then one more for each background read it started.
|
||||
pass := func(differ []assign.Pair) {
|
||||
p.sayCalls(differ)
|
||||
for {
|
||||
select {
|
||||
case <-woke:
|
||||
p.sayCalls(differ)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.call != nil {
|
||||
p.seat.callAll([]assign.ControllerCall{*c.call})
|
||||
<-woke
|
||||
}
|
||||
pass(c.differ)
|
||||
pass(c.differ)
|
||||
for _, cond := range conds {
|
||||
if cond.State == asks.StateCleared {
|
||||
t.Fatalf("cleared while the sources differ: %+v", conds)
|
||||
}
|
||||
}
|
||||
c.agree(bus)
|
||||
p.w.put(delivery.ControllerComposedBucket, "laptop.x", []byte("moved")) // a pin changes a composition
|
||||
pass(nil)
|
||||
if len(conds) == 0 {
|
||||
t.Fatal("nothing was raised")
|
||||
}
|
||||
last := conds[len(conds)-1]
|
||||
if last.Key != c.key || last.State != asks.StateCleared || !strings.Contains(last.Resolved, "agree") {
|
||||
t.Fatalf("the hand call does not clear it: %+v", conds)
|
||||
}
|
||||
n := len(conds)
|
||||
pass(nil)
|
||||
if len(conds) != n {
|
||||
t.Errorf("said again after the clear: %+v", conds[n:])
|
||||
}
|
||||
})
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
p.clearStaleCalls()
|
||||
if len(conds) != 2 || conds[0].State != asks.StateCleared || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
conds[1].Key != "mesh-delivery-assigner.controller-pin.laptop.db" {
|
||||
t.Fatalf("the old conditions cleared = %+v", conds)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,8 +55,8 @@ type putActs struct {
|
||||
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs {
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{rec: handacts.Recorder{Ask: bus.Call}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
@@ -97,7 +97,7 @@ func (p *passer) puts(ctx context.Context) {
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned, _ := p.w.assigned()
|
||||
assigned := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
@@ -226,9 +226,9 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest,
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if gkey, err := delivery.PartGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
if g, err := delivery.DecodeGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,14 +10,12 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/judge"
|
||||
@@ -59,80 +57,23 @@ func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) p
|
||||
}
|
||||
}
|
||||
|
||||
// seatActs keeps what the seat's calls of the controller came to while this process runs: the export read again at a
|
||||
// seed's warrant, and each call that keeps the controller's assignments in step (§8).
|
||||
// seatActs keeps the seat's acts while this process runs: the warrant recorded in the hand-act log. The seat calls no verb of
|
||||
// the controller: placement is the one source (ADR 0567 §8, issue 774).
|
||||
type seatActs struct {
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
export map[string]exportCheck
|
||||
// calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to.
|
||||
calls map[string]string
|
||||
// outcomes is, by the key of a call's condition, the last call made for its pair once answered or refused: refused
|
||||
// "" when the controller answered it (issue 774: a refused call is a condition, never only a journal line).
|
||||
outcomes map[string]callOutcome
|
||||
// pins is, by node, the controller's pins as its placement-export last answered them (pinsOf).
|
||||
pins map[string]pinRead
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
// rec records the seat's acts in the hand-act log (ADR 0565 §11).
|
||||
rec handacts.Recorder
|
||||
}
|
||||
|
||||
// callOutcome is what one call of the controller came to: the call, and the controller's refusal ("" when it answered).
|
||||
type callOutcome struct {
|
||||
call assign.ControllerCall
|
||||
refused string
|
||||
}
|
||||
|
||||
// exportCheck is the export read again for one proposal: running, its answer, or why it was not read.
|
||||
type exportCheck struct {
|
||||
running bool
|
||||
export map[string]delivery.Placement
|
||||
err string
|
||||
}
|
||||
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{},
|
||||
outcomes: map[string]callOutcome{}, pins: map[string]pinRead{}, rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}}
|
||||
}
|
||||
|
||||
// exportOf is the export read again for a proposal, asked once in the background.
|
||||
func (a *seatActs) exportOf(id string) exportCheck {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if c, ok := a.export[id]; ok {
|
||||
return c
|
||||
}
|
||||
a.export[id] = exportCheck{running: true}
|
||||
go func() {
|
||||
defer a.wake()
|
||||
raw, err := a.bus.Call(exportCall, map[string]any{})
|
||||
c := exportCheck{}
|
||||
if err != nil {
|
||||
c.err = "the controller's placement-export did not answer: " + err.Error()
|
||||
} else if c.export, err = assign.ExportOf(raw); err != nil {
|
||||
c.err = err.Error()
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.export[id] = c
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
return exportCheck{running: true}
|
||||
}
|
||||
|
||||
// forget drops what is kept of a proposal's export, so a later warrant reads it again.
|
||||
func (a *seatActs) forget(id string) {
|
||||
if a == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
delete(a.export, id)
|
||||
a.mu.Unlock()
|
||||
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
|
||||
return &seatActs{bus: bus, wake: wake, logf: logf, rec: handacts.Recorder{Ask: bus.Call}}
|
||||
}
|
||||
|
||||
// record records once, in the hand-act log, that placement was written on the operator's warrant (§5): the router's
|
||||
// record of the teller's ask says who chose and how (ADR 0274). The log is the seat mesh-hand-acts' from its delivery, and
|
||||
// the controller's while that seat has no holder and the controller holds its lease (ADR 0565 §11, handacts.Recorder).
|
||||
// the controller's while that seat has no holder (ADR 0565 §11, handacts.Recorder).
|
||||
func (a *seatActs) record(warrant string) {
|
||||
if a == nil || warrant == "" {
|
||||
return
|
||||
@@ -147,257 +88,35 @@ func (a *seatActs) record(warrant string) {
|
||||
}()
|
||||
}
|
||||
|
||||
// handAct records a hand-back in the hand-act log (§7), as record does.
|
||||
func (a *seatActs) handAct(prop delivery.PlacementProposal) {
|
||||
go func() {
|
||||
if taken, err := a.rec.Record(handacts.Request{What: "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "),
|
||||
Why: prop.Why, Cause: "asked by " + prop.By + ", approved by the operator"}); err != nil {
|
||||
a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err)
|
||||
} else {
|
||||
a.logf("the hand-back %s is recorded in the log of %s", prop.ID, taken.Log)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// callAll makes the controller's calls in order in the background, each said in the journal and kept by its pair.
|
||||
func (a *seatActs) callAll(calls []assign.ControllerCall) {
|
||||
if len(calls) == 0 {
|
||||
// clearStaleCalls clears, once, the conditions of the refused controller calls a previous process raised (findings of the
|
||||
// kinds controller-call and controller-pin): the assigner calls no verb of the controller any more (ADR 0567 §8, issue 774),
|
||||
// so nothing would clear them. It is tried again at the next pass while the findings cannot be read.
|
||||
func (p *passer) clearStaleCalls() {
|
||||
if p.callsCleared || p.say == nil || p.findings.Keys == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
for _, c := range calls {
|
||||
a.calls[c.Node+"."+c.Module] = c.Key() + " is asked"
|
||||
}
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
for _, c := range calls {
|
||||
words, refused := c.Key()+" answered", ""
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+"."+c.Verb, c.Args); err != nil {
|
||||
refused = err.Error()
|
||||
words = c.Key() + " refused: " + refused
|
||||
}
|
||||
a.logf("keeping the controller in step with placement: %s", words)
|
||||
a.mu.Lock()
|
||||
a.calls[c.Node+"."+c.Module] = words
|
||||
a.outcomes[assign.CallConditionKey(c)] = callOutcome{call: c, refused: refused}
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// outcomesNow is a copy of every call's last outcome, by the key of its condition.
|
||||
func (a *seatActs) outcomesNow() map[string]callOutcome {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return maps.Clone(a.outcomes)
|
||||
}
|
||||
|
||||
// pinRead is the controller's pins of one node as its placement-export answered them, for the generation of the
|
||||
// controller's compositions it was asked at: a pin made at the controller changes a composition, which the assigner
|
||||
// watches, so a read is asked again only after such a change, never on a timer.
|
||||
type pinRead struct {
|
||||
gen uint64
|
||||
running bool
|
||||
known bool
|
||||
pins map[string]delivery.Provider
|
||||
}
|
||||
|
||||
// pinsOf is the controller's pins of node as last read, and whether a read answered; a read is asked in the background
|
||||
// when none was made at gen, and wakes a pass when it answers.
|
||||
func (a *seatActs) pinsOf(node string, gen uint64) (map[string]delivery.Provider, bool) {
|
||||
if a == nil {
|
||||
return nil, false
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
r, ok := a.pins[node]
|
||||
if !ok || (r.gen != gen && !r.running) {
|
||||
r.gen, r.running = gen, true
|
||||
a.pins[node] = r
|
||||
go func() {
|
||||
defer a.wake()
|
||||
raw, err := a.bus.Call(exportCall, map[string]any{"node": node})
|
||||
var pins map[string]delivery.Provider
|
||||
var known bool
|
||||
if err == nil {
|
||||
var export map[string]delivery.Placement
|
||||
if export, err = assign.ExportOf(raw); err == nil {
|
||||
pins, known = export[node].Pins, true
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
a.logf("the controller's pins of %s were not read, so a refused pin's condition there waits: %v", node, err)
|
||||
}
|
||||
a.mu.Lock()
|
||||
now := a.pins[node]
|
||||
now.running = false
|
||||
if known {
|
||||
now.pins, now.known = pins, true
|
||||
}
|
||||
a.pins[node] = now
|
||||
a.mu.Unlock()
|
||||
}()
|
||||
}
|
||||
return r.pins, r.known
|
||||
}
|
||||
|
||||
// sayCalls makes a refused call of the controller loud (issue 774): the condition of each pair whose last call was refused
|
||||
// is raised once per refusal, naming the verb, the node, the module and the controller's words. It is cleared when a later
|
||||
// call for the pair is answered, or once the pair, seen differing since the refusal, agrees between the two sources (a
|
||||
// person made the call by hand): for a module's place the controller's assignments and the placement, for a pin the
|
||||
// controller's pins (its placement-export, read again after its compositions change) and the placement's. The first pass
|
||||
// after a start takes over what a previous process raised (its findings) and clears each whose pair agrees; until the
|
||||
// findings are read it is tried again at every pass. A say that fails is made again at the next pass.
|
||||
func (p *passer) sayCalls(differ []assign.Pair) {
|
||||
if p.say == nil {
|
||||
return
|
||||
}
|
||||
if p.callSaid == nil {
|
||||
p.callSaid = map[string]*callSaid{}
|
||||
}
|
||||
differs := map[string]bool{}
|
||||
for _, d := range differ {
|
||||
differs[d.Node+"."+d.Module] = true
|
||||
}
|
||||
outcomes := p.seat.outcomesNow()
|
||||
if !p.callsSwept {
|
||||
p.callsSwept = p.sweepCalls(outcomes)
|
||||
}
|
||||
for key, o := range outcomes {
|
||||
said := p.callSaid[key]
|
||||
switch {
|
||||
case o.refused != "" && (said == nil || said.adopted || said.refused != o.refused):
|
||||
if err := p.say(assign.CallRefusedCondition(o.call, o.refused, true, "")); err != nil {
|
||||
p.logf("that %s was refused could not be said, and is said at the next pass: %v", o.call.Key(), err)
|
||||
continue
|
||||
}
|
||||
p.callSaid[key] = &callSaid{call: o.call, refused: o.refused}
|
||||
case o.refused == "" && said != nil:
|
||||
if err := p.say(assign.CallRefusedCondition(o.call, "", false, o.call.Key()+" was answered.")); err != nil {
|
||||
p.logf("the condition of %s could not be cleared, and is cleared at the next pass: %v", o.call.Key(), err)
|
||||
continue
|
||||
}
|
||||
delete(p.callSaid, key)
|
||||
}
|
||||
}
|
||||
var placements map[string]delivery.Placement
|
||||
for key, said := range p.callSaid {
|
||||
c := said.call
|
||||
var apart bool
|
||||
if c.CallKind() == assign.KindControllerCall {
|
||||
apart = differs[c.Node+"."+c.Module]
|
||||
} else {
|
||||
theirs, known := p.seat.pinsOf(c.Node, p.w.bucketGen(delivery.ControllerComposedBucket))
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
if placements == nil {
|
||||
placements, _ = p.seatPlacements()
|
||||
}
|
||||
ours, has := placements[c.Node].Pins[c.Module]
|
||||
to, had := theirs[c.Module]
|
||||
apart = has != had || ours != to
|
||||
}
|
||||
if apart {
|
||||
said.differed = true
|
||||
continue
|
||||
}
|
||||
if !said.differed {
|
||||
continue // not seen differing since the refusal: the world may not hold the write yet
|
||||
}
|
||||
if err := p.say(assign.CallRefusedCondition(c, "", false, agreeWords(c))); err != nil {
|
||||
p.logf("the condition of %s could not be cleared, and is cleared at the next pass: %v", c.Key(), err)
|
||||
continue
|
||||
}
|
||||
if o, ok := outcomes[key]; ok {
|
||||
p.seat.forgetOutcome(key, o)
|
||||
}
|
||||
delete(p.callSaid, key)
|
||||
}
|
||||
}
|
||||
|
||||
// sweepCalls takes over the refused calls' conditions a previous process raised, from the findings: each is followed as
|
||||
// one seen differing, so it clears at the first pass its pair agrees. It answers whether the findings were read.
|
||||
func (p *passer) sweepCalls(outcomes map[string]callOutcome) bool {
|
||||
if p.findings.Keys == nil {
|
||||
return true
|
||||
}
|
||||
keys, err := p.findings.Keys()
|
||||
if err != nil {
|
||||
p.logf("the findings were not read, so the refused calls a previous process raised are taken over at the next pass: %v", err)
|
||||
return false
|
||||
p.logf("the findings were not read, so the refused controller calls a previous process raised are cleared at the next pass: %v", err)
|
||||
return
|
||||
}
|
||||
p.callsCleared = true
|
||||
for _, k := range keys {
|
||||
rest, ok := strings.CutPrefix(k, "module.")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, kind := range []string{assign.KindControllerCall, assign.KindControllerPin} {
|
||||
pair, ok := strings.CutSuffix(rest, "."+kind)
|
||||
node, module, two := strings.Cut(pair, ".")
|
||||
if !ok || !two {
|
||||
continue
|
||||
if pair, ok := strings.CutSuffix(rest, "."+kind); ok {
|
||||
if node, module, two := strings.Cut(pair, "."); two {
|
||||
if err := p.say(assign.CallClearedCondition(kind, node, module)); err != nil {
|
||||
p.callsCleared = false
|
||||
p.logf("the condition of the old %s call for %s on %s could not be cleared, and is cleared at the next pass: %v", kind, module, node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
verb := "assign"
|
||||
if kind == assign.KindControllerPin {
|
||||
verb = "pin"
|
||||
}
|
||||
c := assign.ControllerCall{Verb: verb, Node: node, Module: module}
|
||||
key := assign.CallConditionKey(c)
|
||||
if _, made := outcomes[key]; made || p.callSaid[key] != nil {
|
||||
continue
|
||||
}
|
||||
p.callSaid[key] = &callSaid{call: c, differed: true, adopted: true}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func agreeWords(c assign.ControllerCall) string {
|
||||
if c.CallKind() == assign.KindControllerPin {
|
||||
return "the controller's pins and the placement agree on " + c.Module + " for " + c.Node + "."
|
||||
}
|
||||
return "the controller's assignments and the placement agree on " + c.Module + " on " + c.Node + "."
|
||||
}
|
||||
|
||||
// callSaid is a raised call's condition: its call, the refusal it said, whether its pair was seen differing since, and
|
||||
// whether it was taken over from a previous process.
|
||||
type callSaid struct {
|
||||
call assign.ControllerCall
|
||||
refused string
|
||||
differed bool
|
||||
adopted bool
|
||||
}
|
||||
|
||||
// forgetOutcome drops a refused call's outcome once its condition is cleared, unless a later call replaced it.
|
||||
func (a *seatActs) forgetOutcome(key string, o callOutcome) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if now, ok := a.outcomes[key]; ok && now.refused == o.refused && now.call.Verb == o.call.Verb {
|
||||
delete(a.outcomes, key)
|
||||
}
|
||||
}
|
||||
|
||||
// callOf is what the last call for a pair came to, "" when none was made.
|
||||
func (a *seatActs) callOf(node, module string) string {
|
||||
if a == nil {
|
||||
return ""
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.calls[node+"."+module]
|
||||
}
|
||||
|
||||
// withCalls names, on each pair the two sources do not both name, the call the assigner made for it (§8).
|
||||
func (p *passer) withCalls(pairs []assign.Pair) []assign.Pair {
|
||||
for i := range pairs {
|
||||
pairs[i].Call = p.seat.callOf(pairs[i].Node, pairs[i].Module)
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
|
||||
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
|
||||
@@ -544,17 +263,22 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos
|
||||
p.endProposal(ctx, prop, prop.Withdrawn)
|
||||
return
|
||||
}
|
||||
if prop.Seed || prop.HandBack {
|
||||
// Adopt and hand-back are gone since the lease write (ADR 0547 §11, ADR 0567 §11): a proposal one left open ends.
|
||||
p.endProposal(ctx, prop, "adopt and hand-back are gone since the lease write: placement is the mesh's alone")
|
||||
return
|
||||
}
|
||||
if prop.Warrant != nil {
|
||||
p.addNode(ctx, prop, revs)
|
||||
return
|
||||
}
|
||||
akey, _ := delivery.PlacementAskKey(prop.ID)
|
||||
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
|
||||
if prop.State == delivery.ProposalAsked {
|
||||
askPulse, _ := delivery.PulseAskKey(prop.ID)
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
if prop.HandBack {
|
||||
p.handBack(ctx, prop, akey, revs)
|
||||
} else {
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
}
|
||||
p.warranted(ctx, prop, rev, akey, current, revs)
|
||||
return
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
p.endProposal(ctx, prop, "declined by the operator")
|
||||
@@ -564,19 +288,6 @@ func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementPropos
|
||||
return
|
||||
}
|
||||
}
|
||||
if prop.HandBack {
|
||||
// The operator's alone (§7): no node answers it; its ask is raised at once, and again on again.
|
||||
if prop.State != delivery.ProposalAsked || again {
|
||||
brief := fmt.Sprintf("%s (asked by %s): every node's placement key is deleted, %s, and each node returns to the "+
|
||||
"controller's compositions; the history is kept", prop.Why, prop.By, strings.Join(prop.Nodes(), ", "))
|
||||
now := p.clock().UTC()
|
||||
prop.Verdict = &delivery.PlacementVerdict{State: delivery.VerdictPass, At: now}
|
||||
if prop, ok := p.ask(ctx, prop, akey, brief); ok {
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
|
||||
return
|
||||
@@ -747,8 +458,7 @@ func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal
|
||||
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
|
||||
held, recorded := p.lays(ctx)
|
||||
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded,
|
||||
Seed: prop.Seed, ExportDigest: prop.ExportDigest, Leased: p.w.handedAway()})
|
||||
Current: current, Revisions: revs, Held: held, Recorded: recorded})
|
||||
if len(findings) > 0 {
|
||||
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
|
||||
return
|
||||
@@ -761,7 +471,6 @@ func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal,
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
|
||||
p.seat.forget(prop.ID)
|
||||
p.writeProposal(ctx, again, rev)
|
||||
}
|
||||
|
||||
@@ -778,35 +487,14 @@ func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
|
||||
}
|
||||
|
||||
// warranted writes an approved proposal (§5): for the seed, after the export is read again and found unchanged (§7);
|
||||
// each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, placement-changed said, and the controller kept in step (§8).
|
||||
// warranted writes an approved proposal (§5): each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
|
||||
// proposal deleted, and placement-changed said.
|
||||
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
if prop.Seed {
|
||||
if p.seat == nil {
|
||||
return
|
||||
}
|
||||
c := p.seat.exportOf(prop.ID)
|
||||
switch {
|
||||
case c.running:
|
||||
return // its answer wakes the next pass
|
||||
case c.err != "":
|
||||
if words := "the export cannot be read again at the warrant: " + c.err; prop.Words != words {
|
||||
prop.Words = words
|
||||
p.writeProposal(ctx, prop, rev)
|
||||
}
|
||||
return
|
||||
case delivery.ExportDigest(c.export) != prop.ExportDigest:
|
||||
p.endProposal(ctx, prop, "the controller's export changed since the seed was proposed: the new export is proposed as the seed")
|
||||
p.proposeSeed(ctx, prop, c.export)
|
||||
return
|
||||
}
|
||||
}
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
@@ -831,7 +519,7 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal,
|
||||
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
|
||||
// again over the keys as this write left them.
|
||||
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
p.followWrite(prop, answer, changes)
|
||||
if rest, ok := assign.Rest(prop, writtenLaid); ok {
|
||||
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
|
||||
} else {
|
||||
@@ -857,22 +545,83 @@ func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal,
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.followWrite(prop, answer, changes, current, writtenLaid)
|
||||
p.followWrite(prop, answer, changes)
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded, placement-changed said, and, until the lease
|
||||
// write, the controller's assignments kept in step for them (§8). The seed calls no controller verb: it is what the
|
||||
// controller holds already (§9).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange,
|
||||
before map[string]delivery.Placement, written map[string]*delivery.Placement) {
|
||||
// addNode writes a joining node on the warrant of the enrolment that issued its token (ADR 0587 §7): the operator's Approve
|
||||
// was the issue's, so no node is waited on and no ask is raised. The key is created at the revision it was laid over (none: a
|
||||
// node with a key is no joining node); the enrolment recorded the warrant in the hand-act log when it was given, so this does
|
||||
// not record it again; placement-changed carries the warrant. A key written meanwhile ends the proposal, saying so.
|
||||
func (p *passer) addNode(ctx context.Context, prop delivery.PlacementProposal, revs map[string]uint64) {
|
||||
current, _ := p.seatPlacements()
|
||||
if moved(prop, revs) {
|
||||
p.endProposal(ctx, prop, "a key it was laid over was written since: the enrolment proposes the node again")
|
||||
return
|
||||
}
|
||||
laid, findings := delivery.LayChanges(current, prop.Changes)
|
||||
findings = append(findings, assign.ControllerRefusals(prop.Changes, laid)...)
|
||||
node := prop.Changes[0].Node
|
||||
if prop.Changes[0].Op == delivery.OpRemoveNode {
|
||||
p.removeNode(ctx, prop, current, findings)
|
||||
return
|
||||
}
|
||||
value, ok := laid[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be added: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, &value)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not written: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(delivery.Placement{Node: node}, value)
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the operator approved the machine %s at %s through the enrolment's ask %s", w.Node, w.Address, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// removeNode deletes the key of a joining node whose enrolment ended unconfirmed, on that enrolment's warrant (the operator's
|
||||
// decision of 2026-11-02 01:05): no node answers and no ask is raised, as for its add-node; placement-changed carries the warrant.
|
||||
// A node that runs modules or holds seats is no joining node and stays (LayChanges' finding ends the proposal).
|
||||
func (p *passer) removeNode(ctx context.Context, prop delivery.PlacementProposal, current map[string]delivery.Placement, findings []string) {
|
||||
node := prop.Changes[0].Node
|
||||
before, ok := current[node]
|
||||
if len(findings) > 0 || !ok {
|
||||
p.endProposal(ctx, prop, "the node cannot be taken off: "+strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
rev, err := p.writeNode(ctx, node, nil)
|
||||
if err != nil {
|
||||
p.endProposal(ctx, prop, "the node's key was not deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
|
||||
c.Before, c.After = prop.Base[node], rev
|
||||
p.deleteCandidates(ctx, prop.ID)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
w := prop.Warrant
|
||||
p.sayChanged(prop, delivery.Answer{Option: assign.OptionYes, Warrant: w.Warrant,
|
||||
Words: fmt.Sprintf("the machine %s left placement: its enrolment (ask %s) ended before it confirmed", w.Node, w.Ask)},
|
||||
[]delivery.PlacementChange{c})
|
||||
}
|
||||
|
||||
// followWrite follows the keys a warrant wrote: the warrant recorded and placement-changed said. No verb of the
|
||||
// controller is called: the placement is the one source of where modules run (ADR 0567 §8, issue 774).
|
||||
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
if p.seat != nil && !p.w.handedAway() && !prop.Seed {
|
||||
p.seat.callAll(assign.ControllerCalls(before, written))
|
||||
}
|
||||
}
|
||||
|
||||
func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
@@ -884,44 +633,6 @@ func sortedNodes(m map[string]*delivery.Placement) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// handBack deletes every key on the operator's warrant (§7) at the revision the hand-back read, records it in the
|
||||
// controller's hand-act log and says placement-changed; every node returns to the controller's compositions. A key moved
|
||||
// since ends it, and the operator hands back again.
|
||||
func (p *passer) handBack(ctx context.Context, prop delivery.PlacementProposal, akey string, revs map[string]uint64) {
|
||||
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
|
||||
p.endProposal(ctx, prop, "its warrant does not bind this hand-back, so nothing is deleted: "+err.Error())
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
current, held := p.seatPlacements()
|
||||
var changes []delivery.PlacementChange
|
||||
for _, node := range prop.Nodes() {
|
||||
key, _ := delivery.PlacementNodeKey(node)
|
||||
err := error(nil)
|
||||
if held[node] != prop.Base[node] {
|
||||
err = fmt.Errorf("its placement is at revision %d, the hand-back read %d", held[node], prop.Base[node])
|
||||
} else {
|
||||
_, err = p.writeNode(ctx, node, nil)
|
||||
}
|
||||
if err != nil {
|
||||
p.sayChanged(prop, answer, changes)
|
||||
p.endProposal(ctx, prop, fmt.Sprintf("%s moved since the hand-back was asked (%v): hand back again", key, err))
|
||||
return
|
||||
}
|
||||
c := delivery.ChangeOf(current[node], delivery.Placement{Node: node})
|
||||
c.Before = prop.Base[node]
|
||||
changes = append(changes, c)
|
||||
}
|
||||
if p.seat != nil {
|
||||
p.seat.handAct(prop)
|
||||
}
|
||||
p.seat.record(answer.Warrant)
|
||||
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.deletePulses(ctx, prop.ID)
|
||||
p.sayChanged(prop, answer, changes)
|
||||
}
|
||||
|
||||
// sayChanged says placement-changed for the keys written.
|
||||
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
|
||||
if len(changes) == 0 {
|
||||
@@ -938,26 +649,6 @@ func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Ans
|
||||
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
|
||||
}
|
||||
|
||||
// proposeSeed proposes an export as the seed again, by the same proposer and why.
|
||||
func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal, export map[string]delivery.Placement) {
|
||||
held, recorded := p.lays(ctx)
|
||||
now := p.clock()
|
||||
seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: p.w.handedAway()})
|
||||
if len(findings) > 0 {
|
||||
p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; "))
|
||||
return
|
||||
}
|
||||
for _, node := range seed.Nodes() {
|
||||
key, _ := delivery.PlacementCandidateKey(seed.ID, node)
|
||||
c := delivery.PlacementCandidate{Proposal: seed.ID, Placement: laid[node]}
|
||||
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
|
||||
}
|
||||
seed.Written = now.UTC()
|
||||
p.writeProposal(ctx, seed, 0)
|
||||
}
|
||||
|
||||
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
|
||||
// whole with why it ended.
|
||||
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
|
||||
@@ -973,7 +664,6 @@ func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposa
|
||||
return
|
||||
}
|
||||
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
|
||||
p.seat.forget(prop.ID)
|
||||
p.logf("the proposal %s ends: %s", prop.ID, why)
|
||||
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -207,7 +206,7 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
for _, c := range changes {
|
||||
if c.Op == delivery.OpSeed {
|
||||
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
current, revs, err := s.placements()
|
||||
@@ -218,9 +217,16 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
warrant, err := warrantOf(args["warrant"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
|
||||
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Warrant: warrant})
|
||||
if refusal := delivery.AddNodeRefusal(p, moduleOfCaller(by)); refusal != "" {
|
||||
findings = append(findings, refusal)
|
||||
}
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
@@ -230,6 +236,42 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// warrantOf reads propose's warrant, an enrolment's for an add-node: an object, given as one or as its text; none is none.
|
||||
func warrantOf(v any) (*delivery.AddNodeWarrant, error) {
|
||||
var raw []byte
|
||||
switch w := v.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case string:
|
||||
if strings.TrimSpace(w) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw = []byte(w)
|
||||
default:
|
||||
var err error
|
||||
if raw, err = json.Marshal(w); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var out delivery.AddNodeWarrant
|
||||
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("the warrant is no enrolment's warrant {node, address, ask, warrant}: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// moduleOfCaller is the module a caller is, as the runtime proves it, for AddNodeRefusal: mesh-enrolment when the proven
|
||||
// caller is that module's (its bus user `<node>.mesh-enrolment`, never a person's `person.mesh-enrolment`), else the caller
|
||||
// as proven, which the refusal names.
|
||||
func moduleOfCaller(by string) string {
|
||||
if (stdio.Callers{delivery.EnrolmentModule}).Allows(by) {
|
||||
return delivery.EnrolmentModule
|
||||
}
|
||||
return by
|
||||
}
|
||||
|
||||
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
|
||||
// placement-withdrawn carrying it whole.
|
||||
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
|
||||
@@ -270,94 +312,6 @@ func (s *seatTools) doAgain(args map[string]any) (any, error) {
|
||||
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
|
||||
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
|
||||
|
||||
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
|
||||
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
|
||||
current, _, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(current) > 0 {
|
||||
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
|
||||
}
|
||||
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
|
||||
for _, id := range ids {
|
||||
if p, err := s.proposal(id); err == nil && p.Seed {
|
||||
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := s.ask(exportCall, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
|
||||
}
|
||||
export, err := assign.ExportOf(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now()
|
||||
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
|
||||
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
|
||||
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
|
||||
if len(findings) > 0 {
|
||||
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
if err := s.write(p, laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
|
||||
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
|
||||
// unreadable, is not handed away.
|
||||
func (s *seatTools) leased() bool {
|
||||
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
|
||||
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
|
||||
// records the hand act and says placement-changed; every node returns to the controller's compositions.
|
||||
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
|
||||
if s.leased() {
|
||||
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
|
||||
}
|
||||
why := arg(args, "why")
|
||||
if why == "" {
|
||||
return nil, errors.New("hand-back says why: it is recorded with the hand act")
|
||||
}
|
||||
if by == "" {
|
||||
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
|
||||
}
|
||||
_, revs, err := s.placements()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(revs) == 0 {
|
||||
return nil, errors.New("there is no placement key to hand back")
|
||||
}
|
||||
now := s.now()
|
||||
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
|
||||
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
|
||||
raw, err := delivery.EncodePlacementProposal(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
|
||||
return nil, fmt.Errorf("the hand-back is not written: %w", err)
|
||||
}
|
||||
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
|
||||
"node returns to the controller's compositions"}, nil
|
||||
}
|
||||
|
||||
// placementAnswer is what the verb placement answers.
|
||||
type placementAnswer struct {
|
||||
Keys map[string]keyed `json:"keys,omitempty"`
|
||||
@@ -531,8 +485,6 @@ func (s *seatTools) tools() []stdio.Tool {
|
||||
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbAgain: s.doAgain,
|
||||
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
|
||||
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
|
||||
}
|
||||
var out []stdio.Tool
|
||||
for _, v := range seat.Serves {
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
@@ -36,9 +35,8 @@ func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
|
||||
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
|
||||
}
|
||||
|
||||
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt proposes the
|
||||
// controller's export whole as the seed, once; hand-back records the hand act and deletes every key; placement and
|
||||
// ADR 0567 §2, §7, §10: the seat's five verbs over the states. propose lays a proposal and writes it with its candidates,
|
||||
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; placement and
|
||||
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
|
||||
func TestTheSeatsVerbs(t *testing.T) {
|
||||
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
|
||||
@@ -53,32 +51,22 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
|
||||
st[haveState].values[n+".node"] = raw
|
||||
}
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor})
|
||||
var asked []string
|
||||
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
if key == exportCall {
|
||||
out, _ := json.Marshal(map[string]any{"ok": true, "output": string(export)})
|
||||
return out, nil
|
||||
}
|
||||
return json.RawMessage(`{"ok":true}`), nil
|
||||
})
|
||||
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
|
||||
|
||||
// adopt: the export proposed whole as the seed, once.
|
||||
got, err := s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop")
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
got, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := got.(proposed).Proposal
|
||||
if !seed.Seed || seed.ExportDigest != delivery.ExportDigest(map[string]delivery.Placement{"anchor": anchor}) || !slices.Equal(asked, []string{exportCall}) ||
|
||||
len(st[delivery.PlacementCandidatesState].values) != 1 || st[delivery.PlacementProposalsState].values[seed.ID] == nil {
|
||||
t.Fatalf("adopt = %+v, asked %v", seed, asked)
|
||||
}
|
||||
if _, err := s.doAdopt(map[string]any{"why": "again"}, "op"); err == nil || !strings.Contains(err.Error(), "is open") {
|
||||
t.Errorf("a second adopt with a seed open: %v", err)
|
||||
}
|
||||
// withdraw and again mark the proposal; the holder ends or judges it.
|
||||
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -95,10 +83,8 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
|
||||
|
||||
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
|
||||
raw, _ := delivery.EncodePlacement(anchor)
|
||||
st[delivery.PlacementState].values["placement.anchor"] = raw
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
|
||||
!strings.Contains(err.Error(), "ghost") || !strings.Contains(err.Error(), "lease write") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
!strings.Contains(err.Error(), "ghost") || len(st[delivery.PlacementProposalsState].values) != 0 {
|
||||
t.Errorf("a refused proposal: %v", err)
|
||||
}
|
||||
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
|
||||
@@ -146,32 +132,62 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// hand-back: the operator's alone, written as a hand-back proposal naming every key at its revision, for the assigner
|
||||
// to ask; nothing is deleted by the verb.
|
||||
got, err = s.doHandBack(map[string]any{"why": "back"}, "op")
|
||||
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
|
||||
t.Errorf("hand-back = %+v, %v", got, err)
|
||||
}
|
||||
// Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and
|
||||
// there is no controller to hand placement back to.
|
||||
st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}}
|
||||
// Issue 774 (ADR 0567 §8): every kind of change is proposed, and there is no controller to hand placement back to.
|
||||
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
|
||||
t.Errorf("a hold after the lease write: %v", err)
|
||||
t.Errorf("a hold: %v", err)
|
||||
}
|
||||
if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") {
|
||||
t.Errorf("a hand-back after the lease write: %v", err)
|
||||
// 774/S59/AC1 (ADR 0587 §7): an add-node from mesh-enrolment whose warrant names this node and this address is proposed, with
|
||||
// the node a joining one with no record; from another caller, naming another node or address, or with no warrant it is refused.
|
||||
addNode := func(by, warrant string) error {
|
||||
_, err := s.doPropose(map[string]any{"why": "enrolment of ace", "warrant": warrant, "changes": []any{map[string]any{"op": "add-node", "node": "ace",
|
||||
"decided": map[string]any{"address": "10.9.0.2"}, "keys": map[string]any{"overlay": "dHVubmVs"}}}}, by)
|
||||
return err
|
||||
}
|
||||
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
good := `{"node":"ace","address":"10.9.0.2","ask":"enrol-ace-1","warrant":"enrol-ace-1@2026-10-28T20:00:00Z"}`
|
||||
for _, c := range []struct{ by, warrant, want string }{
|
||||
{"novox.mesh-grants", good, "mesh-enrolment's alone"},
|
||||
{"person.mesh-enrolment", good, "mesh-enrolment's alone"},
|
||||
{"novox.mesh-enrolment", `{"node":"g14","address":"10.9.0.2","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
{"novox.mesh-enrolment", `{"node":"ace","address":"10.9.0.9","ask":"a","warrant":"a@t"}`, "carries the warrant of ace at 10.9.0.9"},
|
||||
{"novox.mesh-enrolment", "", "carries no warrant"},
|
||||
} {
|
||||
if err := addNode(c.by, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("an add-node by %s with warrant %q: %v, want a refusal saying %q", c.by, c.warrant, err, c.want)
|
||||
}
|
||||
}
|
||||
if err := addNode("novox.mesh-enrolment", good); err != nil {
|
||||
t.Errorf("an add-node from mesh-enrolment: %v", err)
|
||||
}
|
||||
// A remove-node (the operator's decision of 2026-11-02 01:05) takes a joining node that never confirmed off placement: from
|
||||
// mesh-enrolment, on the warrant naming that node, of a node running nothing, it is proposed with the node's key to delete;
|
||||
// of a node running modules, or with another node's warrant, it is refused.
|
||||
joiner, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "joiner", Decided: delivery.NodeDecided{Address: "10.9.0.7"}, Modules: []delivery.PlacedModule{}})
|
||||
st[delivery.PlacementState].values["placement.joiner"] = joiner
|
||||
removeNode := func(node, warrant string) (any, error) {
|
||||
return s.doPropose(map[string]any{"why": "enrolment ended", "warrant": warrant,
|
||||
"changes": []any{map[string]any{"op": "remove-node", "node": node}}}, "novox.mesh-enrolment")
|
||||
}
|
||||
for _, c := range []struct{ node, warrant, want string }{
|
||||
{"anchor", `{"node":"anchor","address":"10.9.0.1","ask":"a","warrant":"a@t"}`, "runs modules or holds seats"},
|
||||
{"joiner", `{"node":"g14","address":"10.9.0.7","ask":"a","warrant":"a@t"}`, "carries the warrant of g14"},
|
||||
} {
|
||||
if _, err := removeNode(c.node, c.warrant); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("a remove-node of %s: %v, want a refusal saying %q", c.node, err, c.want)
|
||||
}
|
||||
}
|
||||
if got, err := removeNode("joiner", `{"node":"joiner","address":"10.9.0.7","ask":"a","warrant":"a@t"}`); err != nil {
|
||||
t.Errorf("a remove-node from mesh-enrolment: %v", err)
|
||||
} else if rm := got.(proposed).Proposal; !slices.Equal(rm.Nodes(), []string{"joiner"}) || !slices.Equal(rm.RemovedNodes(), []string{"joiner"}) {
|
||||
t.Errorf("the remove-node proposed %+v", rm)
|
||||
}
|
||||
// 1126/B8: the seat serves its five verbs; adopt and hand-back went with the controller (ADR 0590).
|
||||
if tools := s.tools(); len(tools) != 5 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
|
||||
t.Errorf("the seat's verbs as served: %+v", tools)
|
||||
}
|
||||
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the controller's composition until the lease write, the node's own after; an unplaced node is said", func(t *testing.T) {
|
||||
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the node's own composition; an unplaced node is said", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
|
||||
ckey, _ := delivery.ComposedKey("ace", "web", fp)
|
||||
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedNeither,
|
||||
Policy: catalogue.PolicyRoll, PolicyFrom: catalogue.FromDefault})
|
||||
w.putEntry(delivery.ControllerComposedBucket, ckey, cv, 1, time.Now())
|
||||
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
pkey, _ := delivery.LegacyPlacementKey("ace")
|
||||
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
|
||||
@@ -180,15 +196,11 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
hk, _ := delivery.HaveNodeKey(node)
|
||||
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
|
||||
}
|
||||
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("c", 64) || len(w.unplacedNodes()) != 0 {
|
||||
t.Fatalf("before the lease write: %+v %v, unplaced %v", d, ok, w.unplacedNodes())
|
||||
}
|
||||
w.putEntry(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator"}`), 2, time.Now())
|
||||
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Errorf("after the lease write, ace: %+v %v", d, ok)
|
||||
t.Errorf("ace: %+v %v", d, ok)
|
||||
}
|
||||
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
|
||||
t.Errorf("after the lease write, g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
|
||||
t.Errorf("g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
|
||||
}
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
|
||||
@@ -351,12 +363,11 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
|
||||
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
|
||||
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
|
||||
// the bus is refused; compare lists the modules the controller writes another policy for; the process writes policy-split
|
||||
// for a split and deletes it once they agree; bus names the node behind the indexer's newest.
|
||||
// the bus is refused; compare lists each node's composition and the findings; bus names the node behind the indexer's newest.
|
||||
func wantsVerbs(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
|
||||
st := memStore{}
|
||||
for _, state := range []string{modulesState, manifestsState, composedState, policiesState, haveState, putsState, "findings"} {
|
||||
for _, state := range []string{modulesState, manifestsState, policiesState, haveState, putsState, "findings"} {
|
||||
st[state] = &memKV{values: map[string][]byte{}}
|
||||
}
|
||||
manifest := func(name, extra string) {
|
||||
@@ -371,13 +382,7 @@ func wantsVerbs(t *testing.T) {
|
||||
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
|
||||
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
}
|
||||
composed := func(module, policy string) {
|
||||
key, _ := delivery.ComposedKey("anchor", module, fpNew)
|
||||
st[composedState].values[key], _ = json.Marshal(delivery.ComposedDigest{V: delivery.Version, Digest: "sha256:" + strings.Repeat("c", 64),
|
||||
Judgement: delivery.JudgedNeither, Policy: policy, PolicyFrom: catalogue.FromPerson, PolicyWhy: "chosen at the controller"})
|
||||
}
|
||||
composed("web", catalogue.PolicyRecord)
|
||||
composed("photos", catalogue.PolicyRecord)
|
||||
st[haveState].values["anchor.node"], _ = delivery.EncodeHaveNode(delivery.HaveNode{Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
var asked []string
|
||||
var said []string
|
||||
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -397,10 +402,12 @@ func wantsVerbs(t *testing.T) {
|
||||
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
|
||||
t.Errorf("nats = %+v", p)
|
||||
}
|
||||
// compare: web rolls at the assigner and records at the controller.
|
||||
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
|
||||
// controller being gone (1126/B8).
|
||||
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
|
||||
if splits := out.(map[string]any)["splits"].([]assign.Split); len(splits) != 1 || splits[0].Module != "web" || out.(map[string]any)["findings"] == nil {
|
||||
t.Errorf("splits = %+v", splits)
|
||||
if got := out.(map[string]any); got["splits"] != nil || got["findings"] == nil ||
|
||||
got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) || len(got["composed"].(map[string]string)) != 1 {
|
||||
t.Errorf("compare = %+v", got)
|
||||
}
|
||||
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
|
||||
// large states (the modules, their manifests, the composed record).
|
||||
@@ -422,14 +429,14 @@ func wantsVerbs(t *testing.T) {
|
||||
if werr != nil {
|
||||
t.Fatal(werr)
|
||||
}
|
||||
if ws := wout.(map[string]any)["splits"].([]assign.Split); len(ws) != 1 || ws[0].Module != "web" {
|
||||
t.Errorf("splits by watch = %+v", ws)
|
||||
if got := wout.(map[string]any); got["composed"].(map[string]string)["anchor"] != "sha256:"+strings.Repeat("c", 64) {
|
||||
t.Errorf("compare by watch = %+v", got)
|
||||
}
|
||||
}
|
||||
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
|
||||
t.Errorf("the manifests were read by key %d times, want each once", n)
|
||||
}
|
||||
for _, state := range []string{modulesState, composedState, health.FindingsState} {
|
||||
for _, state := range []string{modulesState, haveState, health.FindingsState} {
|
||||
if watches[state] != 1 {
|
||||
t.Errorf("%s was watched %d times, want once", state, watches[state])
|
||||
}
|
||||
@@ -443,29 +450,7 @@ func wantsVerbs(t *testing.T) {
|
||||
if watches[policiesState] != 0 {
|
||||
t.Errorf("the seat's own small policies state was watched: %v", watches)
|
||||
}
|
||||
// The process says the split as its finding, and deletes it once they agree.
|
||||
findings := map[string][]byte{}
|
||||
p := &passer{w: newWorld(), logf: func(string, ...any) {}, findings: assign.Findings{
|
||||
Put: func(k string, v []byte) error { findings[k] = v; return nil },
|
||||
Delete: func(k string) error { delete(findings, k); return nil },
|
||||
Keys: func() ([]string, error) { return []string{"module.gone.policy-split"}, nil }}}
|
||||
for k, v := range st[composedState].values {
|
||||
p.w.putEntry(delivery.ControllerComposedBucket, k, v, 1, time.Now())
|
||||
}
|
||||
man := map[string]catalogue.Manifest{}
|
||||
for _, name := range []string{"web", "photos"} {
|
||||
raw, _, _, _ := st.get(modulesState, name)
|
||||
mod, _ := delivery.DecodeModule(raw)
|
||||
mraw, _, _, _ := st.get(manifestsState, mod.Manifest)
|
||||
kept, _ := delivery.DecodeManifest(mraw)
|
||||
man[name], _ = catalogue.Decode([]byte(kept.JSON))
|
||||
}
|
||||
findings["module.gone.policy-split"] = []byte("{}")
|
||||
p.policySplits(man, nil)
|
||||
if _, ok := findings[assign.PolicySplitKey("web")]; !ok || len(findings) != 1 {
|
||||
t.Errorf("findings = %v, want web's split alone (a stale one deleted)", findings)
|
||||
}
|
||||
// a person's record for web: written, recorded, said; the split is gone.
|
||||
// a person's record for web: written, recorded, said.
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -474,10 +459,6 @@ func wantsVerbs(t *testing.T) {
|
||||
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
|
||||
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
|
||||
}
|
||||
p.policySplits(man, chosen)
|
||||
if len(findings) != 0 {
|
||||
t.Errorf("findings after the choice = %v", findings)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil {
|
||||
t.Error("the bus was given a policy")
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/handacts"
|
||||
@@ -20,15 +19,14 @@ import (
|
||||
|
||||
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
|
||||
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from
|
||||
// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// (each set recorded as a hand act and said as policy-changed), and with compare lists the digest of the composition each node's
|
||||
// own have names (the controller writes no policy any more, so no module's policy differs); `bus` says what a bus upgrade would do. The assigner's process reads
|
||||
// `policies` when it judges a move and writes the finding policy-split while a split stands.
|
||||
|
||||
// The states the verbs read, as the runtime names them.
|
||||
const (
|
||||
policiesState = delivery.AssignerPoliciesState
|
||||
manifestsState = delivery.IndexerModule + ".manifests"
|
||||
composedState = "mesh-controller.composed"
|
||||
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
|
||||
PolicyChangedEvent = "policy-changed"
|
||||
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
|
||||
@@ -56,13 +54,6 @@ func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error),
|
||||
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
|
||||
}
|
||||
|
||||
// leased says the controller's lease is handed away.
|
||||
func (s *wantsTools) leased() bool {
|
||||
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
|
||||
return away
|
||||
}
|
||||
|
||||
// values reads every key of a state: from the watched copy when there is one, else key by key.
|
||||
func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
if s.watched != nil {
|
||||
@@ -142,7 +133,7 @@ type policyRow struct {
|
||||
}
|
||||
|
||||
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
|
||||
// back with default; with compare list the splits.
|
||||
// back with default; with compare each node's composition and the assigner's findings.
|
||||
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
manifests, _, err := s.manifests()
|
||||
if err != nil {
|
||||
@@ -154,16 +145,20 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
chosen, unreadable := assign.DecodePolicies(raw)
|
||||
if arg(args, "compare") == "true" {
|
||||
composed, err := s.values(composedState)
|
||||
haves, err := s.values(haveState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed))
|
||||
if splits == nil {
|
||||
splits = []assign.Split{}
|
||||
composed := map[string]string{}
|
||||
for key, raw := range haves {
|
||||
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
|
||||
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Composed != "" {
|
||||
composed[node] = h.Composed
|
||||
}
|
||||
}
|
||||
}
|
||||
out := map[string]any{"splits": splits, "empty": len(splits) == 0}
|
||||
// The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15).
|
||||
out := map[string]any{"composed": composed}
|
||||
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
|
||||
if found, err := s.values(health.FindingsState); err != nil {
|
||||
out["findings-unread"] = err.Error()
|
||||
} else {
|
||||
@@ -229,7 +224,7 @@ func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[str
|
||||
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
|
||||
}
|
||||
}
|
||||
rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased}
|
||||
rec := handacts.Recorder{Ask: s.ask}
|
||||
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
|
||||
recorded := taken.Log
|
||||
if rerr != nil {
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
@@ -19,7 +18,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
@@ -28,18 +26,6 @@ import (
|
||||
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
|
||||
const noModuleWords = delivery.CandidateNoneNoModule
|
||||
|
||||
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
|
||||
// internal/view/view.go): an assignment stands while it has not ended and has no state.
|
||||
type assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Build string `json:"build"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||
|
||||
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
|
||||
// digest. The watches write it; a pass reads it.
|
||||
type world struct {
|
||||
@@ -210,38 +196,15 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
|
||||
return p
|
||||
}
|
||||
|
||||
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
|
||||
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
|
||||
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
|
||||
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
|
||||
// the lease write's own check refuses that order (ADR 0547 §11).
|
||||
placed, seeded := w.placed()
|
||||
byAssignment, all := w.assignedByController()
|
||||
if away := w.handedAway(); away && seeded {
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, away)
|
||||
return both, map[string]assignment{}
|
||||
// assigned is each node's modules the seat mesh-placement's placement names, in name order (ADR 0549 §5, ADR 0567 §8): the
|
||||
// placement is the one source of where modules run. The controller's assignments are read no more (issue 774).
|
||||
func (w *world) assigned() map[string][]string {
|
||||
placed, _ := w.placed()
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
}
|
||||
both, _ := assign.Placed(byAssignment, placed, seeded, false)
|
||||
return both, all
|
||||
}
|
||||
|
||||
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
|
||||
// the controller's lease is handed away, since the placement is then the one source.
|
||||
func (w *world) placementDiffers() []assign.Pair {
|
||||
placed, seeded := w.placed()
|
||||
byAssignment, _ := w.assignedByController()
|
||||
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
|
||||
return differ
|
||||
}
|
||||
|
||||
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
|
||||
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
|
||||
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
|
||||
func (w *world) handedAway() bool {
|
||||
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
|
||||
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
|
||||
return away
|
||||
return out
|
||||
}
|
||||
|
||||
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
|
||||
@@ -271,24 +234,6 @@ func (w *world) placedNodes() map[string]bool {
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
|
||||
// name order.
|
||||
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
|
||||
out, all := map[string][]string{}, map[string]assignment{}
|
||||
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||
var a assignment
|
||||
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
|
||||
continue
|
||||
}
|
||||
out[a.Node] = append(out[a.Node], a.Module)
|
||||
all[a.Node+"."+a.Module] = a
|
||||
}
|
||||
for _, mods := range out {
|
||||
sort.Strings(mods)
|
||||
}
|
||||
return out, all
|
||||
}
|
||||
|
||||
// trunk is every module the indexer holds at the trunk, with its value.
|
||||
func (w *world) trunk() map[string]delivery.Module {
|
||||
out := map[string]delivery.Module{}
|
||||
@@ -307,7 +252,7 @@ func (w *world) trunk() map[string]delivery.Module {
|
||||
// mesh is the mesh as the trunk holds it, for the composition check.
|
||||
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Digests: map[string]string{}}
|
||||
m.Assigned, _ = w.assigned()
|
||||
m.Assigned = w.assigned()
|
||||
for name, mod := range w.trunk() {
|
||||
if mod.Manifest == "" {
|
||||
continue
|
||||
@@ -612,49 +557,14 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
//
|
||||
// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a
|
||||
// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets
|
||||
// no want (and the pass says so, unplacedFindings).
|
||||
// composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own
|
||||
// have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The
|
||||
// controller's copy (mesh-controller_composed) is read no more (issue 774).
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
if w.handedAway() {
|
||||
if !w.placedNodes()[node] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return w.ownComposed(node)
|
||||
}
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
if !w.placedNodes()[node] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
if w.handedAway() {
|
||||
return out // the controller composes no node from the lease write (ADR 0571 §6)
|
||||
}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
return w.ownComposed(node)
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
@@ -719,7 +629,7 @@ func (w *world) runningCommits(module string) []string {
|
||||
var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState
|
||||
|
||||
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
|
||||
// what rule 7 reads from the lease write. A node whose have names none is not composed yet.
|
||||
// what rule 7 reads. A node whose have names none is not composed yet.
|
||||
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
@@ -737,12 +647,9 @@ func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
|
||||
Words: "the node composes its own"}, true
|
||||
}
|
||||
|
||||
// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no
|
||||
// unplacedNodes are the nodes whose engine has a have and that no placement names: they get no
|
||||
// want, since nothing composes them.
|
||||
func (w *world) unplacedNodes() []string {
|
||||
if !w.handedAway() {
|
||||
return nil
|
||||
}
|
||||
placed := w.placedNodes()
|
||||
var out []string
|
||||
for key := range w.read(delivery.HaveBucket) {
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
@@ -128,33 +127,22 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
// ADR 0571 clause 9: grants are read from the grants part's bucket in every case, a lease held, handed away, absent or
|
||||
// unreadable alike, and the controller's own grants count for nothing.
|
||||
// ADR 0571 clause 9: grants are read from the grants part's bucket, the one bucket of grants (ADR 0590).
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}})
|
||||
gkey, _ := delivery.PartGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{Accounts: []string{"ace.web"}})
|
||||
for _, row := range []struct {
|
||||
name, lease string
|
||||
inPart, inController bool
|
||||
granted bool
|
||||
name string
|
||||
inPart bool
|
||||
granted bool
|
||||
}{
|
||||
{"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, true},
|
||||
{"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, false},
|
||||
{"a controller holds it, the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, true},
|
||||
{"a controller holds it, only its own grant", `{"instance":"novox/1"}`, false, true, false},
|
||||
{"no lease read", ``, true, false, true},
|
||||
{"an unreadable lease", `[1]`, true, false, true},
|
||||
{"the part's grant", true, true},
|
||||
{"no grant", false, false},
|
||||
} {
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
if row.lease != "" {
|
||||
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease))
|
||||
}
|
||||
if row.inPart {
|
||||
p.w.put(delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
if row.inController {
|
||||
p.w.put(delivery.ControllerGrantsBucket, gkey, grant)
|
||||
}
|
||||
if got := p.w.grantsBucket(); got != delivery.PartGrantsBucket {
|
||||
t.Errorf("%s: grants are read from %s, want the part's", row.name, got)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,23 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364 h1:UUDPWfUjyerptar+wPwI891jTUAtbbuXori0j3phTE4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3 h1:lb6hz+SBsoCAzsiCCvdtPKyE66Y5Ng/DBAkonAexPzA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101025153-c77d17fb42e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -78,7 +78,7 @@ func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
func TestTheAssignersConditions(t *testing.T) {
|
||||
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
|
||||
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only until the take-over", saidConditionsAreFindings)
|
||||
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only with no findings bucket", saidConditionsAreFindings)
|
||||
}
|
||||
|
||||
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
@@ -106,8 +106,9 @@ func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear; it is said straight to the router while the take-over
|
||||
// key follows the controller, and only as a finding from the take-over.
|
||||
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear, and only as a finding: the keeper is the one
|
||||
// source since the controller's end (novox/hq ADR 0590 §3, issue 1126 B8). Said straight to the router only where the
|
||||
// findings bucket cannot be written to at all.
|
||||
func saidConditionsAreFindings(t *testing.T) {
|
||||
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
@@ -117,29 +118,29 @@ func saidConditionsAreFindings(t *testing.T) {
|
||||
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
|
||||
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
|
||||
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
|
||||
{PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"},
|
||||
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
|
||||
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
|
||||
} {
|
||||
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
|
||||
keeper := false
|
||||
f := Findings{Put: func(k string, v []byte) error {
|
||||
got, err := health.DecodeFinding(v)
|
||||
puts[k] = got
|
||||
return err
|
||||
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, ToKeeper: func() bool { return keeper }, Now: func() time.Time { return at }}
|
||||
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
|
||||
say := f.Say(func(asks.Condition) error { said++; return nil })
|
||||
if err := say(c.raised); err != nil {
|
||||
t.Fatalf("%s: %v", c.key, err)
|
||||
}
|
||||
got, ok := puts[c.key]
|
||||
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
|
||||
got.Subject.Machine != c.raised.Subject.Machine || said != 1 {
|
||||
got.Subject.Machine != c.raised.Subject.Machine || said != 0 {
|
||||
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
|
||||
}
|
||||
keeper = true
|
||||
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 1 {
|
||||
t.Fatalf("%s: deleted %v, said %d after the take-over (%v)", c.key, deletes, said, err)
|
||||
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 0 {
|
||||
t.Fatalf("%s: deleted %v, said %d (%v)", c.key, deletes, said, err)
|
||||
}
|
||||
if err := (Findings{}).Say(func(asks.Condition) error { said++; return nil })(c.raised); err != nil || said != 1 {
|
||||
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
|
||||
|
||||
@@ -68,33 +68,33 @@ func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
|
||||
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
|
||||
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
|
||||
// consumer's machine is touched when its provider changes.
|
||||
func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
|
||||
v := Compose(meshOf(), c, "sdk", 0)
|
||||
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
|
||||
m := meshOf()
|
||||
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
|
||||
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
|
||||
v := Compose(m, c, "sdk", 0)
|
||||
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
|
||||
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
|
||||
// 1297/AC2 and B3c: a change that removes a module a machine runs fails, naming the machine and the module; a machine that
|
||||
// composes on the trunk and not with the change fails it, naming the machine and why (the consumer's machine is touched when
|
||||
// its provider changes); what was already broken on the trunk is said and fails nothing. One table (three tests merged).
|
||||
func TestAChangeThatBreaksAMachineFailsAndWhatWasBrokenFailsNothing(t *testing.T) {
|
||||
broken := meshOf()
|
||||
broken.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
mesh Mesh
|
||||
c Change
|
||||
state string
|
||||
must func(delivery.CompositionVerdict) bool
|
||||
}{
|
||||
{"removing a module a machine runs", meshOf(), Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}, delivery.CompositionFail,
|
||||
func(v delivery.CompositionVerdict) bool {
|
||||
return v.Machine == "g14" && strings.Contains(v.Words, "g14 runs web")
|
||||
}},
|
||||
{"a machine the change breaks", meshOf(), Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}},
|
||||
delivery.CompositionFail, func(v delivery.CompositionVerdict) bool {
|
||||
return v.Machine == "ace" && strings.Contains(v.Words, "database")
|
||||
}},
|
||||
{"what was broken before", broken, Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}},
|
||||
delivery.CompositionPass, func(v delivery.CompositionVerdict) bool { return hasWarning(v, "on the trunk either") }},
|
||||
} {
|
||||
if v := Compose(c.mesh, c.c, "sdk", 0); v.State != c.state || !c.must(v) {
|
||||
t.Errorf("%s: composition = %+v", c.name, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,9 +23,9 @@ const (
|
||||
// composer is built at its repository's root, at the mesh-sdk its go.mod pins.
|
||||
const SDKDir = "go"
|
||||
|
||||
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the controller composes and sends
|
||||
// them, the node-engine applies them. Each is judged at its repository.
|
||||
var ReaderModules = map[string]string{"mesh-controller": "novox/mesh-controller", "mesh-host": "novox/mesh-host"}
|
||||
// The modules whose mesh-sdk reads the manifests a plan sends (ADR 0482 clause 5): the node-engine composes and applies
|
||||
// them, the controller being gone (novox/hq ADR 0590, issue 1126). Each is judged at its repository.
|
||||
var ReaderModules = map[string]string{"mesh-host": "novox/mesh-host"}
|
||||
|
||||
// HeadJob is the compose job of a group with a mesh-sdk member (ADR 0482 clauses 3 and 4, issue 1622): the composer
|
||||
// built at the member's head composes the machines the change touches, as the trunk holds them and as the change leaves
|
||||
|
||||
@@ -59,18 +59,17 @@ func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised
|
||||
return key, value, true, err
|
||||
}
|
||||
|
||||
// Findings is where the assigner's findings go: the bucket's put and delete, and whether the keeper's conditions are the
|
||||
// ones said (the take-over key names the keeper).
|
||||
// Findings is where the assigner's findings go: the bucket's put and delete.
|
||||
type Findings struct {
|
||||
Put func(key string, value []byte) error
|
||||
Delete func(key string) error
|
||||
Keys func() ([]string, error)
|
||||
ToKeeper func() bool
|
||||
Now func() time.Time
|
||||
Put func(key string, value []byte) error
|
||||
Delete func(key string) error
|
||||
Keys func() ([]string, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Say is say with each condition also written as a finding (or its finding deleted), and said straight to the router only
|
||||
// while the take-over key follows the controller. A finding that cannot be written is the error, beside the say's.
|
||||
// Say is say with each condition written as a finding (or its finding deleted), which the condition keeper carries, the
|
||||
// one source since the controller's end (novox/hq ADR 0590 §3); said straight to the router only where there is no
|
||||
// findings bucket at all. A finding that cannot be written is the error.
|
||||
func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
var errs []error
|
||||
@@ -93,7 +92,7 @@ func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error
|
||||
}
|
||||
}
|
||||
}
|
||||
if f.ToKeeper == nil || !f.ToKeeper() {
|
||||
if f.Put == nil {
|
||||
if say != nil {
|
||||
if err := say(c); err != nil {
|
||||
errs = append(errs, err)
|
||||
|
||||
@@ -1,122 +1,12 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
|
||||
// the seat mesh-placement's placement (ADR 0549 §5, ADR 0567 §8). The assigner writes wants only for a pair both name:
|
||||
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
|
||||
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
|
||||
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
|
||||
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
|
||||
// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease
|
||||
// handed away) the placement is the one source.
|
||||
|
||||
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
|
||||
// OnlyPlacement.
|
||||
type Pair struct {
|
||||
Node, Module, Only string
|
||||
// Call is what became of the assigner's call of the controller's verb for the pair, in words, when it made one.
|
||||
Call string
|
||||
}
|
||||
|
||||
// The source that names a pair alone.
|
||||
const (
|
||||
OnlyAssignments = "the controller's assignments"
|
||||
OnlyPlacement = "the seat mesh-placement's placement"
|
||||
)
|
||||
|
||||
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
|
||||
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's
|
||||
// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the
|
||||
// assignments are then a source of nothing.
|
||||
func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) {
|
||||
if handedAway && seeded {
|
||||
out := make(map[string][]string, len(placed))
|
||||
for node, modules := range placed {
|
||||
out[node] = slices.Sorted(slices.Values(modules))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
if !seeded {
|
||||
return assigned, nil
|
||||
}
|
||||
set := func(m map[string][]string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for node, modules := range m {
|
||||
for _, module := range modules {
|
||||
out[node+"\x00"+module] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
inAssigned, inPlaced := set(assigned), set(placed)
|
||||
both := map[string][]string{}
|
||||
var differ []Pair
|
||||
for node, modules := range assigned {
|
||||
for _, module := range modules {
|
||||
if inPlaced[node+"\x00"+module] {
|
||||
both[node] = append(both[node], module)
|
||||
} else {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
|
||||
}
|
||||
}
|
||||
}
|
||||
for node, modules := range placed {
|
||||
for _, module := range modules {
|
||||
if !inAssigned[node+"\x00"+module] {
|
||||
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, modules := range both {
|
||||
sort.Strings(modules)
|
||||
}
|
||||
sort.Slice(differ, func(i, j int) bool {
|
||||
if differ[i].Node != differ[j].Node {
|
||||
return differ[i].Node < differ[j].Node
|
||||
}
|
||||
return differ[i].Module < differ[j].Module
|
||||
})
|
||||
return both, differ
|
||||
}
|
||||
|
||||
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
|
||||
// both name it or neither does.
|
||||
func PlacementCondition(p Pair, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
|
||||
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("the controller's assignments and the seat's placement agree on %s on %s.", p.Module, p.Node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
var needs string
|
||||
if p.Only == OnlyAssignments {
|
||||
needs = fmt.Sprintf("propose placing %s on %s (mesh-placement.propose), or unassign it at the controller", p.Module, p.Node)
|
||||
} else {
|
||||
needs = fmt.Sprintf("assign %s to %s at the controller, as the approved placement says, or propose taking it off", p.Module, p.Node)
|
||||
}
|
||||
if p.Call != "" {
|
||||
needs += "; the assigner's call: " + p.Call
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
|
||||
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
|
||||
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
|
||||
}
|
||||
|
||||
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
|
||||
// §1): raised naming each, cleared when none is left.
|
||||
func SeatWaitsCondition(unbound []string) asks.Condition {
|
||||
|
||||
@@ -17,8 +17,6 @@ import (
|
||||
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
|
||||
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
|
||||
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled.
|
||||
// Until the lease write the controller still writes a policy beside each composition's digest; where it differs from the
|
||||
// assigner's, the assigner says so as its finding `policy-split`, naming mesh-wants.upgrade as the remedy.
|
||||
|
||||
// PoliciesBucket is the assigner's state `policies`.
|
||||
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
|
||||
@@ -48,26 +46,6 @@ func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (p
|
||||
return catalogue.Upgrade(m, c)
|
||||
}
|
||||
|
||||
// ControllerPolicies is, by module, every policy the controller writes beside a composition's digest of it.
|
||||
func ControllerPolicies(composed map[string][]byte) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for key, raw := range composed {
|
||||
_, module, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(raw)
|
||||
if err != nil || d.Policy == "" {
|
||||
continue
|
||||
}
|
||||
if !contains(out[module], d.Policy) {
|
||||
out[module] = append(out[module], d.Policy)
|
||||
sort.Strings(out[module])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
@@ -77,64 +55,6 @@ func contains(list []string, s string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Split is a module whose policy at the assigner differs from one the controller writes beside a composition of it.
|
||||
type Split struct {
|
||||
Module string `json:"module"`
|
||||
Policy string `json:"policy"`
|
||||
From string `json:"from"`
|
||||
Controller []string `json:"controller"`
|
||||
}
|
||||
|
||||
// Splits is every module of manifests the controller writes a policy for that differs from the assigner's, sorted.
|
||||
func Splits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy, controller map[string][]string) []Split {
|
||||
var out []Split
|
||||
for module, theirs := range controller {
|
||||
m, ok := manifests[module]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
policy, from, _, _ := PolicyOf(m, chosen)
|
||||
for _, p := range theirs {
|
||||
if p != policy {
|
||||
out = append(out, Split{Module: module, Policy: policy, From: from, Controller: theirs})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// KindPolicySplit is the assigner's finding of a split policy (ADR 0565 §8).
|
||||
const KindPolicySplit = "policy-split"
|
||||
|
||||
// PolicySplitKey is a module's policy-split finding's key.
|
||||
func PolicySplitKey(module string) string {
|
||||
return health.FindingKey(health.ScopeModule, module, KindPolicySplit)
|
||||
}
|
||||
|
||||
// PolicySplitFinding is the finding of one split, in mesh-sdk's layout.
|
||||
func PolicySplitFinding(s Split, at time.Time) ([]byte, error) {
|
||||
summary := fmt.Sprintf("%s's upgrade policy is %s at the assigner (%s) and %s at the controller", s.Module, s.Policy, s.From,
|
||||
strings.Join(s.Controller, " and "))
|
||||
return health.EncodeFinding(health.Finding{Key: PolicySplitKey(s.Module), Kind: KindPolicySplit,
|
||||
Subject: health.Subject{Scope: health.ScopeModule, ID: s.Module}, Severity: health.Warning, Summary: summary,
|
||||
Headline: clipHeadline(s.Module + "'s upgrade policy is split"),
|
||||
Needs: "set it with mesh-wants.upgrade, so the assigner and the controller agree",
|
||||
Explanation: asks.NeedsYou + "set it with mesh-wants.upgrade, so the assigner and the controller agree." + " The controller still " +
|
||||
"writes a policy beside each composition until it hands its lease away; a choice made at its upgrade verb after the copy " +
|
||||
"is not the assigner's.",
|
||||
Resolved: s.Module + "'s upgrade policy agrees again", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindPolicySplit, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func clipHeadline(s string) string {
|
||||
if len([]rune(s)) <= 60 {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:60])
|
||||
}
|
||||
|
||||
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
|
||||
const KindProvisionFailed = "provision-failed"
|
||||
|
||||
@@ -206,3 +126,10 @@ func UnplacedFinding(node string, at time.Time) ([]byte, error) {
|
||||
Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
|
||||
Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()})
|
||||
}
|
||||
|
||||
func clipHeadline(s string) string {
|
||||
if len([]rune(s)) <= 60 {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:60])
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@ func StepPut(w PutWorld) PutNext {
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its placement first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -19,30 +19,25 @@ import (
|
||||
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
|
||||
// digest.
|
||||
type ProposeIn struct {
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
Seed bool
|
||||
ExportDigest string
|
||||
// Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk
|
||||
// buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then
|
||||
// the holder keeps the controller in step and refuses what its verbs cannot take.
|
||||
Leased bool
|
||||
ID, By, Why string
|
||||
At time.Time
|
||||
Changes []delivery.PlacementProposalChange
|
||||
Current map[string]delivery.Placement
|
||||
Revisions map[string]uint64
|
||||
Held func(module string) bool
|
||||
Recorded func(node string) bool
|
||||
// Warrant is the enrolment's warrant an add-node carries (ADR 0587 §7); only that change has one.
|
||||
Warrant *delivery.AddNodeWarrant
|
||||
}
|
||||
|
||||
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change the
|
||||
// controller's verbs cannot take before the lease write (§8), a change that does not fit its node, and what
|
||||
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change that does not fit its node, and what
|
||||
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
|
||||
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
|
||||
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
|
||||
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
|
||||
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
|
||||
Changes: in.Changes, Seed: in.Seed, ExportDigest: in.ExportDigest, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
Changes: in.Changes, Warrant: in.Warrant, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
|
||||
var findings []string
|
||||
if p.By == "" {
|
||||
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
|
||||
@@ -54,18 +49,13 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla
|
||||
findings = append(findings, "a proposal changes something: it names no change")
|
||||
}
|
||||
for i, c := range in.Changes {
|
||||
if (c.Op == delivery.OpSeed) != in.Seed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
|
||||
continue
|
||||
}
|
||||
if !in.Leased {
|
||||
if why := c.BeforeLease(); why != "" {
|
||||
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
|
||||
}
|
||||
if c.Op == delivery.OpSeed {
|
||||
findings = append(findings, fmt.Sprintf("change %d: the seed is gone with adopt, the controller's export being gone", i+1))
|
||||
}
|
||||
}
|
||||
laid, lay := delivery.LayChanges(in.Current, in.Changes)
|
||||
findings = append(findings, lay...)
|
||||
findings = append(findings, ControllerRefusals(in.Changes, laid)...)
|
||||
all := map[string]delivery.Placement{}
|
||||
for node, pl := range in.Current {
|
||||
all[node] = pl
|
||||
@@ -80,6 +70,16 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla
|
||||
out[node] = &pl
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
// A remove-node deletes the key of a joining node that never confirmed (the operator's decision of 2026-11-02 01:05): its
|
||||
// candidate is no placement, and the checks across nodes read the mesh without it. LayChanges said why one cannot go.
|
||||
for _, node := range p.RemovedNodes() {
|
||||
if _, ok := in.Current[node]; !ok {
|
||||
continue
|
||||
}
|
||||
delete(all, node)
|
||||
out[node] = nil
|
||||
p.Base[node] = in.Revisions[node]
|
||||
}
|
||||
if len(findings) == 0 && len(out) == 0 {
|
||||
findings = append(findings, "the changes leave every placement as it is")
|
||||
}
|
||||
@@ -90,12 +90,55 @@ func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Pla
|
||||
if recorded == nil {
|
||||
recorded = func(string) bool { return false }
|
||||
}
|
||||
// A joining node writes its record only once it holds its account, which the add-node's write gives it: this one change by
|
||||
// this one caller is the exception to a node with no record (ADR 0587 §7, changing ADR 0567 §2).
|
||||
joining := map[string]bool{}
|
||||
for _, c := range in.Changes {
|
||||
if c.Op == delivery.OpAddNode {
|
||||
joining[c.Node] = true
|
||||
}
|
||||
}
|
||||
if len(joining) > 0 {
|
||||
was := recorded
|
||||
recorded = func(node string) bool { return joining[node] || was(node) }
|
||||
}
|
||||
for _, r := range delivery.CheckPlacements(all, held, recorded) {
|
||||
findings = append(findings, r)
|
||||
}
|
||||
return p, out, findings
|
||||
}
|
||||
|
||||
// ControllerModule is the controller's module, which nothing places again (issue 1126).
|
||||
const ControllerModule = "mesh-controller"
|
||||
|
||||
// ControllerEnded is why a placement of the controller is refused: the parts that acted only once its lease was handed
|
||||
// (consumers, the user list, enrolment, dead letters) now act without reading the lease, so a controller placed again
|
||||
// would write beside them, silently.
|
||||
const ControllerEnded = "the controller ended by its unassignment on 2026-11-02; placing it again would make two writers (issue 1126)"
|
||||
|
||||
// ControllerRefusals names every change, and every node as laid, that places the controller: by a place, by a seed's
|
||||
// whole placement, or in what the changes lay. None when nothing does.
|
||||
func ControllerRefusals(changes []delivery.PlacementProposalChange, laid map[string]delivery.Placement) []string {
|
||||
var out []string
|
||||
nodes := map[string]bool{}
|
||||
for i, c := range changes {
|
||||
places := c.Module == ControllerModule && c.Op != delivery.OpTakeOff
|
||||
if c.Whole != nil && slices.Contains(c.Whole.Assigned(), ControllerModule) {
|
||||
places = true
|
||||
}
|
||||
if places {
|
||||
out = append(out, fmt.Sprintf("change %d places %s on %s: %s", i+1, ControllerModule, c.Node, ControllerEnded))
|
||||
nodes[c.Node] = true
|
||||
}
|
||||
}
|
||||
for _, node := range slices.Sorted(maps.Keys(laid)) {
|
||||
if !nodes[node] && slices.Contains(laid[node].Assigned(), ControllerModule) {
|
||||
out = append(out, fmt.Sprintf("%s would run %s: %s", node, ControllerModule, ControllerEnded))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Write is one node a warrant writes (§5): the node, its node key `<node>.node` (the commit of the node's keys,
|
||||
// mesh-sdk delivery.PlacementKeys.Writes), the revision of the node's placement it is written at (0: it has none, so
|
||||
// it is created) and its value (nil: it is deleted).
|
||||
@@ -118,112 +161,19 @@ func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) (
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ControllerCall is one call of the controller's seat verbs the holder makes after writing an approved proposal, until
|
||||
// the lease write (§8): assign, unassign, pin or unpin, with its arguments.
|
||||
type ControllerCall struct {
|
||||
Verb string
|
||||
Args map[string]any
|
||||
// Node and Module name the pair the call keeps in step (a pin names no module).
|
||||
Node, Module string
|
||||
}
|
||||
|
||||
// Key names the call, as the condition of its pair says it.
|
||||
func (c ControllerCall) Key() string {
|
||||
return fmt.Sprintf("mesh-controller.%s %s %s", c.Verb, c.Node, c.Module)
|
||||
}
|
||||
|
||||
// ControllerCalls is every call that brings the controller's assignments and pins to the placement after: assign for a
|
||||
// module placed on a node before not, unassign for one taken off, pin for a pin set or changed, unpin for one removed. In
|
||||
// node order, unassigns before assigns, so a module moved between nodes is first taken off.
|
||||
//
|
||||
// Each call carries only the arguments the controller's seat contract declares for its verb (catalogue.ControllerVerbs):
|
||||
// the controller refuses any other argument and does nothing (issue 774: a "why" added to every call had every approved
|
||||
// placement refused since 2026-10-28). The proposal and its warrant are in the assigner's own record instead: the
|
||||
// placement-changed event and the hand-act log.
|
||||
func ControllerCalls(before map[string]delivery.Placement, after map[string]*delivery.Placement) []ControllerCall {
|
||||
nodes := make([]string, 0, len(after))
|
||||
for node := range after {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var off, on, pins []ControllerCall
|
||||
for _, node := range nodes {
|
||||
was, now := before[node], delivery.Placement{}
|
||||
if a := after[node]; a != nil {
|
||||
now = *a
|
||||
}
|
||||
had, has := map[string]bool{}, map[string]bool{}
|
||||
for _, m := range was.Modules {
|
||||
had[m.Module] = true
|
||||
}
|
||||
for _, m := range now.Modules {
|
||||
has[m.Module] = true
|
||||
if !had[m.Module] {
|
||||
on = append(on, ControllerCall{Verb: "assign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, m := range was.Modules {
|
||||
if !has[m.Module] {
|
||||
off = append(off, ControllerCall{Verb: "unassign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(now.Pins) {
|
||||
to := now.Pins[provision]
|
||||
if old, ok := was.Pins[provision]; !ok || old != to {
|
||||
pins = append(pins, ControllerCall{Verb: "pin", Node: node, Module: provision,
|
||||
Args: map[string]any{"node": node, "provision": provision, "from": to.Node, "module": to.Module}})
|
||||
}
|
||||
}
|
||||
for _, provision := range sortedKeys(was.Pins) {
|
||||
if _, ok := now.Pins[provision]; !ok {
|
||||
pins = append(pins, ControllerCall{Verb: "unpin", Node: node, Module: provision, Args: map[string]any{"node": node, "provision": provision}})
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(append(off, on...), pins...)
|
||||
}
|
||||
|
||||
// CallKind is the kind of the condition of a refused call: one for a module's place (assign, unassign), one for a pin.
|
||||
func (c ControllerCall) CallKind() string {
|
||||
if c.Verb == "pin" || c.Verb == "unpin" {
|
||||
return KindControllerPin
|
||||
}
|
||||
return KindControllerCall
|
||||
}
|
||||
|
||||
// The kinds of a refused controller call's condition (and finding).
|
||||
// The kinds of the condition (and finding) of a refused controller call, which the assigner raised until it stopped calling
|
||||
// the controller's verbs (ADR 0567 §8, issue 774).
|
||||
const (
|
||||
KindControllerCall = "controller-call"
|
||||
KindControllerPin = "controller-pin"
|
||||
)
|
||||
|
||||
// CallConditionKey is the key of the condition of a call refused for its pair (node and module, or node and provision).
|
||||
func CallConditionKey(c ControllerCall) string {
|
||||
return delivery.AssignerModule + "." + c.CallKind() + "." + c.Node + "." + c.Module
|
||||
}
|
||||
|
||||
// CallRefusedCondition is the condition the assigner raises when the controller refused (or did not answer) a call that
|
||||
// keeps its assignments in step with an approved placement (ADR 0567 §8, issue 774): the placement is written, the
|
||||
// controller's assignments are not, so nothing new reaches the node (no bus account, no composition). It names the verb,
|
||||
// the node, the module (or provision) and the controller's words, keyed by the pair, and is cleared (raised false) when a
|
||||
// later call for the pair is answered, or both sources agree on it, with resolved saying which.
|
||||
func CallRefusedCondition(c ControllerCall, refusal string, raised bool, resolved string) asks.Condition {
|
||||
key := CallConditionKey(c)
|
||||
subject := asks.Scope{Scope: "module", ID: c.Module, Machine: c.Node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: resolved}
|
||||
}
|
||||
headline := fmt.Sprintf("the controller refused %s %s on %s", c.Verb, c.Module, c.Node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
needs := fmt.Sprintf("call mesh-controller.%s for %s on %s by hand once the refusal's cause is fixed", c.Verb, c.Module, c.Node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. An approved placement was written, and the assigner's call %s to keep the controller's "+
|
||||
"assignments in step was refused: %s. Until the controller holds it, the placement and the assignments differ, so "+
|
||||
"the assigner writes no want for the pair and nothing new reaches %s. This clears when a call for it is answered "+
|
||||
"or both sources agree.", asks.NeedsYou, needs, c.Key(), refusal, c.Node)}
|
||||
// CallClearedCondition clears the condition of a refused controller call a previous process raised, for its pair (node and
|
||||
// module, or node and provision for a pin).
|
||||
func CallClearedCondition(kind, node, module string) asks.Condition {
|
||||
return asks.Condition{Key: delivery.AssignerModule + "." + kind + "." + node + "." + module,
|
||||
Subject: asks.Scope{Scope: "module", ID: module, Machine: node}, State: asks.StateCleared,
|
||||
Resolved: "the assigner calls no verb of the controller any more: placement is the one source."}
|
||||
}
|
||||
|
||||
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
|
||||
@@ -246,9 +196,6 @@ func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking
|
||||
}
|
||||
nodes := p.Nodes()
|
||||
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
|
||||
if p.Seed {
|
||||
headline = "Take placement over from the controller?"
|
||||
}
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
|
||||
}
|
||||
@@ -268,57 +215,6 @@ func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking
|
||||
}, Asking: asking}, nil
|
||||
}
|
||||
|
||||
// ExportOf reads the controller's answer to placement-export (§7): its verb answers what the command printed, the
|
||||
// JSON of every node's placement, in `output` (the controller's verbAnswer; `answer` when it carries it as data). A
|
||||
// refused or unreadable answer is an error naming it.
|
||||
func ExportOf(raw []byte) (map[string]delivery.Placement, error) {
|
||||
var a struct {
|
||||
Output string `json:"output"`
|
||||
OK *bool `json:"ok"`
|
||||
Answer json.RawMessage `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export answered no verb answer: %w", err)
|
||||
}
|
||||
if a.OK != nil && !*a.OK {
|
||||
return nil, fmt.Errorf("the controller's placement-export refused: %s", strings.TrimSpace(a.Output))
|
||||
}
|
||||
var byNode map[string]json.RawMessage
|
||||
if len(a.Answer) > 0 && string(a.Answer) != "null" {
|
||||
if err := json.Unmarshal(a.Answer, &byNode); err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export is no placement by node: %w", err)
|
||||
}
|
||||
} else if err := json.NewDecoder(strings.NewReader(a.Output)).Decode(&byNode); err != nil {
|
||||
// The command prints its JSON first; what its standard error said after it is left.
|
||||
return nil, fmt.Errorf("the controller's placement-export printed no placement by node: %w", err)
|
||||
}
|
||||
out := map[string]delivery.Placement{}
|
||||
for node, rawP := range byNode {
|
||||
p, err := delivery.DecodePlacement(rawP)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's placement-export of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if p.Node != node {
|
||||
return nil, fmt.Errorf("the controller's placement-export names %s under %s", p.Node, node)
|
||||
}
|
||||
out[node] = p
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("the controller's placement-export names no node")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SeedChanges is the seed's changes: every node's placement of the export, whole, in node order.
|
||||
func SeedChanges(export map[string]delivery.Placement) []delivery.PlacementProposalChange {
|
||||
var out []delivery.PlacementProposalChange
|
||||
for _, node := range sortedKeys(export) {
|
||||
p := export[node]
|
||||
out = append(out, delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: node, Whole: &p})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
|
||||
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
|
||||
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -38,39 +37,35 @@ func TestThePlacementSeatDecides(t *testing.T) {
|
||||
in ProposeIn
|
||||
want string
|
||||
}{
|
||||
"a port before the lease": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}}), "lease write"},
|
||||
"a holding": {in(delivery.PlacementProposalChange{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "mesh-store", Scope: "mesh"}), "lease write"},
|
||||
"a binding": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}), "lease write"},
|
||||
"a config place": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}), "lease write"},
|
||||
"a config access": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}}), "lease write"},
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "adopt's alone"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
|
||||
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
|
||||
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
|
||||
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
|
||||
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
|
||||
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "gone with adopt"},
|
||||
// 1126: the controller ended by its unassignment; placed again it would be a second writer beside the parts that
|
||||
// act without its lease.
|
||||
"the controller placed again": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "mesh-controller"}), "ended by its unassignment"},
|
||||
"the controller seeded whole": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "anchor", Whole: &delivery.Placement{V: delivery.Version, Node: "anchor",
|
||||
Decided: anchor.Decided, Modules: []delivery.PlacedModule{{Module: "mesh-controller"}}}}), "two writers (issue 1126)"},
|
||||
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
|
||||
} {
|
||||
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
|
||||
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's config values may be proposed before the lease write (§13); once the lease is written every change is
|
||||
// the state's alone, and none waits.
|
||||
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
|
||||
t.Errorf("config values before the lease write: %v", f)
|
||||
}
|
||||
// Every kind of change is taken: the placement is the one source, and the controller's verbs limit nothing (ADR 0567 §8).
|
||||
for _, c := range []delivery.PlacementProposalChange{
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}},
|
||||
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
|
||||
{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}},
|
||||
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
|
||||
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
|
||||
} {
|
||||
leased := in(c)
|
||||
leased.Leased = true
|
||||
if _, _, f := Propose(leased); len(f) != 0 {
|
||||
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
|
||||
if _, _, f := Propose(in(c)); len(f) != 0 {
|
||||
t.Errorf("%s is refused: %v", c.Op, f)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,32 +87,10 @@ func TestThePlacementSeatDecides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126: the seed places the private network, which no repository holds; it is the mesh's own, and passes.
|
||||
wired := laptop
|
||||
wired.Modules = append(slices.Clone(laptop.Modules), delivery.PlacedModule{Module: "mesh-wireguard"})
|
||||
seed := in(SeedChanges(map[string]delivery.Placement{"laptop": wired})...)
|
||||
seed.Seed, seed.ExportDigest, seed.Current, seed.Revisions = true, "sha256:x", map[string]delivery.Placement{}, map[string]uint64{}
|
||||
seed.Held = func(m string) bool { return m != "mesh-wireguard" } // the indexer holds every module but it
|
||||
if _, _, f := Propose(seed); len(f) != 0 {
|
||||
t.Errorf("a seed placing the private network is refused: %v", f)
|
||||
}
|
||||
|
||||
writes, err := Writes(p, laid)
|
||||
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
|
||||
t.Fatalf("writes = %+v, %v", writes, err)
|
||||
}
|
||||
pinned := *laid["laptop"]
|
||||
pinned.Pins = map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
|
||||
calls := ControllerCalls(map[string]delivery.Placement{"anchor": anchor, "laptop": laptop},
|
||||
map[string]*delivery.Placement{"anchor": laid["anchor"], "laptop": &pinned})
|
||||
var said []string
|
||||
for _, c := range calls {
|
||||
said = append(said, c.Key())
|
||||
}
|
||||
if !slices.Equal(said, []string{"mesh-controller.unassign anchor web", "mesh-controller.assign laptop web", "mesh-controller.pin laptop db"}) {
|
||||
t.Errorf("the controller's calls = %v", said)
|
||||
}
|
||||
|
||||
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
|
||||
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
@@ -127,112 +100,4 @@ func TestThePlacementSeatDecides(t *testing.T) {
|
||||
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
|
||||
t.Error("the ask's yes binds changes it was not asked about")
|
||||
}
|
||||
|
||||
export, _ := json.Marshal(map[string]any{"anchor": anchor, "laptop": laptop})
|
||||
for name, c := range map[string]struct {
|
||||
raw string
|
||||
want int
|
||||
}{
|
||||
"in output": {`{"ok":true,"output":` + quote(string(export)+"\n") + `}`, 2},
|
||||
"as answer": {`{"ok":true,"output":"","answer":` + string(export) + `}`, 2},
|
||||
"refused": {`{"ok":false,"output":"no store"}`, 0},
|
||||
"no node": {`{"ok":true,"output":"{}"}`, 0},
|
||||
"no version": {`{"ok":true,"output":"{\"anchor\":{\"node\":\"anchor\",\"modules\":[]}}"}`, 0},
|
||||
} {
|
||||
got, err := ExportOf([]byte(c.raw))
|
||||
if len(got) != c.want || (c.want == 0) != (err != nil) {
|
||||
t.Errorf("%s: %d nodes, %v", name, len(got), err)
|
||||
}
|
||||
}
|
||||
if seed := SeedChanges(map[string]delivery.Placement{"laptop": laptop, "anchor": anchor}); len(seed) != 2 || seed[0].Node != "anchor" || seed[0].Op != delivery.OpSeed {
|
||||
t.Errorf("the seed's changes = %+v", seed)
|
||||
}
|
||||
}
|
||||
|
||||
func quote(s string) string {
|
||||
raw, _ := json.Marshal(s)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// Issue 774: every call the assigner makes of the controller carries only the arguments the controller's seat contract
|
||||
// declares for its verb, and every one it requires; the controller refuses any other argument and does nothing. A "why"
|
||||
// added to every call (2026-10-28) had each approved placement refused.
|
||||
func TestControllerCallsFollowTheContract(t *testing.T) {
|
||||
declared := map[string]map[string]any{}
|
||||
required := map[string][]string{}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
declared[v.Name] = props
|
||||
switch r := v.Input["required"].(type) {
|
||||
case []string:
|
||||
required[v.Name] = r
|
||||
}
|
||||
}
|
||||
db := map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
|
||||
moved := map[string]delivery.Provider{"db": {Node: "laptop", Module: "store"}}
|
||||
placed := func(node string, pins map[string]delivery.Provider, modules ...string) delivery.Placement {
|
||||
p := delivery.Placement{Node: node, Pins: pins}
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
return p
|
||||
}
|
||||
ptr := func(p delivery.Placement) *delivery.Placement { return &p }
|
||||
for name, c := range map[string]struct {
|
||||
before map[string]delivery.Placement
|
||||
after map[string]*delivery.Placement
|
||||
verbs []string
|
||||
}{
|
||||
"assign": {map[string]delivery.Placement{}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil, "web"))}, []string{"assign"}},
|
||||
"unassign": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil))}, []string{"unassign"}},
|
||||
"key deleted": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": nil}, []string{"unassign", "unpin"}},
|
||||
"pin set": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", db, "web"))}, []string{"pin"}},
|
||||
"pin changed": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", moved, "web"))}, []string{"pin"}},
|
||||
"moved and pinned": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil)), "laptop": ptr(placed("laptop", db, "web"))}, []string{"unassign", "assign", "pin"}},
|
||||
} {
|
||||
calls := ControllerCalls(c.before, c.after)
|
||||
var verbs []string
|
||||
for _, call := range calls {
|
||||
verbs = append(verbs, call.Verb)
|
||||
props, ok := declared[call.Verb]
|
||||
if !ok {
|
||||
t.Errorf("%s: the controller's seat has no verb %q", name, call.Verb)
|
||||
continue
|
||||
}
|
||||
for arg := range call.Args {
|
||||
if _, ok := props[arg]; !ok {
|
||||
t.Errorf("%s: %s gives %q, which mesh-controller.%s does not take, so the controller refuses it", name, call.Key(), arg, call.Verb)
|
||||
}
|
||||
}
|
||||
for _, arg := range required[call.Verb] {
|
||||
if _, ok := call.Args[arg]; !ok {
|
||||
t.Errorf("%s: %s lacks %q, which mesh-controller.%s requires", name, call.Key(), arg, call.Verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !slices.Equal(verbs, c.verbs) {
|
||||
t.Errorf("%s: the calls = %v, want %v", name, verbs, c.verbs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 774: a refused call is a condition keyed by its pair, a module's place apart from a pin, naming the verb, the node, the
|
||||
// module and the refusal; its clear says why.
|
||||
func TestCallRefusedCondition(t *testing.T) {
|
||||
assign := ControllerCall{Verb: "assign", Node: "anchor", Module: "web"}
|
||||
pin := ControllerCall{Verb: "pin", Node: "anchor", Module: "web"}
|
||||
c := CallRefusedCondition(assign, `assign takes no argument "why"`, true, "")
|
||||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
|
||||
!strings.Contains(c.Headline, "assign web on anchor") || !strings.Contains(c.Explanation, `takes no argument "why"`) {
|
||||
t.Errorf("the raised condition = %+v", c)
|
||||
}
|
||||
if CallConditionKey(pin) == c.Key {
|
||||
t.Error("a pin's condition shares the key of a module's place")
|
||||
}
|
||||
if cl := CallRefusedCondition(assign, "", false, "answered"); cl.State != asks.StateCleared || cl.Key != c.Key || cl.Resolved != "answered" {
|
||||
t.Errorf("the clear = %+v", cl)
|
||||
}
|
||||
if _, _, raised, err := FindingOf(c, time.Unix(0, 0)); err != nil || !raised {
|
||||
t.Errorf("the condition is no finding: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,8 +39,8 @@ type Module struct {
|
||||
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
||||
Chosen *catalogue.PersonPolicy
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
// Composed answers the composition of the module for a node at a fingerprint: the digest the node's own have names (a
|
||||
// placed node composes its own, ADR 0571 §6), and whether it names one. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
@@ -131,7 +131,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
||||
}
|
||||
if words, waiting := m.provisionWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
@@ -412,7 +412,8 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||
// A placed node's want names the build alone: it is the same target while the build is.
|
||||
if had && !before.Removed && !before.Put && before.FP == target && (before.Composed == d.Digest || (before.Self && m.selfOn(node))) {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
@@ -534,7 +535,7 @@ func (m Module) selfDerived(node string, before delivery.Want, had bool, next de
|
||||
d, there := m.composition(node, next.FP)
|
||||
if !there || d.Digest == "" {
|
||||
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
|
||||
"and the controller has not composed it at %s, so its want stays", m.Name, node, short(next.FP))}
|
||||
"and no composition of its own is named at %s now, so its want stays", m.Name, node, short(next.FP))}
|
||||
return next, false
|
||||
}
|
||||
at := m
|
||||
@@ -732,8 +733,8 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's Grant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
func GrantNamesNode(g delivery.Grant, node, module string) bool {
|
||||
return g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
@@ -293,7 +293,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
||||
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
||||
m, n := ready()
|
||||
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
grant := delivery.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew {
|
||||
@@ -312,7 +312,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
{[]string{"g14.mailer"}, "g14", "mail", false},
|
||||
{[]string{"ace.mail"}, "g14", "mail", false},
|
||||
} {
|
||||
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
if got := GrantNamesNode(delivery.Grant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
||||
}
|
||||
}
|
||||
@@ -815,6 +815,13 @@ func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
||||
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer was erased: %+v", w)
|
||||
}
|
||||
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
|
||||
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
|
||||
m.Self = func(string) bool { return true }
|
||||
placed := map[string]delivery.Want{"ace": {FP: fNew, Self: true, Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
||||
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer in a placed node's want was erased: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.composed", "mesh-controller.lease",
|
||||
want := []string{"mesh-build-go.steps", "mesh-config.kept", "mesh-config.resolved",
|
||||
"mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
@@ -60,14 +60,14 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), says each put's step as the event put, and calls the hand-act seat and the bus node's backup holder.
|
||||
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream
|
||||
// placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its
|
||||
// hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches.
|
||||
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its five verbs (adopt and hand-back went with the
|
||||
// controller, ADR 0590); keeps the seat's events on its stream placement for 30 days; calls the hand-act log; and reads
|
||||
// mesh-config's resolved values to compose every node a proposal touches.
|
||||
// ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus.
|
||||
if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again"}) ||
|
||||
m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) {
|
||||
t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs and mesh-wants with its two", m.Claims)
|
||||
t.Errorf("the assigner claims %+v, want mesh-placement with its five verbs and mesh-wants with its two", m.Claims)
|
||||
}
|
||||
if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil {
|
||||
t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat))
|
||||
@@ -82,8 +82,7 @@ func TestManifest(t *testing.T) {
|
||||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
|
||||
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
|
||||
}
|
||||
if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin",
|
||||
if want := []string{
|
||||
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
|
||||
"seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
|
||||
@@ -11,9 +11,7 @@
|
||||
"proposals",
|
||||
"propose",
|
||||
"withdraw",
|
||||
"again",
|
||||
"adopt",
|
||||
"hand-back"
|
||||
"again"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -27,12 +25,8 @@
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-conditions.takeover",
|
||||
"mesh-config.kept",
|
||||
"mesh-config.resolved",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.lease",
|
||||
"mesh-delivery-builder.build-log",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
@@ -49,11 +43,6 @@
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.assign",
|
||||
"seat:mesh-controller.pin",
|
||||
"seat:mesh-controller.placement-export",
|
||||
"seat:mesh-controller.unassign",
|
||||
"seat:mesh-controller.unpin",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-hand-acts.hand-acts",
|
||||
"seat:mesh-hand-acts.warranted",
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -19,7 +20,7 @@ import (
|
||||
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
|
||||
// controller had never taken in, and the builder kept them done.
|
||||
|
||||
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
|
||||
// heldVerb answers the seat mesh-artifact-sweep's verb held on the bus, as its holder does: {result: {output, ok,
|
||||
// answer}}; a reference is gone while gone says so.
|
||||
type heldVerb struct {
|
||||
mu sync.Mutex
|
||||
@@ -29,7 +30,7 @@ type heldVerb struct {
|
||||
|
||||
func (h *heldVerb) serve(t *testing.T, b *bed) {
|
||||
t.Helper()
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
|
||||
sub, err := b.js.Conn().Subscribe("mesh.seat."+catalogue.ArtifactSweepSeat+".call.held.>", func(m *nats.Msg) {
|
||||
var args struct {
|
||||
References []string `json:"references"`
|
||||
}
|
||||
|
||||
@@ -467,15 +467,20 @@ func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { //
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4).
|
||||
func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
// TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4). A module whose path is "."
|
||||
// is given to the build tools as it is (1615/AC4). One table: the two tests were merged to pay for the enrolment's.
|
||||
func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
|
||||
for _, c := range []struct{ name, path string }{{"no toolchain", "modules/ca-trust"}, {"root", delivery.RootPath}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", c.path, commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -530,14 +535,6 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
|
||||
}
|
||||
}
|
||||
|
||||
// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is.
|
||||
func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", delivery.RootPath, commit)
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
}
|
||||
|
||||
func mustKey(t *testing.T, module string) string {
|
||||
k, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
|
||||
@@ -188,8 +188,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer cc.Stop()
|
||||
}
|
||||
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
|
||||
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
|
||||
// The seat mesh-artifact-sweep says artifacts-gone when it found a key's blob gone (issue 1937, ADR 0571 §2): the
|
||||
// builder's durable consumer on EVENTS, made by the grants part from the manifest's consumes. Where it is not there yet the builder says
|
||||
// so; it still asks held at its start and at every module value.
|
||||
switch cons, err := cred.EventConsumer(ctx, js, eventsStream); {
|
||||
case err != nil:
|
||||
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// eventsStream is the stream every module's and role's events are on, where the controller makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
|
||||
// eventsStream is the stream every module's and role's events are on, where the grants part makes the builder's durable
|
||||
// consumer of artifacts-gone (mesh-sdk buscompose EventsStream).
|
||||
const eventsStream = "EVENTS"
|
||||
|
||||
// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2).
|
||||
@@ -27,8 +27,7 @@ type toolCaller func(key string, body any) (json.RawMessage, error)
|
||||
|
||||
// storedVia is build.Builder's Stored: it asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2, issue 1937)
|
||||
// and answers the references the store no longer holds. One it could not ask about, with none gone, is an error: the key
|
||||
// is then left done, and the controller's take-in still refuses a gone blob. The controller's `held` is not asked: it
|
||||
// is no longer a fallback (issue 1955).
|
||||
// is then left done, and asked again at the next module value or artifacts-gone.
|
||||
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
|
||||
return func(_ context.Context, references []string) ([]string, error) {
|
||||
body := map[string]any{"references": references}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -177,6 +178,42 @@ func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is how long one ask may go on being declined, from its first decline, before its key fails (the
|
||||
// checker's DefaultDeclineBound, issue 890); DefaultDeclinePause is how long the builder waits before it puts a declined
|
||||
// ask again, so a lone node off the agreed toolchain cannot spin on it.
|
||||
const (
|
||||
DefaultDeclineBound = 30 * time.Minute
|
||||
DefaultDeclinePause = 5 * time.Second
|
||||
)
|
||||
|
||||
func (b *Builder) declineBound() time.Duration {
|
||||
if b.DeclineBound > 0 {
|
||||
return b.DeclineBound
|
||||
}
|
||||
return DefaultDeclineBound
|
||||
}
|
||||
|
||||
func (b *Builder) declinePause() time.Duration {
|
||||
if b.DeclinePause != 0 {
|
||||
return max(b.DeclinePause, 0)
|
||||
}
|
||||
return DefaultDeclinePause
|
||||
}
|
||||
|
||||
// declinedWords names the nodes that declined an ask and why, in the order of their names.
|
||||
func declinedWords(by map[string]string) string {
|
||||
nodes := make([]string, 0, len(by))
|
||||
for n := range by {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var parts []string
|
||||
for _, n := range nodes {
|
||||
parts = append(parts, n+": "+by[n])
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
@@ -255,6 +292,8 @@ func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
// errDeclined: a build node on another toolchain than the agreed one sat out of the ask (ADR 0584 clause 2).
|
||||
errDeclined = errors.New("declined")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
@@ -268,11 +307,42 @@ func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, ask
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
var firstDecline time.Time
|
||||
declines := map[string]string{} // node -> why it declined, the last
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errDeclined):
|
||||
// Not an asking: a node that sits out would use them all while it idles and pulls the ask again at once.
|
||||
// Bounded by time from the first decline, then failed loudly naming the nodes and their reasons (as the
|
||||
// checker's decline loop is, issue 890).
|
||||
now := time.Now()
|
||||
if firstDecline.IsZero() {
|
||||
firstDecline = now
|
||||
}
|
||||
declines[step.Node] = why
|
||||
if bound := b.declineBound(); now.Sub(firstDecline) > bound {
|
||||
return step, ask, fmt.Errorf("the ask %s of %s was declined for %s and no build node took it: %s", ask.ID, l.module, bound.Round(time.Second), declinedWords(declines))
|
||||
}
|
||||
// The pause is a context's bound, not a timer (ADR 0437 §8).
|
||||
pause, stop := context.WithTimeout(ctx, b.declinePause())
|
||||
<-pause.Done()
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
return step, ask, ctx.Err()
|
||||
}
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("put %s again after %s declined it", l.module, step.Node)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s put again after a decline could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s was declined on %s and is put again as %s: %s", key, old, l.module, step.Node, ask.ID, why)
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
@@ -317,6 +387,11 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if _, declined := delivery.DeclinedOf(step); err == nil && declined {
|
||||
// A build node off the agreed toolchain sat out (ADR 0584 clause 2). It is no asking: answer() puts the ask again
|
||||
// after a pause, within DeclineBound.
|
||||
return step, step.Reason, errDeclined
|
||||
}
|
||||
if err == nil && buildrun.IsLost(step) {
|
||||
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
|
||||
// words of why, not at the check bound.
|
||||
|
||||
@@ -67,6 +67,9 @@ type Builder struct {
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
|
||||
// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
|
||||
DeclineBound, DeclinePause time.Duration
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
|
||||
@@ -257,8 +260,8 @@ func (b *Builder) storedOnce(key string) func(ctx context.Context, references []
|
||||
}
|
||||
}
|
||||
|
||||
// ArtifactsGone takes the controller's event artifacts-gone in (issue 1937): a take-in found that the store does not
|
||||
// hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// ArtifactsGone takes the seat mesh-artifact-sweep's event artifacts-gone in (issue 1937, ADR 0571 §2): the sweep found
|
||||
// that the store does not hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
|
||||
// builder: what is built rests on the store's answer, never on the event alone.
|
||||
func (b *Builder) ArtifactsGone(ctx context.Context, e delivery.ArtifactsGone) {
|
||||
@@ -415,7 +418,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
if planErr == nil && !nothing {
|
||||
if why := b.stale(m, plan); why != "" {
|
||||
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
|
||||
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
|
||||
// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
|
||||
b.logf("%s at %s is not built: %s", module, m.FP2, why)
|
||||
return
|
||||
}
|
||||
@@ -506,10 +509,11 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
|
||||
continue // an archive is packed with no toolchain
|
||||
}
|
||||
language := buildrun.LanguageOf[bm]
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
want, ok := m.Toolchains[language]
|
||||
if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
|
||||
@@ -543,7 +547,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
repository := delivery.RepositoryName(m.Repository)
|
||||
var lanes []*lane
|
||||
used := map[string]delivery.ToolchainUsed{}
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
|
||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||
for _, bm := range plan.Modules() {
|
||||
if prior != nil && len(only) > 0 && !only[bm] {
|
||||
kept = append(kept, bm)
|
||||
@@ -553,10 +558,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
|
||||
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
|
||||
language := buildrun.LanguageOf[bm]
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, none)
|
||||
ag := delivery.ToolchainAgreement(b.Toolchains.Entries(language))
|
||||
if ag.Wait != "" {
|
||||
return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
|
||||
}
|
||||
tc := ag.Toolchain
|
||||
sitting = append(sitting, SittingOutWords(language, ag)...)
|
||||
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
||||
}
|
||||
@@ -653,6 +660,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
||||
}
|
||||
}
|
||||
step := "built " + strings.Join(steps, "; ")
|
||||
if len(sitting) > 0 {
|
||||
step += " (" + strings.Join(sitting, "; ") + ")"
|
||||
}
|
||||
if len(used) == 0 {
|
||||
used = nil
|
||||
}
|
||||
@@ -942,3 +952,13 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
|
||||
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
|
||||
return e
|
||||
}
|
||||
|
||||
// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
|
||||
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
|
||||
func SittingOutWords(language string, a delivery.Agreement) []string {
|
||||
var out []string
|
||||
for _, s := range a.SittingOut {
|
||||
out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -223,6 +223,14 @@ var (
|
||||
|
||||
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
|
||||
r := newRig(t, goBundle, archiveA, tsBundle)
|
||||
// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
|
||||
r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false)
|
||||
defer func() {
|
||||
if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") {
|
||||
t.Errorf("the key does not say ace sits out: %s", st)
|
||||
}
|
||||
}()
|
||||
var logged []delivery.BuildLogEntry
|
||||
var lmu sync.Mutex
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
|
||||
@@ -730,6 +738,25 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
})
|
||||
t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.DeclinePause, r.b.DeclineBound = -1, 150*time.Millisecond
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}})
|
||||
waitForAsks(t, r, 1)
|
||||
for ended, deadline := 0, time.Now().Add(5*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn
|
||||
r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined})
|
||||
ended++
|
||||
}
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") {
|
||||
t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings)
|
||||
}
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
askings int
|
||||
|
||||
@@ -112,9 +112,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
|
||||
}
|
||||
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
|
||||
// starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4).
|
||||
// 1955/L7: and the seat mesh-artifact-sweep's, the controller's until the lease write (its take-in sweeps until then).
|
||||
if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
// starts; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the
|
||||
// controller's no longer (1126/B9, ADR 0590).
|
||||
if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
|
||||
@@ -16,8 +16,7 @@
|
||||
"consumes": [
|
||||
"mesh-artifact-sweep.artifacts-gone",
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
"mesh-controller.artifacts-gone"
|
||||
"mesh-build-node.toolchain.reported"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-artifact-sweep.held"
|
||||
|
||||
@@ -189,7 +189,7 @@ func SeatTryTools(tr Trying) []stdio.Tool {
|
||||
func Tools(r check.Reader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: check.VerdictsReadTool,
|
||||
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, what failed first, the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
|
||||
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, every failure it names one line each (every member's failed tests, refused steps and conflicts, the manifests' refusals and the composition), the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
|
||||
"report of the status each pull request carries (the line the forge shows is cut at 140 characters, this is not). With a group, that group's only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
||||
@@ -199,7 +199,7 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
|
||||
}},
|
||||
{Name: check.PullReadTool,
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||
Description: "The verdict that judges one pull request, every failure it names one line each (all of them, not only the first the status line shows), and the full report of its status mesh/check. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
|
||||
|
||||
+1
@@ -36,6 +36,7 @@ func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerd
|
||||
// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not
|
||||
// "not yet run": a pass names its machine, a fail fails the check with the assigner's words.
|
||||
func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) {
|
||||
t.Run("the read awaits the assigner's composition", checkerAwaitsTheAssignersComposition)
|
||||
for _, c := range []struct {
|
||||
state, owed, words string
|
||||
}{
|
||||
|
||||
+2
-1
@@ -21,7 +21,8 @@ import (
|
||||
|
||||
// 1297/AC3 (its bus half): on the real bus, the checker's read of the assigner's composition waits while the key is
|
||||
// absent and while the assigner says it waits, and answers the verdict the assigner then writes under the check's key.
|
||||
func TestTheCheckerAwaitsTheAssignersComposition(t *testing.T) {
|
||||
// A case of TestTheCheckersLineSaysTheAssignersComposition (merged into it for the test budget, issue 774).
|
||||
func checkerAwaitsTheAssignersComposition(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "comp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102153155-43788bbfdef3
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102153155-43788bbfdef3 h1:K3Uw1b+IPAKd8NeGzJNqggQifnEYYv9IUxBFpunx05c=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102153155-43788bbfdef3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -88,7 +88,12 @@ func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
|
||||
if machine == "" {
|
||||
machine = "the composition"
|
||||
}
|
||||
v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: machine, Words: comp.Words}
|
||||
f := delivery.FailedFirst{Machine: machine, Words: comp.Words}
|
||||
if v.State == delivery.VerdictFail {
|
||||
addFailure(v, f) // the tests failed too: the composition is named after them (the operator, 2026-11-02)
|
||||
return
|
||||
}
|
||||
v.State, v.Failed = delivery.VerdictFail, f
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,9 +103,9 @@ func notJudged(v delivery.Verdict) bool {
|
||||
}
|
||||
|
||||
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
|
||||
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (a composition is read
|
||||
// only into a passing verdict), so a later composition of the same heads is taken in on its event, with no new check
|
||||
// (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
|
||||
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (the composition is
|
||||
// its first failure, and a composition is named after every other failure), so a later composition of the same heads is
|
||||
// taken in on its event, with no new check (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
|
||||
func unanswered(v delivery.Verdict) bool {
|
||||
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
|
||||
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
|
||||
@@ -141,7 +146,7 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
|
||||
case v.State == delivery.VerdictPass && notJudged(v):
|
||||
case unanswered(v) && comp.Words != v.Composition.Words:
|
||||
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
|
||||
v.State, v.Failed = delivery.VerdictPass, delivery.FailedFirst{}
|
||||
v.State, v.Failed, v.Failures = delivery.VerdictPass, delivery.FailedFirst{}, nil
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
@@ -33,21 +32,23 @@ const (
|
||||
|
||||
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
|
||||
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
|
||||
// checks/deps). Which commit of each is besideCommit's rule.
|
||||
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
|
||||
// unassignment (novox/hq ADR 0590, issue 1126); its own repository keeps its siblings while it holds a module.
|
||||
var besideRules = map[string][]string{
|
||||
controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository},
|
||||
nodeEngineRepo: {catalogueRepository, SDKRepository, controllerRepo},
|
||||
nodeEngineRepo: {catalogueRepository, SDKRepository},
|
||||
SDKRepository: {catalogueRepository, nodeEngineRepo},
|
||||
catalogueRepository: {SDKRepository, controllerRepo},
|
||||
catalogueRepository: {SDKRepository},
|
||||
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
|
||||
}
|
||||
|
||||
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the catalogue at its trunk, always, even in a
|
||||
// group holding a catalogue pull request; the node-engine at the commit the mesh runs (the commit of the indexer's module
|
||||
// mesh-host, its trunk head when none is known); mesh-sdk, for the controller's tests, at the commit the controller's go.mod
|
||||
// pins; any other sibling at the head of the group's pull request for it when the group holds one, else at its trunk. And
|
||||
// first, the change's own repository at the trunk it is merged onto, so a test comparing a shrink-only list with the
|
||||
// trunk's copy reads it at $MESH_CHECK_BESIDE/<name> (issue 1887 A12, ADR 0529 §2).
|
||||
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the node-engine at the commit the mesh runs
|
||||
// (the commit of the indexer's module mesh-host, its trunk head when none is known); any other sibling, the catalogue
|
||||
// included, at the head of the group's pull request for it when the group holds one, else at its trunk, since a group's
|
||||
// check judges each member against the group's heads (ADR 0482 §2, ADR 0573 §4; issue 1126: a judge a group changed was
|
||||
// judged against the catalogue's trunk). And first, the change's own repository at the trunk it is merged onto, so a test
|
||||
// comparing a shrink-only list with the trunk's copy reads it at $MESH_CHECK_BESIDE/<name> (issue 1887 A12, ADR 0529 §2):
|
||||
// that is where a catalogue change's own trunk copy is placed.
|
||||
func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef, error) {
|
||||
var heads map[string]string
|
||||
if j.Trunks != nil {
|
||||
@@ -59,10 +60,6 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
}
|
||||
for _, repo := range besideRules[p.Repository] {
|
||||
switch {
|
||||
case repo == catalogueRepository:
|
||||
if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
|
||||
}
|
||||
case repo == nodeEngineRepo:
|
||||
commit := ""
|
||||
if j.Runs != nil {
|
||||
@@ -82,10 +79,6 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
||||
} else if head := heads[repo]; head != "" {
|
||||
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
|
||||
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
|
||||
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
|
||||
// checker that cannot place it says so, where placing none left that test to skip.
|
||||
return nil, fmt.Errorf("the trunk head of %s is not known, so the controller the tests of %s#%d read cannot be placed beside them", repo, p.Repository, p.Number)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,10 +14,17 @@ type fakeCompositions struct {
|
||||
v delivery.CompositionVerdict
|
||||
err error
|
||||
asked []string
|
||||
// waits answers only that the composition waits, as the assigner does for a member the indexer holds no manifest of:
|
||||
// Await returns only when its context ends.
|
||||
waits bool
|
||||
}
|
||||
|
||||
func (f *fakeCompositions) Await(_ context.Context, key string) (delivery.CompositionVerdict, error) {
|
||||
func (f *fakeCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
|
||||
f.asked = append(f.asked, key)
|
||||
if f.waits {
|
||||
<-ctx.Done()
|
||||
return delivery.CompositionVerdict{}, ctx.Err()
|
||||
}
|
||||
return f.v, f.err
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *sta
|
||||
t.Helper()
|
||||
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
|
||||
return v, st
|
||||
return settled(joinFailures(v)), st
|
||||
}
|
||||
|
||||
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
|
||||
@@ -58,7 +58,7 @@ func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
|
||||
}
|
||||
|
||||
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
|
||||
// failed, and each entry's line is cut to MaxFailureLine.
|
||||
// failed and names the first, cut so "and N more" stays whole, and each entry's line is cut to MaxFailureLine.
|
||||
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
|
||||
var fs []Failure
|
||||
for i := range 60 {
|
||||
@@ -75,16 +75,20 @@ func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
|
||||
t.Errorf("failedCount = %d, want 60", n)
|
||||
}
|
||||
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
|
||||
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
|
||||
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60): integration: x/p0.TestT: yyy") ||
|
||||
!strings.Contains(line, "…; and 59 more | 0123456789ab on fedcba987654") {
|
||||
t.Errorf("line %q", line)
|
||||
}
|
||||
}
|
||||
// A composition failing after them is counted in the last entry (the operator, 2026-11-02).
|
||||
withComposition(v, delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "route-proxy is defined twice"})
|
||||
if n := failedCount(*v); n != 61 || v.Failed.Test != "integration: x/p0.TestT" {
|
||||
t.Errorf("after the composition failedCount = %d, failed %+v", n, v.Failed)
|
||||
}
|
||||
|
||||
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
|
||||
// before.
|
||||
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
|
||||
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
|
||||
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
|
||||
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
|
||||
// before.
|
||||
one := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
|
||||
if got := failedLines(one); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,11 +10,13 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
@@ -342,6 +344,11 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
|
||||
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
|
||||
}
|
||||
for _, a := range asks { // ADR 0584: every check ask names the agreed toolchain, so a node off it declines
|
||||
if a.Check.Toolchain == nil || a.Check.Toolchain.Version != "go1.27.1" || a.Check.Toolchain.Recipe != "7" {
|
||||
t.Errorf("a check ask names toolchain %+v, want go1.27.1 recipe 7", a.Check.Toolchain)
|
||||
}
|
||||
}
|
||||
for _, a := range asks[1:] {
|
||||
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
|
||||
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
|
||||
@@ -426,32 +433,87 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
|
||||
}
|
||||
}
|
||||
|
||||
// 1289/AC1, AC2 (R4c, mesh-sdk buildrun at 0b12b62): the change's own tests run first, and when one fails the level ends
|
||||
// there: no other package of the level and no later level runs, so the rest of the suite is not paid for.
|
||||
func TestTheChangesOwnFailingTestEndsTheLevelThere(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
||||
failing := captured[Tested](t, "go-test-failing")
|
||||
b.test = func(c stepCall) (any, error) {
|
||||
if got := packagesOf(c); len(got) == 1 && got[0] == "m/internal/b" {
|
||||
return failing, nil
|
||||
// The operator, 2026-11-02 ("A failed check shows only its first fault"): a check runs every member, every level and the
|
||||
// composition to its end and its verdict names every failure in the order judged, Failed keeping the first whole; the
|
||||
// change's own tests still run first (issue 1289), and the rest of the level after them. A member that only failed its
|
||||
// tests still gets its preview, so the composition is read and its failure named after the tests'. A member whose head
|
||||
// does not merge has no preview of a merge, so the assigner's composition would only wait: the check does not wait for it
|
||||
// and says why it was not run (review of 650a1d0).
|
||||
func TestEveryFailureOfEveryMemberLevelAndTheCompositionIsNamed(t *testing.T) {
|
||||
for _, conflicts := range []bool{false, true} {
|
||||
b := newBench(t)
|
||||
comps := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionFail, SDK: "x",
|
||||
Machine: "g14", Words: "route-proxy is defined twice"}, waits: conflicts}
|
||||
b.ex.Compositions = comps
|
||||
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
||||
conflict := captured[Fetched](t, "go-fetch-conflict")
|
||||
b.steps.handlers["build_go_fetch"] = func(c stepCall) (any, error) {
|
||||
if conflicts && c.args["repository"] == repoSDK {
|
||||
return conflict, nil
|
||||
}
|
||||
return b.fetch, nil
|
||||
}
|
||||
failing := captured[Tested](t, "go-test-failing")
|
||||
b.test = func(c stepCall) (any, error) {
|
||||
if got := packagesOf(c); c.args["level"] == "integration" || (len(got) == 1 && got[0] == "m/internal/b") {
|
||||
r := failing
|
||||
r.Level = c.args["level"].(string)
|
||||
return r, nil
|
||||
}
|
||||
return captured[Tested](t, "go-test-slow"), nil
|
||||
}
|
||||
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
|
||||
done := make(chan delivery.Verdict, 1)
|
||||
go func() { done <- b.judge(t, cs).Verdict }()
|
||||
var v delivery.Verdict
|
||||
select {
|
||||
case v = <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("conflicts %v: the check waits on a composition that only waits", conflicts)
|
||||
}
|
||||
var got []string
|
||||
for _, f := range v.Failures {
|
||||
got = append(got, firstFailed(delivery.Verdict{Failed: f}))
|
||||
}
|
||||
tests := []string{"unit: fixture/failing/bad.TestThreeIsFour", "integration: fixture/failing/bad.TestThreeIsFour"}
|
||||
want := append(append(slices.Clone(tests), tests...), "g14: route-proxy is defined twice") // both members fail their tests
|
||||
if conflicts {
|
||||
want = append([]string{"shanks: novox/mesh-sdk#12 does not merge into the trunk: files in conflict: a.go"}, tests...)
|
||||
}
|
||||
if v.State != delivery.VerdictFail || len(got) != len(want) {
|
||||
t.Fatalf("conflicts %v: verdict %+v: failures\n%s\nwant\n%s", conflicts, v, strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
for i := range want {
|
||||
if !strings.HasPrefix(got[i], want[i]) {
|
||||
t.Errorf("conflicts %v: failure %d is %q, want %q", conflicts, i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
keys, _ := b.prev.Keys(context.Background())
|
||||
switch {
|
||||
case !conflicts && (len(keys) != 2 || v.Composition == nil || v.Composition.State != delivery.CompositionFail):
|
||||
t.Errorf("tests alone failed: previews %v, composition %+v, want both previews and the composition read", keys, v.Composition)
|
||||
case conflicts && (len(comps.asked) != 0 || v.Composition == nil || v.Composition.State != delivery.CompositionNotRun ||
|
||||
!strings.Contains(v.Composition.Words, "novox/mesh-sdk#12")):
|
||||
t.Errorf("a member did not merge: asked %v, composition %+v, want not run naming novox/mesh-sdk#12", comps.asked, v.Composition)
|
||||
}
|
||||
var unit []string
|
||||
for _, c := range b.steps.tool("test") {
|
||||
if c.args["level"] == "unit" && c.args["workspace"] != nil {
|
||||
unit = append(unit, strings.Join(packagesOf(c), ","))
|
||||
}
|
||||
}
|
||||
if conflicts && (strings.Join(unit, " | ") != "m/internal/b | m/internal/a" || len(b.steps.tool("digest")) != 2) {
|
||||
t.Errorf("the unit level ran %v and %d digests, want the changed test's package, then the rest, at both levels", unit, len(b.steps.tool("digest")))
|
||||
}
|
||||
v.Group, v.Heads = "feat/x", "0123456789ab"
|
||||
if _, err := delivery.EncodeVerdict(v); err != nil {
|
||||
t.Errorf("conflicts %v: the verdict cannot be written: %v", conflicts, err)
|
||||
}
|
||||
if line := Line(v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, fmt.Sprintf("failed (%d): %s", len(want), want[0][:20])) ||
|
||||
!strings.Contains(line, fmt.Sprintf("; and %d more | ", len(want)-1)) || utf8.RuneCountInString(line) > MaxLine {
|
||||
t.Errorf("conflicts %v: line %q", conflicts, line)
|
||||
}
|
||||
return captured[Tested](t, "go-test-slow"), nil
|
||||
}
|
||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
|
||||
t.Fatalf("verdict = %+v", out.Verdict)
|
||||
}
|
||||
tests := b.steps.tool("test")
|
||||
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
|
||||
t.Fatalf("test calls = %+v, want m/internal/b alone: its failure ends the level", tests)
|
||||
}
|
||||
if n := len(b.steps.tool("digest")); n != 1 {
|
||||
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
|
||||
}
|
||||
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
|
||||
t.Errorf("composition = %+v", out.Verdict.Composition)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -76,8 +76,9 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
|
||||
|
||||
// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
|
||||
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
|
||||
// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and
|
||||
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
||||
// and the integration level, taking the kept verdicts of unchanged packages and running the change's own tests first;
|
||||
// every level and member runs to its end also after a failure, so the verdict names every failure (the operator,
|
||||
// 2026-11-02). It judges the step's answer as it judged the tools' answers.
|
||||
type Examiner struct {
|
||||
Asks Asks
|
||||
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
|
||||
@@ -166,7 +167,9 @@ type target struct {
|
||||
module string
|
||||
node string
|
||||
toolchain string
|
||||
fetched Fetched
|
||||
// agreed is the toolchain the ask names, so a node off it declines the check (ADR 0584); nil when none is agreed.
|
||||
agreed *delivery.ToolchainUsed
|
||||
fetched Fetched
|
||||
// fetchArgs are the arguments of the fetch its check ask runs.
|
||||
fetchArgs map[string]any
|
||||
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
|
||||
@@ -229,6 +232,9 @@ type state struct {
|
||||
judgeHead string
|
||||
// predefinedHead is the catalogue member's head when its merge changes predefined.json: the list is read there.
|
||||
predefinedHead string
|
||||
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
|
||||
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
|
||||
unpreviewed []string
|
||||
}
|
||||
|
||||
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
||||
@@ -236,23 +242,20 @@ type state struct {
|
||||
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
|
||||
beside: map[string]string{}}
|
||||
if v := st.judgeAll(ctx); v != nil {
|
||||
return st.finish(ctx, *v), nil
|
||||
}
|
||||
// Every manifest the change touches, judged by the indexer's judge (ADR 0578 §2): a refusal fails the check.
|
||||
if v := st.judgeManifests(ctx); v != nil {
|
||||
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
|
||||
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
|
||||
tests := st.judgeAll(ctx)
|
||||
if v := settled(joinFailures(tests, st.judgeManifests(ctx))); v != nil {
|
||||
return st.finish(ctx, *v), nil
|
||||
}
|
||||
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
|
||||
}
|
||||
|
||||
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check
|
||||
// whose tests passed, the assigner's composition of it.
|
||||
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry and the assigner's
|
||||
// composition of it, read also when the tests failed, so a failed composition is named beside them.
|
||||
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
||||
out := st.outcome(v)
|
||||
if out.Verdict.State == delivery.VerdictPass {
|
||||
st.composed(ctx, &out.Verdict)
|
||||
}
|
||||
st.composed(ctx, &out.Verdict)
|
||||
out.Builds = st.j.buildNotes(ctx, st.c)
|
||||
return out
|
||||
}
|
||||
@@ -271,6 +274,12 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
if st.j.Compositions == nil || st.c.Try != nil {
|
||||
return
|
||||
}
|
||||
if len(st.unpreviewed) > 0 {
|
||||
// The assigner would only say it waits for a manifest the indexer will not have: the check does not wait for it.
|
||||
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: "no preview of a merge of " +
|
||||
strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer), so the change cannot be composed"}
|
||||
return
|
||||
}
|
||||
comp, err := st.j.Compositions.Await(ctx, st.c.Key)
|
||||
switch {
|
||||
case errors.Is(err, ErrAssignerNotRunning):
|
||||
@@ -352,8 +361,9 @@ func (st *state) levelsOf() []string {
|
||||
}
|
||||
|
||||
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
|
||||
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
||||
// (ADR 0484 clause 13).
|
||||
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails is a failure of that
|
||||
// member; every other member is still judged, and the verdict names every failure (the operator, 2026-11-02), unjoined
|
||||
// to its first lines (settled does that). Only a check whose context ended stops.
|
||||
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
||||
// Node fetches take the group's package (ADR 0500 §5).
|
||||
@@ -368,12 +378,21 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
return 0
|
||||
})
|
||||
var groupPackage map[string]any
|
||||
var failures []*delivery.Verdict
|
||||
for _, p := range pulls {
|
||||
if ctx.Err() != nil {
|
||||
if len(failures) == 0 { // an ask that ended with the context said it already
|
||||
failures = append(failures, fail(delivery.CheckerModule, ctx.Err().Error()))
|
||||
}
|
||||
break
|
||||
}
|
||||
var merged []Fetched
|
||||
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
||||
modules := st.j.Modules.BuildModules(p.Repository)
|
||||
for _, bm := range modules {
|
||||
t := &target{pull: p, module: bm}
|
||||
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
||||
t.toolchain = tc.Version
|
||||
t.agreed = &delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
|
||||
}
|
||||
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
||||
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
|
||||
@@ -382,7 +401,8 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
if bm == buildrun.GoBuildModule {
|
||||
beside, err := st.j.besideFor(ctx, st.c, p)
|
||||
if err != nil {
|
||||
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
||||
failures = append(failures, fail(forgeMachine(err, bm), err.Error()+unheardWords(err)))
|
||||
continue
|
||||
}
|
||||
if len(beside) > 0 {
|
||||
args["beside"] = beside
|
||||
@@ -396,18 +416,24 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
t.fetchArgs = args
|
||||
st.targets = append(st.targets, t)
|
||||
if v := st.ask(ctx, t, &merged); v != nil {
|
||||
return v
|
||||
failures = append(failures, v)
|
||||
}
|
||||
}
|
||||
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
|
||||
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
|
||||
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
|
||||
switch {
|
||||
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
|
||||
case len(merged) == len(modules):
|
||||
st.j.writePreview(ctx, p, merged)
|
||||
default:
|
||||
st.unpreviewed = append(st.unpreviewed, pullKey(p))
|
||||
}
|
||||
st.judgedByMember(p, merged)
|
||||
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
|
||||
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return joinFailures(failures...)
|
||||
}
|
||||
|
||||
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
|
||||
@@ -428,7 +454,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
|
||||
p := t.pull
|
||||
placed := len(needs) > 0
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages, Toolchain: t.agreed}
|
||||
if st.j.Nodes != nil {
|
||||
names, err := st.j.Nodes.Names(ctx)
|
||||
if err == nil && len(names) == 0 {
|
||||
@@ -495,12 +521,17 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
bound = DefaultDeclineBound
|
||||
}
|
||||
if now.Sub(firstDecline) > bound {
|
||||
lacks := strings.Join(needs, ", ")
|
||||
if lacks == "" {
|
||||
lacks = strings.Join(d.Lacks, ", ") // a node off the agreed toolchain declined it (ADR 0584)
|
||||
}
|
||||
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
|
||||
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
|
||||
p.Repository, p.Number, lacks, bound, step.Node))
|
||||
}
|
||||
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
|
||||
a.ID, a.At = buildstep.NewID(), now.UTC()
|
||||
}
|
||||
var failures []*delivery.Verdict
|
||||
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
||||
took := time.Duration(0)
|
||||
if !step.Started.IsZero() && step.Ended.After(step.Started) {
|
||||
@@ -532,7 +563,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
if !placed { // the placed ask fetched the same merge: its preview was read from the first
|
||||
*merged = append(*merged, t.fetched)
|
||||
if v := st.checkBinariesOf(t); v != nil {
|
||||
return v
|
||||
failures = append(failures, v) // the levels still run: the verdict names every failure
|
||||
}
|
||||
}
|
||||
ran := 0.0
|
||||
@@ -548,13 +579,12 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
v, elapsed := st.level(ctx, t, la)
|
||||
ran += elapsed
|
||||
if v != nil {
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
return v
|
||||
failures = append(failures, v) // the next level was run too (buildstep): its failures are named as well
|
||||
}
|
||||
}
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
|
||||
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
|
||||
// Every level ran, its passes kept, and some packages were left to a node with what their tests need: the levels
|
||||
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
|
||||
var again, packages []string
|
||||
seen := map[string]bool{}
|
||||
@@ -570,9 +600,9 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
}
|
||||
}
|
||||
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
|
||||
return st.askNeeding(ctx, t, merged, again, unmet, packages)
|
||||
failures = append(failures, st.askNeeding(ctx, t, merged, again, unmet, packages))
|
||||
}
|
||||
return nil
|
||||
return joinFailures(failures...)
|
||||
}
|
||||
|
||||
// unmetNames names the unmet packages and their needs.
|
||||
@@ -858,10 +888,58 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
|
||||
}
|
||||
v.Failed = all[0]
|
||||
v.Failures = failureLines(all)
|
||||
v.Failures = all // whole: settled cuts them to their first lines once every failure of the check is joined
|
||||
return &v
|
||||
}
|
||||
|
||||
// joinFailures is one failed verdict of every failure of vs, in the order given (a nil one passed), unsettled: Failed is
|
||||
// the first whole, and Failures every failure whole, or none when the one failure is no level's (a conflict, a refused
|
||||
// step: one line, as before issue 1921). Nil when none failed.
|
||||
func joinFailures(vs ...*delivery.Verdict) *delivery.Verdict {
|
||||
var out *delivery.Verdict
|
||||
var all []delivery.FailedFirst
|
||||
listed := false
|
||||
for _, v := range vs {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = &delivery.Verdict{State: delivery.VerdictFail, Failed: v.Failed}
|
||||
}
|
||||
if len(v.Failures) > 0 {
|
||||
all, listed = append(all, v.Failures...), true
|
||||
continue
|
||||
}
|
||||
all = append(all, v.Failed)
|
||||
}
|
||||
if out != nil && (listed || len(all) > 1) {
|
||||
out.Failures = all
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// settled is a joined verdict as it is written: each failure of its list cut to its first line, at most
|
||||
// delivery.MaxFailures and then how many more (failureLines).
|
||||
func settled(v *delivery.Verdict) *delivery.Verdict {
|
||||
if v != nil && len(v.Failures) > 0 {
|
||||
v.Failures = failureLines(v.Failures)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// addFailure is a settled list of failures with one more at its end: the first failure is put first when the list is
|
||||
// empty (a verdict of one failure keeps none), and past delivery.MaxFailures the last entry counts one more.
|
||||
func addFailure(v *delivery.Verdict, f delivery.FailedFirst) {
|
||||
if len(v.Failures) == 0 {
|
||||
v.Failures = failureLines([]delivery.FailedFirst{v.Failed})
|
||||
}
|
||||
if n := len(v.Failures); n > delivery.MaxFailures {
|
||||
v.Failures[n-1].Test = fmt.Sprintf("and %d more", failedCount(*v)-delivery.MaxFailures+1)
|
||||
return
|
||||
}
|
||||
v.Failures = append(v.Failures, failureLines([]delivery.FailedFirst{f})...)
|
||||
}
|
||||
|
||||
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
|
||||
// and then one entry saying how many more failed.
|
||||
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
|
||||
|
||||
@@ -65,7 +65,35 @@ func failedLines(v delivery.Verdict) []string {
|
||||
return lines
|
||||
}
|
||||
|
||||
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
|
||||
// failedHead is the head of a failed verdict's line: what failed first, and when more did, how many failed and "and N
|
||||
// more" after it (the operator, 2026-11-02), so a line cut to MaxLine still says that more failed. The words of the first
|
||||
// are cut to fit room characters when room is not negative; the count and "and N more" are never cut.
|
||||
func failedHead(v delivery.Verdict, room int) string {
|
||||
first := firstFailed(v)
|
||||
n := failedCount(v)
|
||||
if n <= 1 {
|
||||
return clipRunes("failed: "+first, room)
|
||||
}
|
||||
prefix, more := fmt.Sprintf("failed (%d): ", n), fmt.Sprintf("; and %d more", n-1)
|
||||
if room >= 0 {
|
||||
first = clipRunes(first, room-utf8.RuneCountInString(prefix+more))
|
||||
}
|
||||
return prefix + first + more
|
||||
}
|
||||
|
||||
// clipRunes cuts s to at most n characters, the last an ellipsis; a negative n leaves it whole.
|
||||
func clipRunes(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if n < 0 || len(r) <= n {
|
||||
return s
|
||||
}
|
||||
if n < 1 {
|
||||
return ""
|
||||
}
|
||||
return string(r[:n-1]) + "…"
|
||||
}
|
||||
|
||||
// failedCount is how many failures a verdict names, the ones after delivery.MaxFailures counted too.
|
||||
func failedCount(v delivery.Verdict) int {
|
||||
n := len(v.Failures)
|
||||
if n > delivery.MaxFailures {
|
||||
@@ -232,10 +260,7 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
||||
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
|
||||
}
|
||||
default:
|
||||
pc.head = "failed: " + firstFailed(v)
|
||||
if n := failedCount(v); n > 1 {
|
||||
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
|
||||
}
|
||||
pc.head = failedHead(v, -1)
|
||||
pc.commits = commits
|
||||
}
|
||||
var counts []string
|
||||
@@ -325,6 +350,11 @@ func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []
|
||||
// what follows, and is cut in its first part rather than lose it.
|
||||
func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
|
||||
pc := parts(v, cs, p, builds, slowest, judged)
|
||||
if v.State == delivery.VerdictFail {
|
||||
// The head gives way first to the commits and what is never dropped: its count and "and N more" stay whole.
|
||||
kept := joinSet(pc.commits, pc.carried, pc.warning, pc.judged)
|
||||
pc.head = failedHead(v, MaxLine-utf8.RuneCountInString(" | "+kept))
|
||||
}
|
||||
for drop := 0; drop <= 4; drop++ {
|
||||
if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
|
||||
return s
|
||||
@@ -342,6 +372,12 @@ func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []st
|
||||
return string(head[:room]) + "…" + rest
|
||||
}
|
||||
}
|
||||
if v.State == delivery.VerdictFail {
|
||||
// A failure's head, sized above, and the commits fit: the counts and the composition are in the report.
|
||||
if s := joinSet(pc.head, pc.commits, pc.carried, pc.warning, pc.judged); utf8.RuneCountInString(s) <= MaxLine {
|
||||
return s
|
||||
}
|
||||
}
|
||||
runes := []rune(pc.join(4))
|
||||
return string(runes[:MaxLine-1]) + "…"
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
@@ -43,6 +44,7 @@ func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCom
|
||||
|
||||
func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { // 1300/AC2
|
||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||
long := delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + "TestSomething", Words: strings.Repeat("because ", 40)}
|
||||
for name, c := range map[string]struct {
|
||||
v delivery.Verdict
|
||||
want string
|
||||
@@ -50,8 +52,14 @@ func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { //
|
||||
"test": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}, "failed: unit: m/a.TestX: want 4"},
|
||||
"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
|
||||
"refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
|
||||
// ADR 0591: a first failure longer than the forge's limit is shortened; "failed (N)" and "and N-1 more" are never cut.
|
||||
"many, the first too long": {delivery.Verdict{State: delivery.VerdictFail, Failed: long, Failures: []delivery.FailedFirst{long, {Machine: "g14", Words: "route-proxy is defined twice"},
|
||||
{Test: "unit: m/b.TestY", Words: "boom"}}}, "failed (3): integration: pkg/pkg/"},
|
||||
} {
|
||||
line := Line(c.v, cs, p, nil, nil, nil)
|
||||
if more := len(c.v.Failures) - 1; more > 0 && (!strings.Contains(line, fmt.Sprintf("…; and %d more | 111111111111", more)) || utf8.RuneCountInString(line) > MaxLine) {
|
||||
t.Errorf("%s: line = %q (%d), want the first's words shortened, the count and \"and %d more\" whole, within %d", name, line, utf8.RuneCountInString(line), more, MaxLine)
|
||||
}
|
||||
if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
|
||||
t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package check
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -85,7 +86,8 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
b := newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"})
|
||||
b.ex.Runs = runsAt{"mesh-host": hostRuns}
|
||||
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
||||
// A group of the controller and the catalogue: the controller's tests read the group's catalogue head, the heads the group
|
||||
// merges (ADR 0482 §2, ADR 0573 §4; issue 1126: a judge in the group was judged against the catalogue's trunk).
|
||||
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
|
||||
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
|
||||
@@ -93,8 +95,8 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
|
||||
}
|
||||
got := besideOf(t, b, controller)
|
||||
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
|
||||
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
|
||||
if c := got[catalogue]; c.Commit != head2 || c.Ref != "refs/pull/12/head" {
|
||||
t.Errorf("the catalogue beside the controller in a group = %+v, want the member's head %s", c, head2)
|
||||
}
|
||||
if h := got[host]; h.Commit != hostRuns {
|
||||
t.Errorf("the node-engine beside the controller = %+v, want the commit the mesh runs %s", h, hostRuns)
|
||||
@@ -104,36 +106,42 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
if s := got[SDKRepository]; s.Commit != "9999999999999999999999999999999999999999" {
|
||||
t.Errorf("mesh-sdk beside the controller = %+v, want its trunk", s)
|
||||
}
|
||||
// Alone, the controller's tests read the catalogue's trunk, and the verdict names it.
|
||||
b = newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: trunkS})
|
||||
if out := b.judge(t, caseOf(t, pullOf(controller, 7, head1))); out.Beside[catalogue] != trunkS {
|
||||
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkS)
|
||||
}
|
||||
if c := besideOf(t, b, controller)[catalogue]; c.Commit != trunkS || c.Ref != "" {
|
||||
t.Errorf("the catalogue beside the controller alone = %+v, want its trunk %s", c, trunkS)
|
||||
}
|
||||
}
|
||||
|
||||
// 1295/R4g: the node-engine's tests read the controller's manifest, so a mesh-host pull request has mesh-controller beside it:
|
||||
// at the group member's head when the group holds one, else at its trunk.
|
||||
func hostCase(t *testing.T, pulls ...delivery.CandidatePull) *Case {
|
||||
t.Helper()
|
||||
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pulls...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cs := CaseOf(c, map[string]string{"novox/mesh-host": trunkA, controllerRepo: trunkA2})
|
||||
if cs == nil {
|
||||
t.Fatal("no case")
|
||||
}
|
||||
return cs
|
||||
}
|
||||
|
||||
func TestTheNodeEnginesTestsHaveTheControllerBesideThem(t *testing.T) {
|
||||
// 1126/B8 (novox/hq ADR 0590): the controller is gone, so no repository's tests read it: a mesh-host pull request has
|
||||
// the catalogue and mesh-sdk beside it and no mesh-controller, even where its trunk is known and the group holds one.
|
||||
func TestNoRepositorysTestsHaveTheControllerBesideThem(t *testing.T) {
|
||||
const host = "novox/mesh-host"
|
||||
b := newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
|
||||
b.judge(t, hostCase(t, pullOf(host, 5, head1)))
|
||||
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != trunkA2 || c.Ref != "" {
|
||||
t.Errorf("the controller beside the node-engine alone = %+v, want its trunk %s", c, trunkA2)
|
||||
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b = newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
|
||||
b.judge(t, hostCase(t, pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
|
||||
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != head2 || c.Ref != "refs/pull/9/head" {
|
||||
t.Errorf("the controller beside the node-engine in a group = %+v, want the member's head %s", c, head2)
|
||||
if v := b.judge(t, CaseOf(c, map[string]string{host: trunkA, controllerRepo: trunkA2})).Verdict; v.State == delivery.VerdictFail &&
|
||||
strings.Contains(v.Failed.Words, "mesh-controller") {
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
got := besideOf(t, b, host)
|
||||
if _, ok := got[controllerRepo]; ok {
|
||||
t.Errorf("the controller is beside the node-engine: %+v", got)
|
||||
}
|
||||
if _, ok := got[catalogueRepository]; !ok {
|
||||
t.Errorf("the catalogue is not beside the node-engine: %+v", got)
|
||||
}
|
||||
for repo, siblings := range besideRules {
|
||||
if repo != controllerRepo && slices.Contains(siblings, controllerRepo) {
|
||||
t.Errorf("the tests of %s read the controller", repo)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,15 +156,15 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A12: a catalogue change's siblings hold the catalogue itself at the trunk it is merged onto, beside mesh-sdk and
|
||||
// mesh-controller, so the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog.
|
||||
// 1887/A12: a catalogue change's siblings hold the catalogue itself at the trunk it is merged onto, beside mesh-sdk, so
|
||||
// the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog.
|
||||
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
|
||||
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
|
||||
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
|
||||
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != "" || out.Beside[catalogueRepository] != "" {
|
||||
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk at its trunk and neither the controller nor the catalogue", out.Beside)
|
||||
}
|
||||
got := besideOf(t, b, catalogueRepository)
|
||||
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
|
||||
@@ -188,15 +196,3 @@ func TestAMeshLabHeadGetsACheck(t *testing.T) {
|
||||
t.Fatalf("mesh-lab was not fetched: %+v", fetched)
|
||||
}
|
||||
}
|
||||
|
||||
// 1295: the node-engine's step-nine test fails without the controller beside it, so a checker that knows no trunk head of the
|
||||
// controller fails the check at the placing, loudly, and does not place none.
|
||||
func TestTheNodeEnginesCheckFailsWhereTheControllersTrunkIsNotKnown(t *testing.T) {
|
||||
const host = "novox/mesh-host"
|
||||
b := newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, catalogueRepository: trunkS, SDKRepository: trunkS})
|
||||
v := b.judge(t, hostCase(t, pullOf(host, 5, head1))).Verdict
|
||||
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "mesh-controller") || !strings.Contains(v.Failed.Words, "cannot be placed") {
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,11 +52,14 @@ type Reader struct {
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||
// candidate still holds those heads; without it the report is that of the verdict alone.
|
||||
// VerdictRead is a verdict as stored with every failure it names, one line each, and the full report of the status of each
|
||||
// pull request it judged (Report), when the candidate still holds those heads; without it the report is that of the verdict alone.
|
||||
type VerdictRead struct {
|
||||
Key string `json:"key"`
|
||||
Verdict json.RawMessage `json:"verdict"`
|
||||
Key string `json:"key"`
|
||||
Verdict json.RawMessage `json:"verdict"`
|
||||
// Failed is everything the verdict names as failed, one line each, in the order judged (failedLines): every member's
|
||||
// failed tests, refused steps and conflicts, the manifests' refusals and the composition (the operator, 2026-11-02).
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Reports map[string]string `json:"reports,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
@@ -80,6 +83,7 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
|
||||
}
|
||||
out.Failed = failedLines(v)
|
||||
c, ok := r.candidate(v.Group)
|
||||
if !ok || (c.HeadsHash != v.Heads && GateHeads(c) != v.Heads) {
|
||||
out.Note = "the candidate no longer holds these heads"
|
||||
|
||||
@@ -67,6 +67,13 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
|
||||
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||
t.Errorf("another group's verdicts = %+v", only)
|
||||
}
|
||||
// A failed verdict is answered with every failure it names, one line each (the operator, 2026-11-02).
|
||||
raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.VerdictFail,
|
||||
Failed: delivery.FailedFirst{Test: "unit: m/a.TestA", Words: "a_test.go:3: boom\nmore"},
|
||||
Failures: []delivery.FailedFirst{{Test: "unit: m/a.TestA", Words: "a_test.go:3: boom"}, {Machine: "g14", Words: "route-proxy is defined twice"}}})
|
||||
if got, _ := rd.read(key, raw); strings.Join(got.Failed, " | ") != "unit: m/a.TestA: a_test.go:3: boom | g14: route-proxy is defined twice" {
|
||||
t.Errorf("failed = %q", got.Failed)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A10: an hq pull request is answered as any other, its report its verdict's (ADR 0527 §2), never a note that a script
|
||||
|
||||
@@ -204,7 +204,8 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], "TestThreeIsFour") {
|
||||
if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 2 || !strings.Contains(got.Failed[0], "unit: fixture/failing/bad.TestThreeIsFour") ||
|
||||
!strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02)
|
||||
t.Errorf("try = %+v", got)
|
||||
}
|
||||
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).
|
||||
|
||||
+3
-1
@@ -423,7 +423,9 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
|
||||
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}},
|
||||
// The mesh packages its go.sum names (ADR 0573 §2, issue 1585): this module's own go.sum names mesh-sdk.
|
||||
Packages: map[string][]string{"modules/a": {"git.novox.be/novox/mesh-sdk/go"}}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0 h1:q6ywchpTM9fwaJ1UNwDSuOjb5FYDFcsaeiTrKoB4scc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -16,11 +16,10 @@
|
||||
// check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host
|
||||
// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's
|
||||
// go.mod pins. Each repository's tests read these siblings, and the checker places exactly these
|
||||
// (mesh-delivery's checker, besideRules, follows this list): mesh-controller's read mesh-catalog,
|
||||
// mesh-host and mesh-sdk; mesh-host's read mesh-catalog, mesh-sdk and mesh-controller (its step-nine
|
||||
// test reads the controller's own manifest, novox/hq issue 1295); mesh-sdk's read mesh-catalog and
|
||||
// mesh-host; mesh-catalog's read mesh-sdk and mesh-controller; mesh-delivery's read mesh-sdk and
|
||||
// mesh-catalog. In a mesh where either module has no source repository nothing is
|
||||
// (mesh-delivery's checker, besideRules, follows this list): mesh-host's read mesh-catalog and
|
||||
// mesh-sdk; mesh-sdk's read mesh-catalog and mesh-host; mesh-catalog's read mesh-sdk; mesh-delivery's
|
||||
// read mesh-sdk and mesh-catalog. None reads mesh-controller, which ended by its unassignment (novox/hq
|
||||
// ADR 0590). In a mesh where either module has no source repository nothing is
|
||||
// cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the
|
||||
// change. And in a delivery group that holds a mesh-catalog pull request, these tests read the
|
||||
// catalogue's main, not the group's head.
|
||||
@@ -50,23 +49,37 @@ const Env = "MESH_CHECK_BESIDE"
|
||||
// CapturedSuffix is what a captured file's name carries that the repository's own does not.
|
||||
const CapturedSuffix = ".captured"
|
||||
|
||||
// notPlaced are the repositories the checker places beside no mesh-delivery check (besideRules in
|
||||
// mesh-delivery-checker/internal/check/beside.go places mesh-sdk and the catalogue): a test reads them from the captured
|
||||
// copy even in a merge check, and says so (issue 1585). Every other sibling missing from a merge check fails the test.
|
||||
var notPlaced = map[string]bool{"mesh-host": true}
|
||||
|
||||
// Dir is the named repository a test reads — the clone beside a merge check, or the captured copy — and
|
||||
// says which in the test's log.
|
||||
func Dir(t testing.TB, repository string) string {
|
||||
t.Helper()
|
||||
if root := os.Getenv(Env); root != "" {
|
||||
root := os.Getenv(Env)
|
||||
if root != "" {
|
||||
dir := filepath.Join(root, repository)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
_, err := os.Stat(dir)
|
||||
switch {
|
||||
case err == nil:
|
||||
t.Logf("judged against %s as cloned beside this check", dir)
|
||||
return dir
|
||||
case !notPlaced[repository]:
|
||||
// A sibling the checker places that is missing is the checker's fault, never judged against a stale copy.
|
||||
t.Fatalf("%s is not beside this check in %s=%s, so its agreement with this repository cannot be "+
|
||||
"judged here: %v", repository, Env, root, err)
|
||||
}
|
||||
t.Logf("judged against %s as cloned beside this check", dir)
|
||||
return dir
|
||||
}
|
||||
from := filepath.Join(Captured(), repository)
|
||||
if _, err := os.Stat(from); err != nil {
|
||||
t.Fatalf("no captured copy of %s at %s: %v", repository, from, err)
|
||||
}
|
||||
if root != "" {
|
||||
t.Logf("%s is not beside this check in %s=%s (the checker places it beside no mesh-delivery check): read from "+
|
||||
"its captured copy", repository, Env, root)
|
||||
}
|
||||
dir, err := layOut(repository, from)
|
||||
if err != nil {
|
||||
t.Fatalf("the captured copy of %s could not be laid out: %v", repository, err)
|
||||
|
||||
@@ -84,6 +84,7 @@ type Indexer struct {
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
|
||||
@@ -22,7 +22,8 @@ import (
|
||||
// 1851), so a module added by the merge gets a value, and a fingerprint, with no build anywhere. The value of today's
|
||||
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
|
||||
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
|
||||
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
|
||||
// clause 7), and kept waiting: once the holding repository's trunk drops the module, the merge that drops it hands it to the
|
||||
// waiting repository instead of deleting it (moves.go). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
|
||||
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||
//
|
||||
@@ -36,6 +37,12 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWri
|
||||
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
|
||||
// drops none.
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
|
||||
return ix.applyMoving(ctx, f, drop, nil)
|
||||
}
|
||||
|
||||
// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them
|
||||
// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting.
|
||||
func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
|
||||
w := newMergeWrite(f)
|
||||
if drop {
|
||||
if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
|
||||
@@ -71,8 +78,12 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
|
||||
named[key] = true
|
||||
cur, has := held[key]
|
||||
if has && cur.Repository != "" && !delivery.SameRepository(cur.Repository, f.Repository) {
|
||||
ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is (a move between repositories is the operator's)", f.Repository, f.Merge, dir, name, cur.Repository)
|
||||
continue
|
||||
if !moves[key] {
|
||||
ix.waitToMove(key, f, dir)
|
||||
ix.logf("%s at %s: %q names the module %s, which the indexer holds of %s; left as it is until %s's trunk drops it, and then moved here (a move between repositories is the merge of the group that moves it, ADR 0420 clause 7)", f.Repository, f.Merge, dir, name, cur.Repository, cur.Repository)
|
||||
continue
|
||||
}
|
||||
ix.logf("%s at %s: the module %s moves here from %s, whose trunk no longer holds it", f.Repository, f.Merge, name, cur.Repository)
|
||||
}
|
||||
if other, dup := dirOf[name]; dup {
|
||||
if !has || cur.Path != pathOf(dir) {
|
||||
@@ -190,11 +201,16 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri
|
||||
// dropRemoved deletes, from the modules the bucket holds of the repository, each one the trunk no longer holds (issue 1900):
|
||||
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
|
||||
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
|
||||
var handed []string
|
||||
for _, key := range slices.Sorted(maps.Keys(held)) {
|
||||
cur := held[key]
|
||||
if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
|
||||
continue
|
||||
}
|
||||
if ix.movingTo(key, repository) {
|
||||
handed = append(handed, key) // written in place by the repository it moves to, never deleted first
|
||||
continue
|
||||
}
|
||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
@@ -209,6 +225,11 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
|
||||
}
|
||||
ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
|
||||
}
|
||||
for _, key := range handed {
|
||||
if err := ix.handOver(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -402,20 +403,64 @@ func TestAMergeWritesOnlyTheModulesItsManifestsName(t *testing.T) { // I1
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleHeldOfAnotherRepositoryIsLeftAndSaid(t *testing.T) { // I1: a move between repositories is the operator's (ADR 0420 clause 7)
|
||||
ix, mods := modulesIndexer(t)
|
||||
var logs []string
|
||||
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-catalog", map[string]string{"modules/a": "t"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := string(mods.vals["a"])
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t2"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(mods.vals["a"]) != before || len(logs) == 0 {
|
||||
t.Errorf("modules.a = %s (was %s), logs %v: want it left, and said", mods.vals["a"], before, logs)
|
||||
// deletesSeen is a modules store that remembers each key deleted.
|
||||
type deletesSeen struct {
|
||||
*memStore
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (d *deletesSeen) Delete(ctx context.Context, key string, rev uint64) error {
|
||||
d.deleted = append(d.deleted, key)
|
||||
return d.memStore.Delete(ctx, key, rev)
|
||||
}
|
||||
|
||||
// I1: a move between repositories is the merges of the group that moves it (ADR 0420 clause 7; issue 1126, route-proxy
|
||||
// leaving the controller's repository). A merge naming a module another repository's trunk still holds leaves it, said; the
|
||||
// holding repository's merge that drops it hands it over in place, never deleting its key; merged the other way round,
|
||||
// the module is deleted and then written new.
|
||||
func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
const from, to = "novox/mesh-controller", "novox/mesh-networking"
|
||||
held := mergedMsg(from, map[string]string{"modules/a": "t", "modules/b": "t"})
|
||||
named := mergedMsg(to, map[string]string{"modules/a": "t2"})
|
||||
dropped := mergedMsg(from, map[string]string{"modules/b": "t"})
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
merges []delivery.Forge
|
||||
repository string // modules.a's repository at the end, "" for none
|
||||
deletedKeyA bool
|
||||
}{
|
||||
{"named while its trunk holds it: left", []delivery.Forge{held, named}, from, false},
|
||||
{"named, then dropped by its trunk: handed over in place", []delivery.Forge{held, named, dropped}, to, false},
|
||||
{"dropped by its trunk, then named: deleted, then written new", []delivery.Forge{held, dropped, named}, to, true},
|
||||
{"dropped by its trunk with no merge naming it: deleted", []delivery.Forge{held, dropped}, "", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mods := &deletesSeen{memStore: newMem()}
|
||||
ix := &Indexer{Store: newMem(), Modules: mods, Builds: NewBuilds()}
|
||||
var logs []string
|
||||
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
|
||||
for _, f := range tc.merges {
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m, ok := moduleOf(t, mods.memStore, "a")
|
||||
if got := map[bool]string{true: m.Repository}[ok]; got != tc.repository {
|
||||
t.Errorf("modules.a of %q, want %q (logs %v)", got, tc.repository, logs)
|
||||
}
|
||||
if ok && m.Path != "modules/a" {
|
||||
t.Errorf("modules.a at %q, want modules/a", m.Path)
|
||||
}
|
||||
if deleted := slices.Contains(mods.deleted, "a"); deleted != tc.deletedKeyA {
|
||||
t.Errorf("modules.a deleted %v, want %v", deleted, tc.deletedKeyA)
|
||||
}
|
||||
if _, ok := mods.vals["b"]; !ok {
|
||||
t.Error("modules.b, which its trunk still holds, is gone")
|
||||
}
|
||||
if len(logs) == 0 {
|
||||
t.Error("nothing was said")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A module moves between repositories by the merges of the delivery group that moves it (novox/hq ADR 0420 clause 7, ADR
|
||||
// 0501 clause 3; issue 1126: route-proxy leaves the controller's repository, which is archived). The merge of the
|
||||
// repository it moves to names it while the indexer holds it of another: that merge leaves it, says so, and waits here.
|
||||
// The merge of the repository that held it, whose trunk no longer holds it, then hands it over: the module is written from
|
||||
// the waiting merge, in place, so its key never goes missing between the two (dropRemoved). Merged the other way round,
|
||||
// the first merge deletes the key and the second writes it as a new module. Nothing else moves a module: a merge of a
|
||||
// repository naming a module whose holding trunk still holds it changes nothing.
|
||||
//
|
||||
// The wait is the indexer's memory of a merge it heard, not state of its own: an indexer started between the two merges
|
||||
// has forgotten it, and the module is deleted by the second; the next merge of the repository it moved to writes it.
|
||||
|
||||
// waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another
|
||||
// repository. A later merge of the same repository replaces it.
|
||||
func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) {
|
||||
one := delivery.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
|
||||
Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}}
|
||||
if l := f.Locks[dir]; len(l) > 0 {
|
||||
one.Locks = map[string]map[string]string{dir: l}
|
||||
}
|
||||
if p := f.Paths[dir]; len(p) > 0 {
|
||||
one.Paths = map[string]map[string]string{dir: p}
|
||||
}
|
||||
if p := f.Packages[dir]; len(p) > 0 {
|
||||
one.Packages = map[string][]string{dir: p}
|
||||
}
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if ix.moving == nil {
|
||||
ix.moving = map[string]delivery.Forge{}
|
||||
}
|
||||
ix.moving[key] = one
|
||||
}
|
||||
|
||||
// movingTo says a merge of another repository than from waits to take the module at key.
|
||||
func (ix *Indexer) movingTo(key, from string) bool {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
f, ok := ix.moving[key]
|
||||
return ok && !delivery.SameRepository(f.Repository, from)
|
||||
}
|
||||
|
||||
// handOver writes the module at key from the merge waiting to take it, and forgets the wait.
|
||||
func (ix *Indexer) handOver(ctx context.Context, key string) error {
|
||||
ix.mu.Lock()
|
||||
f, ok := ix.moving[key]
|
||||
delete(ix.moving, key)
|
||||
ix.mu.Unlock()
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
|
||||
}
|
||||
// A module moved in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1).
|
||||
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
|
||||
ix.logf("judge: the manifests are not judged after moving %s to %s: %v", key, f.Repository, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -2450,19 +2450,24 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M
|
||||
// holding mesh-broker is granted the snapshot API and nothing else, so anything it declared to say
|
||||
// or hear on the bus would be granted nothing — refused here rather than silently dropped.
|
||||
//
|
||||
// **But its tools bundle may say what it declares** (novox/hq ADR 0357): the node's runtime carries
|
||||
// the bundle and is granted the events the module emits, under the module's name, as for any module it
|
||||
// carries — the bus's account is not who publishes them. So emits are refused only for a bus module
|
||||
// with no bundle to say them.
|
||||
// **But its tools bundle may say and ask what it declares** (novox/hq ADR 0357): the node's runtime carries
|
||||
// the bundle, is granted the events the module emits under the module's name, as for any module it carries,
|
||||
// and asks every tool and seat verb on the bundle's behalf (mesh-sdk go/buscompose, the runtime's invoke of
|
||||
// `*`) — the bus's account is not who publishes or asks. So emits and invokes are refused only for a bus
|
||||
// module with no bundle to say or ask them (issue 774: nats's bundle records its hand act).
|
||||
if _, account := m.OwnSecrets["broker"]; account && m.ClaimsSeat(catalogue.BrokerSeat) {
|
||||
var said []string
|
||||
if len(m.EmitsAll()) > 0 && !manifestShipsABundle(m) {
|
||||
bundled := manifestShipsABundle(m)
|
||||
if len(m.EmitsAll()) > 0 && !bundled {
|
||||
said = append(said, "emits")
|
||||
}
|
||||
if len(m.Invokes) > 0 && !bundled {
|
||||
said = append(said, "invokes")
|
||||
}
|
||||
for _, f := range []struct {
|
||||
name string
|
||||
n int
|
||||
}{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)}, {"invokes", len(m.Invokes)},
|
||||
}{{"consumes", len(m.Consumes)}, {"uses", len(m.Uses)},
|
||||
{"state", len(m.State)}, {"logs", len(m.Logs)}, {"reads", len(m.Reads)},
|
||||
{"streams", len(m.Streams)}} {
|
||||
if f.n > 0 {
|
||||
@@ -4000,7 +4005,9 @@ func toolsSecretProblems(m catalogue.Manifest) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
var BusRemovals = []string{catalogue.RemovesControllerDeadLetterConsumer, catalogue.RemovesEngineActs, catalogue.RemovesAssignmentsStream}
|
||||
// BusRemovals are what a bus account may remove, the closed set of novox/hq ADR 0564 Limits 4 as ADR 0590 clause 6
|
||||
// changes it: the node-engine's acts and the ended controller's leftovers.
|
||||
var BusRemovals = []string{catalogue.RemovesEngineActs, catalogue.RemovesControllerLeftovers}
|
||||
|
||||
// ReadsHolderProblems is what a holder of node-reads is held to (novox/hq ADR 0310 §3): it reads as an
|
||||
// account of its own, so the node's file modes bound what it sees. Carried by the machine's runtime it would
|
||||
@@ -7246,37 +7253,6 @@ func namesOfAccessIDs(accesses map[string]string) []string {
|
||||
var renderFuncs = template.FuncMap{"quote": quote}
|
||||
|
||||
var defaultSeats = append([]catalogue.Seat{
|
||||
// The control plane states what it did under the seat it holds (novox/hq ADR 0134): a role's
|
||||
// events belong to the role, so they keep their address while the holder is replaced. No accepts,
|
||||
// so no work queue is raised for it — only what its holder may say.
|
||||
// And it serves the mesh's own verbs as the seat's tools (novox/hq ADR 0154): `status`, `push`,
|
||||
// `assign` and the rest are a role's interface, not a container's, and stay addressable while
|
||||
// the control plane is replaced.
|
||||
{Name: catalogue.ControllerSeatName, Scope: catalogue.ScopeMesh, Decision: "novox/hq ADR 0079",
|
||||
// And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4).
|
||||
Emits: []string{"applied", "refused", "built-before",
|
||||
"condition-raised", "condition-changed", "condition-cleared", "doctor-heartbeat",
|
||||
// A value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
||||
"secret-replaced",
|
||||
// Every act a healer takes (novox/hq to-be 45 §7).
|
||||
"healer-acted",
|
||||
// A build put back after its gate failed (novox/hq ADR 0236, to-be 45 §8).
|
||||
"rolled-back",
|
||||
// A pull request's merge check, judged (novox/hq to-be 45 §9).
|
||||
"checked",
|
||||
// A walk kept in a new state, and a tier sent to a machine: kept until the catalogue's main consumes them no
|
||||
// more; nothing emits them (ADR 0539).
|
||||
"plan-moved",
|
||||
"sent",
|
||||
// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
|
||||
"assignment-changed", "declared-config-changed", "predefined-changed",
|
||||
// A builder's key not taken in because the artifact store does not hold an artifact it names (novox/hq
|
||||
// issue 1937): the builder builds it again.
|
||||
"artifacts-gone",
|
||||
// A bucket of a state this release deleted, deleted at the controller's first start (novox/hq ADR 0539
|
||||
// clause 13): the merge window's, the silent sends' and the building plans' hold, each said once.
|
||||
"bucket-deleted"},
|
||||
Serves: catalogue.ControllerVerbs},
|
||||
// The store's first verbs (novox/hq ADR 0159): the smallest set that makes the store askable,
|
||||
// served by whichever module holds the seat with tools of these names.
|
||||
{Name: "mesh-store", Scope: catalogue.ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079",
|
||||
@@ -7394,21 +7370,8 @@ var defaultSeats = append([]catalogue.Seat{
|
||||
Serves: []catalogue.Verb{
|
||||
{Name: "config", Optional: true, Description: "Set, clear, list or read the history of config values: " +
|
||||
"mesh config or node config for a predefined key, or a module's own value on one node, each set " +
|
||||
"with why, validated, and naming each module on each node whose value moves. A trusted value only " +
|
||||
"from the controller's terminal.",
|
||||
"with why, validated, and naming each module on each node whose value moves.",
|
||||
Input: configInput()},
|
||||
// The switch of config to its own sources (novox/hq ADR 0565 §4): optional until every mesh-config serves it.
|
||||
{Name: "compare", Optional: true, Description: "Every difference between config's own sources and the " +
|
||||
"controller's records, each with its reason: a module's declared config against the controller's mirror, " +
|
||||
"the indexer's predefined list against the mirror's, and each open pull request's config candidate against " +
|
||||
"what the controller told. The take-over waits for none.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "take-over", Optional: true, Description: "Ask the operator's Approve to switch config to its own " +
|
||||
"sources (the indexer); refused while compare shows a difference.",
|
||||
Input: schema(map[string]string{"why": "why, in a sentence"}, []string{"why"})},
|
||||
{Name: "hand-back", Optional: true, Description: "Switch config back to the controller's mirrors, until the " +
|
||||
"controller hands its lease away; refused after it.",
|
||||
Input: schema(map[string]string{"why": "why, in a sentence"}, []string{"why"})},
|
||||
}},
|
||||
// The provisioner's server (novox/hq ADR 0349 clause 3, design 57 P4): its holder keeps the registry of which
|
||||
// domain answers which kind of placeholder, and the buckets mesh-provisioner_needs, _applied and _collect,
|
||||
@@ -7952,14 +7915,6 @@ func placementVerbs() []catalogue.Verb {
|
||||
"stand, and raise its ask again when its verdict passes and the operator's channel does not hold the ask. " +
|
||||
"Nothing more: the recovery of a proposal whose judgement or ask was lost.",
|
||||
Input: schema(map[string]string{"id": "the proposal's id"}, []string{"id"})},
|
||||
{Name: catalogue.PlacementVerbAdopt, Optional: true, Description: "Take placement over from the controller: its " +
|
||||
"placement-export is proposed whole as the seed, every node's key, which every node must answer as the same. " +
|
||||
"Its Approve is the take-over; at the warrant the export is read again and a changed one is proposed anew.",
|
||||
Input: schema(map[string]string{"why": "why, shown to the operator"}, []string{"why"})},
|
||||
{Name: catalogue.PlacementVerbHandBack, Optional: true, Description: "The operator's alone, recorded as a hand act: " +
|
||||
"delete every placement key, the history kept, so every node returns to the controller's compositions. " +
|
||||
"Refused after the lease write.",
|
||||
Input: schema(map[string]string{"why": "why, recorded with the hand act"}, []string{"why"})},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7999,7 +7954,7 @@ func conditionsVerbs() []catalogue.Verb {
|
||||
{Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then oldest, " +
|
||||
"raised from the observers' findings (the watchman, the self-check, the healers, the assigner) and cleared when " +
|
||||
"the finding goes, never by hand. Given a key, that one whole with its evidence; with history, every transition " +
|
||||
"lately, the controller's before the take-over included.",
|
||||
"lately.",
|
||||
Input: schema(map[string]string{
|
||||
"key": "a condition's key: that one whole; with history, only its transitions",
|
||||
"scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe, mesh, delivery or module",
|
||||
@@ -8016,17 +7971,6 @@ func conditionsVerbs() []catalogue.Verb {
|
||||
"why": "why, required and recorded",
|
||||
"cause": "the cause in a word (the condition's kind when absent)",
|
||||
}, []string{"key", "for", "why"})},
|
||||
{Name: "compare", Description: "The keeper's open conditions against the controller's: every key open in one and not " +
|
||||
"the other, or open in both with another severity, each with its reason; the kinds that end with the controller " +
|
||||
"and the keys held while their module is delivered listed apart and not counted. The take-over waits for it empty.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
{Name: "take-over", Description: "Ask the operator's Approve to switch who the router, the grants part and " +
|
||||
"mesh-watcher hear from the controller to the keeper; refused while compare shows a difference. On the warrant " +
|
||||
"the keeper writes its take-over key.",
|
||||
Input: schema(map[string]string{"why": "why now, shown to the operator"}, []string{"why"})},
|
||||
{Name: "hand-back", Description: "Switch the readers back to the controller's conditions, until the controller's " +
|
||||
"lease is handed away; refused after it.",
|
||||
Input: schema(map[string]string{"why": "why, kept with the switch"}, []string{"why"})},
|
||||
{Name: "resync", Description: "Read every observer's findings again and raise and clear what differs from the open " +
|
||||
"conditions: the recovery of a keeper that missed a change.",
|
||||
Input: schema(map[string]string{}, nil)},
|
||||
@@ -8048,8 +7992,7 @@ func watchdogsVerbs() []catalogue.Verb {
|
||||
Input: schema(map[string]string{"machine": "one machine"}, nil)},
|
||||
{Name: "dead-letters", Description: "Every message a consumer on the bus gave up on, kept in DEAD_LETTERS: whose " +
|
||||
"consumer, the subject, how often it was handed over and when it was given up, newest first. With id: that one " +
|
||||
"whole. With deliver or drop, a hand act with why, taken only once the controller's lease is handed away; " +
|
||||
"before, refused naming the controller's verb.",
|
||||
"whole. With deliver or drop, a hand act with why.",
|
||||
Input: schema(map[string]string{
|
||||
"id": "a dead letter's id, as the list gives it: that one whole, with its message",
|
||||
"consumer": "a consumer's name, or <stream>.<consumer>: only what that one gave up on",
|
||||
@@ -8084,8 +8027,7 @@ func selfCheckVerbs() []catalogue.Verb {
|
||||
func healersVerbs() []catalogue.Verb {
|
||||
return []catalogue.Verb{
|
||||
{Name: "healers", Description: "The healers: each registered response to one kind of condition, its repair, its " +
|
||||
"budget and what happens when it is spent; every act taken lately with its outcome, the acts it would have " +
|
||||
"taken while the controller's lease names a controller, and the mesh-wide brake.",
|
||||
"budget and what happens when it is spent; every act taken lately with its outcome, and the mesh-wide brake.",
|
||||
Input: schema(map[string]string{"days": "how many days of acts back (default 7)"}, nil)},
|
||||
{Name: "heal", Description: "Take one healer's act for one open condition now, within its budget and the brake.",
|
||||
Input: schema(map[string]string{
|
||||
@@ -8110,13 +8052,12 @@ func wantsVerbs() []catalogue.Verb {
|
||||
{Name: "upgrade", Description: "Whether each module's new builds wait for a person: its policy (roll, together or record) " +
|
||||
"and where it comes from (a person, the module, the bus, its irreplaceable data, the default); the bus is never " +
|
||||
"rolled. With module, that one; with module, policy and why, a person's choice for it, recorded as a hand act; " +
|
||||
"with policy default, the choice taken back. With compare, every module whose policy differs from the one the " +
|
||||
"controller writes beside its compositions.",
|
||||
"with policy default, the choice taken back. With compare, each node's own composition and the assigner's findings.",
|
||||
Input: schema(map[string]string{
|
||||
"module": "one module",
|
||||
"policy": "with module: roll, together, record, or default to take a person's choice back",
|
||||
"why": "with policy: why, required and kept",
|
||||
"compare": "\"true\": every module whose policy at the assigner differs from the controller's",
|
||||
"compare": "\"true\": each node's own composition, by its digest, and the assigner's findings",
|
||||
}, nil, "compare")},
|
||||
{Name: "bus", Description: "What a bus upgrade would do: the fingerprint of the bus module each node runs against the " +
|
||||
"indexer's newest, and the last bus steps from the hand-act log and the assigner's puts of the bus. Read-only: " +
|
||||
|
||||
@@ -2588,7 +2588,7 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, manifest, refused string
|
||||
}{
|
||||
{name: "the bus module's tools removing", manifest: tools(broker, `{"removes":["DEAD_LETTER_NOTICES.controller","mesh-host_acts","ASSIGNMENTS"]}`)},
|
||||
{name: "the bus module's tools removing", manifest: tools(broker, `{"removes":["mesh-host_acts","controller-leftovers"]}`)},
|
||||
{name: "a removal outside the set", manifest: tools(broker, `{"removes":["EVENTS.a_m"]}`), refused: "not one of"},
|
||||
{name: "a removal beside a grant", manifest: tools(broker, `{"removes":["mesh-host_acts"],"invokes":["nats.nats_streams"]}`), refused: "with nothing else"},
|
||||
{name: "another module's tools removing", manifest: tools(`[]`, `{"removes":["mesh-host_acts"]}`), refused: "only the bus module"},
|
||||
@@ -2613,7 +2613,7 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) {
|
||||
}
|
||||
accounts := m.BusAccounts()
|
||||
if len(accounts) != 1 || accounts[0].ID != "nats-tools" || !accounts[0].Tools ||
|
||||
!strSliceEqual(accounts[0].Bus.Removes, []string{catalogue.RemovesControllerDeadLetterConsumer, catalogue.RemovesEngineActs, catalogue.RemovesAssignmentsStream}) {
|
||||
!strSliceEqual(accounts[0].Bus.Removes, []string{catalogue.RemovesEngineActs, catalogue.RemovesControllerLeftovers}) {
|
||||
t.Errorf("%s: the accounts are %+v", c.name, accounts)
|
||||
}
|
||||
// Resolved, the build goes and its bundles keep the bus (build.go).
|
||||
@@ -4337,6 +4337,41 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) {
|
||||
if err == nil || !strings.Contains(err.Error(), "granted the bus's snapshot API and nothing else") {
|
||||
t.Fatalf("a bus module declaring what it emits was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
// Nor what it invokes, with no bundle for the runtime to ask on its behalf.
|
||||
unbundledInvoke := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"invokes":["seat:mesh-hand-acts.hand-act"]}`)
|
||||
if _, err := ParseManifest(unbundledInvoke); err == nil || !strings.Contains(err.Error(), "its invokes would be granted nothing") {
|
||||
t.Fatalf("a bus module with no bundle declaring what it invokes was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
|
||||
// Its tools bundle asks what the module invokes (novox/hq ADR 0357; issue 774): the bundle asks through the
|
||||
// node's runtime, which is granted every tool and seat verb (mesh-sdk go/buscompose), so a bus module that
|
||||
// ships one may declare them. The shape of mesh-catalog's modules/nats/module.json at fc5b849d, cut to the
|
||||
// parts the rule reads, was refused and held every bus user list from 2026-11-02 01:35.
|
||||
nats := []byte(`{"module":"nats","version":"1","provides":[{"name":"mesh-bus","scope":"mesh","credential":{"form":"none"}}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],
|
||||
"own-secrets":{"broker":"/run/broker","broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"},
|
||||
"resources":[{"id":"tools-secret","type":"directory","path":"/var/lib/mesh/nats-tools","mode":"0711","unknown":"program"}],
|
||||
"emits":["consumer-removed","bucket-keys-removed","stream-removed","bucket-expiry-set"],
|
||||
"invokes":["seat:mesh-hand-acts.hand-act"],"consumes":[],
|
||||
"tools":["nats_server","nats_bucket_expiry"],
|
||||
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/nats-tools",
|
||||
"binary":"nats-tools","loads":["nats-tools"],"bus":{"removes":["controller-leftovers"]}}]}}`)
|
||||
nm, err := ParseManifest(nats)
|
||||
if err != nil {
|
||||
t.Fatalf("the bus module whose tools bundle invokes a seat verb was refused: %v", err)
|
||||
}
|
||||
nbuilt, err := nm.Resolve([]catalogue.Built{{Name: "tools", Kind: catalogue.ArtifactBundle, Reference: "store/tools:1", Digest: "sha256:1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if raw, err = json.Marshal(nbuilt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ParseManifestForInputs(raw); err != nil {
|
||||
t.Fatalf("the built manifest of the bus module whose tools bundle invokes a seat verb was refused: %v", err)
|
||||
}
|
||||
|
||||
// Its tools bundle says what it emits (novox/hq ADR 0357): the node's runtime carries the bundle and is
|
||||
// granted the module's events, so a bus module that ships one may declare them.
|
||||
@@ -4935,9 +4970,10 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
delivered[s.Delivers] = s.Name
|
||||
}
|
||||
}
|
||||
// Forty-seven with mesh-placement (novox/hq ADR 0567), serving its seven verbs.
|
||||
// Forty-seven with mesh-placement (novox/hq ADR 0567), serving its five verbs: adopt and hand-back went with the
|
||||
// controller (novox/hq ADR 0590, issue 1126).
|
||||
if s, ok := catalogue.SeatNamed(catalogue.PlacementSeat); !ok || s.Scope != catalogue.ScopeMesh || !reflect.DeepEqual(catalogue.VerbNames(s.Serves),
|
||||
[]string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) {
|
||||
[]string{"placement", "proposals", "propose", "withdraw", "again"}) {
|
||||
t.Errorf("the placement seat is %v, %v", ok, s)
|
||||
}
|
||||
// The checker's seat serves the try, which replaces the shell's merge check (ADR 0547 §10, ADR 0245).
|
||||
@@ -4958,12 +4994,12 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
// The four health seats (novox/hq ADR 0564 clause 1): at mesh scope, each held once, each naming ADR 0564, with the
|
||||
// verbs and events the record names; the events keep the controller's names (clause 3).
|
||||
for _, c := range []struct{ seat, verbs, emits string }{
|
||||
{catalogue.ConditionsSeat, "conditions,silence,compare,take-over,hand-back,resync", "condition-raised,condition-changed,condition-cleared"},
|
||||
{catalogue.ConditionsSeat, "conditions,silence,resync", "condition-raised,condition-changed,condition-cleared"},
|
||||
{catalogue.WatchdogsSeat, "signals,stuck,presence,dead-letters,resync", ""},
|
||||
{catalogue.SelfCheckSeat, "doctor,probes", "checked"},
|
||||
{catalogue.HealersSeat, "healers,heal,drill", "healer-acted"},
|
||||
// And mesh-config's switch (ADR 0565 §4), beside its config.
|
||||
{"mesh-config", "config,compare,take-over,hand-back", ""},
|
||||
// And mesh-config's config; its switch went with the controller (ADR 0590 clause 3).
|
||||
{"mesh-config", "config", ""},
|
||||
// And ADR 0565 clause 1's two.
|
||||
{catalogue.WantsSeat, "upgrade,bus", "policy-changed"},
|
||||
{catalogue.HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"},
|
||||
@@ -5013,7 +5049,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
// Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1);
|
||||
// fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts
|
||||
// (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2).
|
||||
// (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2);
|
||||
// fifty-seven with mesh-grants (ADR 0582 clause 1); fifty-eight with mesh-enrolment (novox/hq ADR 0587 §1); fifty-six
|
||||
// without the controller's seat and mesh-grants, which served its lease write alone (novox/hq ADR 0590, issue 1126).
|
||||
if len(catalogue.Seats()) != 56 {
|
||||
t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(catalogue.Seats()), seatNames())
|
||||
@@ -5250,10 +5288,10 @@ func TestUsingASeatNobodyDeclaresIsRefused(t *testing.T) {
|
||||
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:telegram-sender.status"}}, unclaimed, true},
|
||||
{"a verb of a mesh seat whose holder lives in another repository", "",
|
||||
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-watchdogs.signals"}}, telegram(), false},
|
||||
{"a verb a mesh seat does not promise", `the seat mesh-controller serves no verb "root-freed"`,
|
||||
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-controller.root-freed"}}, telegram(), false},
|
||||
{"the controller's verb: the controller answers its seat", "",
|
||||
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-controller.root-free"}}, telegram(), false},
|
||||
{"a verb a mesh seat does not promise", `the seat mesh-placement serves no verb "adopt"`,
|
||||
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-placement.adopt"}}, telegram(), false},
|
||||
{"the controller's seat is gone (ADR 0590)", `a verb of the seat "mesh-controller", ` + catalogue.UndeclaredSeat,
|
||||
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:mesh-controller.root-free"}}, telegram(), true},
|
||||
{"the engine's verb: every node's engine holds its seat", "",
|
||||
catalogue.Manifest{Module: "shop", Invokes: []string{"seat:node-engine.root-free"}}, telegram(), false},
|
||||
{"a verb the engine's seat does not promise", `the seat node-engine serves no verb "root-freed"`,
|
||||
|
||||
+8
-6
@@ -7,20 +7,22 @@ The copy carries the catalogue's private details: domains, first names in module
|
||||
private addresses. That is acceptable while mesh-sdk is private; if it ever goes public, this copy is
|
||||
a second place to clean besides the catalogue itself.
|
||||
|
||||
Only what go/catalogue's tests read is captured (R4b, issue 1379). Refreshed 2026-10-21 from the mains of both
|
||||
repositories; before that it was mesh-controller's copy, which still held two modules that had left the catalogue.
|
||||
Only what go/catalogue's tests read is captured (R4b, issue 1379), copied here from mesh-sdk with the judge's tests
|
||||
(issue 1585). mesh-host refreshed 2026-11-01 from its main, the catalogue 2026-10-21 from its main; before that it
|
||||
was mesh-controller's copy, which still held two modules that had left the catalogue.
|
||||
|
||||
mesh-catalog b5dbfbb784507e211372a7c9640d91f7d6dadd84 modules/*/module.json, modules/*/files/**, modules/*/templates/**, predefined.json
|
||||
mesh-host cdcba9ca73755854d21e4e657d7e46c32b9b5265 internal/apply/placement_guard.go
|
||||
mesh-host 233bb23bbcb1a829ce2a6164a31a5c5cc4f3d950 internal/apply/placement_guard.go
|
||||
|
||||
To move them, from this repository's go/ directory, with the two repositories checked out beside mesh-sdk:
|
||||
To move them, from mesh-delivery's modules/mesh-delivery-indexer directory, with the two repositories checked out
|
||||
beside mesh-delivery:
|
||||
|
||||
rm -rf testdata/beside/mesh-catalog testdata/beside/mesh-host
|
||||
mkdir -p testdata/beside/mesh-catalog testdata/beside/mesh-host/internal/apply
|
||||
git -C ../../mesh-catalog archive <commit> -- predefined.json ':(glob)modules/*/module.json' \
|
||||
git -C ../../../mesh-catalog archive <commit> -- predefined.json ':(glob)modules/*/module.json' \
|
||||
':(glob)modules/*/files/**' ':(glob)modules/*/templates/**' | tar -x -C testdata/beside/mesh-catalog
|
||||
find testdata/beside -name module.json -exec mv {} {}.captured \;
|
||||
git -C ../../mesh-host show <commit>:internal/apply/placement_guard.go \
|
||||
git -C ../../../mesh-host show <commit>:internal/apply/placement_guard.go \
|
||||
> testdata/beside/mesh-host/internal/apply/placement_guard.go.captured
|
||||
|
||||
and write the commits here. The placement guard is kept as .captured so no Go tool reads it as this
|
||||
|
||||
Vendored
+99
-4
@@ -56,7 +56,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// protectedRoots are paths no directory, access or mount source may be, nor hold.
|
||||
@@ -384,6 +384,9 @@ func ownedByAccount(owner string, a homeAccount) bool {
|
||||
if owner == a.Name {
|
||||
return true
|
||||
}
|
||||
if a.UID < 0 {
|
||||
return false // an account the declaration makes has no uid yet: by name alone, never by a number
|
||||
}
|
||||
user, _, _ := strings.Cut(owner, ":")
|
||||
if uid, err := strconv.Atoi(user); err == nil {
|
||||
return uid == a.UID
|
||||
@@ -413,12 +416,91 @@ const (
|
||||
// refusePath says why a path is not placed or mounted; nil when it may be. module is the resource's module,
|
||||
// owner a directory's declared owner.
|
||||
func refusePath(path string, kind placing, module, owner string) error {
|
||||
return refusePathWith(path, kind, module, owner, nil)
|
||||
}
|
||||
|
||||
// declaredHomes is the home of each account the declaration itself makes (a user resource with a home), by the
|
||||
// account's name; the account is not in the user database yet when the declaration is judged, and the engine makes
|
||||
// it where it is declared, before the directory that is owned by it is reached (novox/hq issue 1542). Its uid is
|
||||
// not known, so a directory is owned by it by name alone. Only for the directory placement guard; no other
|
||||
// reader of accountsOfHomes sees these.
|
||||
func declaredHomes(resources []declaration.Resource) map[string]homeAccount {
|
||||
var accounts []passwdEntry
|
||||
var out map[string]homeAccount
|
||||
for _, r := range resources {
|
||||
u, ok := r.(*declaration.User)
|
||||
if !ok || u.Name == "" {
|
||||
continue
|
||||
}
|
||||
home := filepath.Clean(u.Home)
|
||||
if u.Home == "" || !filepath.IsAbs(home) || home == "/" {
|
||||
continue
|
||||
}
|
||||
if accounts == nil {
|
||||
accounts = passwdEntries()
|
||||
}
|
||||
// Only ever adds a home. An account the machine has keeps the home it has (applyUser never moves it), so
|
||||
// a declared home of one is not its home, unless it is the very home it has: the account an earlier apply
|
||||
// made, as declared, which must be judged the same at every later apply. And a home at or below another
|
||||
// account's, or at or above a protected root, is not a new one.
|
||||
taken := false
|
||||
for _, e := range accounts {
|
||||
if e.name == u.Name && e.home != home {
|
||||
taken = true
|
||||
}
|
||||
if e.name != u.Name && e.home != "" && atOrBelow(home, e.home) {
|
||||
taken = true
|
||||
}
|
||||
}
|
||||
for _, root := range protectedRoots {
|
||||
taken = taken || home == root || below(root, home)
|
||||
}
|
||||
if taken {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]homeAccount{}
|
||||
}
|
||||
out[home] = homeAccount{Name: u.Name, UID: -1}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// passwdEntry is an account of the user database and its home, whatever its uid.
|
||||
type passwdEntry struct{ name, home string }
|
||||
|
||||
// passwdEntries is every account of the user database with its home (empty and root homes as "").
|
||||
func passwdEntries() []passwdEntry {
|
||||
out := []passwdEntry{}
|
||||
f, err := os.Open(passwdFile)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer f.Close()
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
fields := strings.Split(sc.Text(), ":")
|
||||
if len(fields) < 6 {
|
||||
continue
|
||||
}
|
||||
home := filepath.Clean(fields[5])
|
||||
if home == "." || home == "/" || home == "/nonexistent" {
|
||||
home = ""
|
||||
}
|
||||
out = append(out, passwdEntry{fields[0], home})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// refusePathWith is refusePath with the homes of the accounts the declaration makes (declaredHomes) counted
|
||||
// among the machine's.
|
||||
func refusePathWith(path string, kind placing, module, owner string, declared map[string]homeAccount) error {
|
||||
clean := filepath.Clean(path)
|
||||
if !filepath.IsAbs(clean) {
|
||||
return nil // the declaration refuses a relative path already; a named volume is not a path
|
||||
}
|
||||
for _, p := range []string{clean, resolved(clean)} {
|
||||
if err := refuseOne(p, kind, module, owner); err != nil {
|
||||
if err := refuseOne(p, kind, module, owner, declared); err != nil {
|
||||
if p != clean {
|
||||
err.Why = fmt.Sprintf("through a link, it is %s, and %s", p, err.Why)
|
||||
err.Path = clean
|
||||
@@ -429,7 +511,7 @@ func refusePath(path string, kind placing, module, owner string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func refuseOne(path string, kind placing, module, owner string) *PlacementRefusedError {
|
||||
func refuseOne(path string, kind placing, module, owner string, declared map[string]homeAccount) *PlacementRefusedError {
|
||||
for _, root := range protectedRoots {
|
||||
if path == root || below(root, path) {
|
||||
return &PlacementRefusedError{Path: path, Why: root + " is one of the machine's own directories, " +
|
||||
@@ -482,6 +564,18 @@ func refuseOne(path string, kind placing, module, owner string) *PlacementRefuse
|
||||
return nil
|
||||
}
|
||||
homes := accountsOfHomes()
|
||||
if ssh && len(declared) > 0 {
|
||||
merged := map[string]homeAccount{}
|
||||
for home, a := range homes {
|
||||
merged[home] = a
|
||||
}
|
||||
for home, a := range declared {
|
||||
if _, held := merged[home]; !held {
|
||||
merged[home] = a
|
||||
}
|
||||
}
|
||||
homes = merged
|
||||
}
|
||||
var deepest string
|
||||
for home := range homes {
|
||||
if below(path, home) && len(home) > len(deepest) {
|
||||
@@ -518,11 +612,12 @@ func moduleOfID(id string) string {
|
||||
// a file into it, because nothing judged the file's path.
|
||||
func refusedPlaces(resources []declaration.Resource) map[string]error {
|
||||
out := map[string]error{}
|
||||
declared := declaredHomes(resources)
|
||||
for _, r := range resources {
|
||||
var err error
|
||||
switch res := r.(type) {
|
||||
case *declaration.Directory:
|
||||
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
|
||||
err = refusePathWith(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner, declared)
|
||||
// And outside every home, the walk's judgement before anything is applied (novox/hq issue 358): a
|
||||
// link another account could make in its path refuses it, and with it what is below it, whatever
|
||||
// order they are declared in. The apply walks again at the moment of use. A directory handed over
|
||||
|
||||
@@ -37,14 +37,20 @@ func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string
|
||||
}
|
||||
}
|
||||
|
||||
// recordVia reads the router's record of one of this module's asks: the key `<module>.<id>` of the router's bucket asks,
|
||||
// by a direct get on the subject the controller grants for it, `$JS.API.DIRECT.GET.KV_asks.$KV.asks.<module>.>`. The
|
||||
// record is {state, warrant}. An ask the router holds no record of is nil.
|
||||
// messengerAsks is the router's records bucket as the seat operator-channel's manifest declares it (its `records`), the
|
||||
// one the controller composes this account's direct get on (mesh-sdk buscompose SeatTrafficOf): never `asks`, a bucket
|
||||
// no grant names (issue 774). The SDK names no constant for it.
|
||||
const messengerAsks = "messenger_asks"
|
||||
|
||||
// recordVia reads the router's record of one of this module's asks: the key `<module>.<id>` of the router's bucket
|
||||
// messenger_asks, by a direct get on the subject the controller grants for it,
|
||||
// `$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.<module>.>`. The record is {state, warrant}. An ask the
|
||||
// router holds no record of is nil.
|
||||
func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id string) (*stdio.StateEntry, error) {
|
||||
return func(id string) (*stdio.StateEntry, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_asks.$KV.asks."+module+"."+id, nil)
|
||||
msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+messengerAsks+".$KV."+messengerAsks+"."+module+"."+id, nil)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -130,9 +130,9 @@ func TestTellerWaitsForTheGate(t *testing.T) { // 1240/AC2
|
||||
// them (internal/broker): a module's own state is the read grants of its bucket and `$KV.<bucket>.>` to write it
|
||||
// (state.go stateGrants, readGrants), a read of another module's bucket is the read grants alone, and a module that uses
|
||||
// the seat operator-channel submits its own `accept.ask.<module>`, `accept.cancel.<module>` and `accept.condition.<module>`,
|
||||
// hears its own `event.decided.<module>` and reads `$KV.asks.<module>.>` of the router's record (seattraffic.go
|
||||
// hears its own `event.decided.<module>` and reads `$KV.messenger_asks.<module>.>` of the router's record (seattraffic.go
|
||||
// SeatTrafficOf, with the messenger's seat declaring accepts ask, cancel, condition, emits decided, by-caller and records
|
||||
// asks). The gate is a bundle on the tool runner's account (ADR 0437 §3), so the gate's writes are the account node-tools'.
|
||||
// messenger_asks). The gate is a bundle on the tool runner's account (ADR 0437 §3), so the gate's writes are the account node-tools'.
|
||||
// The permissions are these lists written out, not the controller's function: the broker tests of issue 1243 prove that.
|
||||
func readOf(bucket string) []string {
|
||||
stream := "KV_" + bucket
|
||||
@@ -176,7 +176,7 @@ const (
|
||||
|
||||
func fullConf(dir string) string {
|
||||
teller := append(append(append(keepsOf(tellerAnswers), keepsOf(tellerAsked)...), readOf(gateAsksKV)...),
|
||||
seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_asks.$KV.asks.mesh-delivery-teller.>")
|
||||
seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_"+messengerAsks+".$KV."+messengerAsks+".mesh-delivery-teller.>")
|
||||
teller = append(append(teller, readOf(assignerAsks)...), readOf(viewAnswers)...) // issue 1297 B10: the manifest's reads
|
||||
view := append(append(append(readOf(tellerAnswers), readOf(gateAsksKV)...), readOf(assignerAsks)...), keepsOf(viewAnswers)...)
|
||||
assigner := keepsOf(assignerAsks)
|
||||
@@ -244,7 +244,7 @@ func startFull(t *testing.T) *fullBed {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
for name, history := range map[string]uint8{gateAsksKV: delivery.AsksHistory, tellerAnswers: delivery.AnswersHistory, tellerAsked: 1, assignerAsks: delivery.AssignerAsksHistory, viewAnswers: delivery.ViewAnswersHistory} {
|
||||
for name, history := range map[string]uint8{gateAsksKV: delivery.AsksHistory, tellerAnswers: delivery.AnswersHistory, tellerAsked: 1, messengerAsks: 1, assignerAsks: delivery.AssignerAsksHistory, viewAnswers: delivery.ViewAnswersHistory} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -331,6 +331,21 @@ func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
|
||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||
|
||||
// Issue 774: the router's record is read on the teller's own credential from the bucket the controller grants it.
|
||||
if mkv, err := b.js.KeyValue(b.ctx, messengerAsks); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := mkv.Put(b.ctx, "mesh-delivery-teller.r1", []byte(`{"state":"open"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tc, err := nats.Connect(b.srv.ClientURL(), nats.UserInfo("n1.mesh-delivery-teller", "pw"), nats.CustomInboxPrefix("_INBOX.n1.mesh-delivery-teller"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(tc.Close)
|
||||
if rec, err := recordVia(b.ctx, tc, "mesh-delivery-teller")("r1"); err != nil || rec == nil {
|
||||
t.Fatalf("the router's record read as the teller: %v, %v; want it", rec, err)
|
||||
}
|
||||
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
kv, err := tools.KeyValue(b.ctx, gateAsksKV)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -33,12 +33,15 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
"containing, as the view derives it from the indexer's modules, the wants and the haves; a merge at or before it is on the " +
|
||||
"machines), what holds the next one (a module on a node, the indexer's lag), and per module its fingerprint, the nodes " +
|
||||
"that want it, its changes (fingerprint, the merge it first appeared at, its source revision), its floor and the nodes " +
|
||||
"where it is not settled or not healthy (novox/hq ADR 0556). Give a repository (owner/name) for one, none for every one. Read-only.",
|
||||
"where it is not settled or not healthy (novox/hq ADR 0556). Give a repository (owner/name) for one, none for every one; give a module to see only that module of each " +
|
||||
"repository and the holds that name it. Answers the delivery reach of a repository and of a module. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "the repository, owner/name; none for every repository"}}},
|
||||
"repository": map[string]any{"type": "string", "description": "the repository, owner/name; none for every repository"},
|
||||
"module": map[string]any{"type": "string", "description": "the module name; none for every module"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
return r.Reach(repository)
|
||||
module, _ := args["module"].(string)
|
||||
return r.Reach(repository, module)
|
||||
}},
|
||||
{Name: view.ModuleHistoryTool,
|
||||
Description: "The history of one module's value in the indexer's modules state (modules.<module>), with times: each value it " +
|
||||
@@ -216,8 +219,7 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
{Name: view.TrustsReadTool,
|
||||
Description: "What every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2): each node's record's trusts " +
|
||||
"(from node-engine_have <node>.node), whether it names the vault's signing key as mesh-vault publishes it in its " +
|
||||
"state signers and whether it still names the controller's, the nodes that do not name the vault's yet (not_yet), " +
|
||||
"and every: true once all do, the gate before the controller's lease is handed away. Read-only.",
|
||||
"state signers, the nodes that do not name the vault's yet (not_yet), and every: true once all do. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return r.Trusts()
|
||||
|
||||
@@ -89,6 +89,19 @@ func TestModuleToolAnswersTheStoredValue(t *testing.T) {
|
||||
if _, err := run(t, r, view.ReachReadTool, map[string]any{"repository": "novox/gone"}); err == nil || !strings.Contains(err.Error(), "no reach of novox/gone") {
|
||||
t.Errorf("a repository with no reach: %v", err)
|
||||
}
|
||||
// The module argument keeps only that module of each repository, and only the holds that name it.
|
||||
const reach2 = `{"v":1,"repository":"novox/web","reached":"c1","reached_src":12,"modules":{"web":{"nodes":1},"api":{"nodes":2}},"holds":["web on g14 runs 1","api on g14 runs 2","the indexer has not yet taken in x"]}`
|
||||
r.ReachState = fake{"novox/web": reach2, "novox/other": `{"v":1,"repository":"novox/other","modules":{"o":{"nodes":1}}}`}.reader().State
|
||||
if out, err := run(t, r, view.ReachReadTool, map[string]any{"repository": "novox/web", "module": "web"}); err != nil ||
|
||||
!strings.Contains(string(out.(json.RawMessage)), `"web"`) || strings.Contains(string(out.(json.RawMessage)), `"api"`) || strings.Contains(string(out.(json.RawMessage)), "api on") || !strings.Contains(string(out.(json.RawMessage)), "web on g14") {
|
||||
t.Errorf("view_reach novox/web module web = %s, %v", out, err)
|
||||
}
|
||||
if out, err := run(t, r, view.ReachReadTool, map[string]any{"module": "api"}); err != nil || !strings.Contains(fmt.Sprint(out), "api") || strings.Contains(fmt.Sprint(out), "novox/other") {
|
||||
t.Errorf("view_reach module api over every repository = %v, %v", out, err)
|
||||
}
|
||||
if _, err := run(t, r, view.ReachReadTool, map[string]any{"repository": "novox/web", "module": "nope"}); err == nil || !strings.Contains(err.Error(), "no module nope") {
|
||||
t.Errorf("a module the repository does not hold: %v", err)
|
||||
}
|
||||
// Issue 1126 note 76: indexer_module_history answers modules.<module>'s kept values newest first, each with its
|
||||
// revision, time, commit, source revision and fp2; indexer_heard the FORGE messages the indexer handled, newest first.
|
||||
at := time.Date(2026, 10, 28, 13, 7, 46, 0, time.UTC)
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -102,8 +102,61 @@ func (r Reader) Status(now time.Time) Status {
|
||||
// ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556).
|
||||
const ReachReadTool = "view_reach"
|
||||
|
||||
// Reach is the stored reach of one repository, or of every repository when none is named, as the view wrote it.
|
||||
func (r Reader) Reach(repository string) (any, error) {
|
||||
// Reach is the stored reach of one repository, or of every repository when none is named, as the view wrote it. With a
|
||||
// module, each repository's value holds only that module and the holds that name it, and a repository without it is left out.
|
||||
func (r Reader) Reach(repository, module string) (any, error) {
|
||||
out, err := r.reach(repository)
|
||||
if module = strings.TrimSpace(module); module == "" || err != nil {
|
||||
return out, err
|
||||
}
|
||||
if raw, ok := out.(json.RawMessage); ok {
|
||||
one, err := onlyModule(raw, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if one == nil {
|
||||
return nil, fmt.Errorf("no module %s in the reach of %s", module, repository)
|
||||
}
|
||||
return one, nil
|
||||
}
|
||||
var kept []json.RawMessage
|
||||
for _, raw := range out.(map[string]any)["reach"].([]json.RawMessage) {
|
||||
one, err := onlyModule(raw, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if one != nil {
|
||||
kept = append(kept, one)
|
||||
}
|
||||
}
|
||||
return map[string]any{"reach": kept}, nil
|
||||
}
|
||||
|
||||
// onlyModule is a reach value with only module among its modules and only the holds that name it, nil when it holds none.
|
||||
func onlyModule(raw json.RawMessage, module string) (json.RawMessage, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rc, err := delivery.DecodeReach(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mr, ok := rc.Modules[module]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
rc.Modules = map[string]delivery.ModuleReach{module: mr}
|
||||
var holds []string
|
||||
for _, h := range rc.Holds {
|
||||
if strings.HasPrefix(h, module+" ") {
|
||||
holds = append(holds, h)
|
||||
}
|
||||
}
|
||||
rc.Holds = holds
|
||||
return json.Marshal(rc)
|
||||
}
|
||||
|
||||
func (r Reader) reach(repository string) (any, error) {
|
||||
if r.ReachState.Get == nil || r.ReachState.Keys == nil {
|
||||
return nil, fmt.Errorf("the reach state cannot be read here")
|
||||
}
|
||||
@@ -704,15 +757,14 @@ func (r Reader) Haves(node, module string) (KeptAnswer, error) {
|
||||
}
|
||||
|
||||
// TrustsReadTool reads what every node trusts a mesh signature from, in one call (novox/hq ADR 0550 §2; design 47's L3
|
||||
// s6b): the gate before the lease write is that every node's record names the vault's key.
|
||||
// s6b): whether every node's record names the vault's key.
|
||||
const TrustsReadTool = "node_trusts"
|
||||
|
||||
// NodeTrusts is what one node's record says it trusts: its lines (delivery.Trusts) and whether they name the vault's key
|
||||
// the vault publishes and the controller's key.
|
||||
// the vault publishes.
|
||||
type NodeTrusts struct {
|
||||
Trusts []string `json:"trusts"`
|
||||
TrustsVault bool `json:"trusts_vault"`
|
||||
TrustsController bool `json:"trusts_controller"`
|
||||
Trusts []string `json:"trusts"`
|
||||
TrustsVault bool `json:"trusts_vault"`
|
||||
}
|
||||
|
||||
// TrustsAnswer is node_trusts' answer: the vault's key as it publishes it, each node's trusts, the nodes whose record does
|
||||
@@ -771,9 +823,6 @@ func (r Reader) Trusts() (TrustsAnswer, error) {
|
||||
if h.Record != nil {
|
||||
t.Trusts = h.Record.Trusts
|
||||
}
|
||||
for _, line := range t.Trusts {
|
||||
t.TrustsController = t.TrustsController || strings.HasPrefix(line, delivery.SignerController+" key ")
|
||||
}
|
||||
t.TrustsVault = out.VaultKey != "" && delivery.TrustsKey(t.Trusts, delivery.SignerVault, out.VaultKey)
|
||||
out.Nodes[node] = t
|
||||
if !t.TrustsVault {
|
||||
|
||||
@@ -140,9 +140,9 @@ func Tools(states func(string) state, ask func(socket.Ask) (json.RawMessage, err
|
||||
{Name: verb(VerbDeadLetters), Run: dead,
|
||||
Description: "What a consumer gave up on, kept in DEAD_LETTERS (novox/hq ADR 0564 §5, ADR 0264, ADR 0376): newest first, " +
|
||||
"for one consumer (its name or <stream>.<consumer>), at most limit; one whole by id (a restricted one's body " +
|
||||
"withheld). Who keeps them now: the controller until the lease names no controller, the watchman from then. " +
|
||||
"Delivering again and dropping (act deliver or drop, with why) are refused here until the hand-act log's home " +
|
||||
"(issue 1953): before the lease write use mesh-controller.dead-letters.",
|
||||
"withheld). The watchman keeps them once the controller's consumer on DEAD_LETTER_NOTICES is gone, and says the " +
|
||||
"notices wait while it stands. Delivering again and dropping (act deliver or drop, with why) are refused here until " +
|
||||
"the hand-act log's home (issue 1953).",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"consumer": map[string]any{"type": "string", "description": "only this consumer's, its name or <stream>.<consumer>"},
|
||||
"limit": map[string]any{"type": "integer", "description": "at most this many, newest first"},
|
||||
|
||||
@@ -117,12 +117,8 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
|
||||
kvs[c.Bucket] = kv
|
||||
}
|
||||
|
||||
// The controller's lease, its consumer on DEAD_LETTER_NOTICES, the dead letters' streams and a source stream with one
|
||||
// message a consumer will give up on (1952/WM5).
|
||||
lease, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: buscompose.LeaseBucket})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller's consumer on DEAD_LETTER_NOTICES, the dead letters' streams and a source stream with one message a
|
||||
// consumer will give up on (1952/WM5); no lease bucket at all (issue 1126).
|
||||
for _, c := range []jetstream.StreamConfig{
|
||||
{Name: buscompose.DeadLetterNoticesStream, Subjects: []string{buscompose.MaxDeliveriesAdvisories}, Retention: jetstream.WorkQueuePolicy},
|
||||
{Name: buscompose.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}, Duplicates: 24 * time.Hour, AllowMsgTTL: true},
|
||||
@@ -269,15 +265,8 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
|
||||
})
|
||||
until("node n1's silence", func() bool { return finding("machine.n1.silent") })
|
||||
|
||||
// 1952/WM5: while the lease names a controller the watchman takes no notice; handed away while the controller's consumer
|
||||
// stands, it says the notices wait; the consumer removed (the lease write's step), it makes its own and keeps a notice
|
||||
// with its headers.
|
||||
if _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer); err == nil {
|
||||
t.Fatal("the watchman made its consumer while the lease names a controller")
|
||||
}
|
||||
if _, err := lease.Put(ctx, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator","why":"the lease write"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 1952/WM5, 1126/A2/AC2: with no lease written, while the controller's consumer stands the watchman says the notices
|
||||
// wait; the consumer removed, it makes its own and keeps a notice with its headers.
|
||||
waits := watch.FindingKey(watch.ScopeBus, "DEAD_LETTER_NOTICES.controller", watch.KindNoticesWaiting)
|
||||
until("the notices said waiting behind the controller's consumer", func() bool { return finding(waits) })
|
||||
if _, err := js.Consumer(ctx, buscompose.DeadLetterNoticesStream, deadletters.Consumer); err == nil {
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
@@ -171,7 +170,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants
|
||||
// and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build
|
||||
// seats' steps, and says that moves are not watched (issue 1945).
|
||||
var kvs struct{ presence, pulse, stuck, findings, wants, haves, lease jetstream.KeyValue }
|
||||
var kvs struct{ presence, pulse, stuck, findings, wants, haves jetstream.KeyValue }
|
||||
for _, b := range []struct {
|
||||
name string
|
||||
into *jetstream.KeyValue
|
||||
@@ -186,7 +185,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it
|
||||
}
|
||||
|
||||
kvs.lease = process.OptionalBucket(ctx, js, buscompose.LeaseBucket, "the lease is read as naming a controller, so the dead letters stay the controller's", logger.Printf)
|
||||
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now)
|
||||
w.Logf(logger.Printf)
|
||||
// A test shortens how long a move may take; the mesh never sets it.
|
||||
@@ -379,23 +377,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
errs <- follow(ctx, provWatch, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) })
|
||||
}()
|
||||
}
|
||||
if kvs.lease != nil {
|
||||
lw, err := kvs.lease.Watch(ctx, buscompose.ControllerLeaseKey)
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, buscompose.LeaseBucket, err, "the lease is read as naming a controller, so the dead letters stay the controller's")
|
||||
} else {
|
||||
go func() {
|
||||
errs <- follow(ctx, lw, func(e jetstream.KeyValueEntry) {
|
||||
var raw []byte
|
||||
if opOf(e) == watch.Put {
|
||||
raw = e.Value()
|
||||
}
|
||||
on, _ := buscompose.LeaseHandedAway(raw, len(raw) > 0, nil)
|
||||
dead.lease(on)
|
||||
})
|
||||
}()
|
||||
}
|
||||
}
|
||||
if l, err := socket.Listen(socket.PathOf(getenv)); err != nil {
|
||||
logger.Printf("the tools' socket cannot be opened (%v): dead-letters and resync answer nothing until the watchman starts again", err)
|
||||
} else {
|
||||
|
||||
@@ -37,6 +37,13 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
|
||||
if routeOf("mesh.mod.web.event.started") != "" {
|
||||
t.Error("another module's event goes to a row")
|
||||
}
|
||||
// 1126/B9 (ADR 0590 §3): the lease write's events went with it; the grants part says no lease event any more.
|
||||
for _, gone := range []string{"mesh.seat.mesh-grants.event.lease-altered", "mesh.seat.mesh-grants.event.removal-failed",
|
||||
"$JS.EVENT.ADVISORY.STREAM.DELETED.ASSIGNMENTS"} {
|
||||
if r := routeOf(gone); r != "" && r != routeAdvisory {
|
||||
t.Errorf("%s goes to the row %s", gone, r)
|
||||
}
|
||||
}
|
||||
})
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||
|
||||
@@ -95,7 +95,7 @@ func heard(w *watch.Watchman, dead *deadKeeper, m *nats.Msg, logf func(string, .
|
||||
return
|
||||
}
|
||||
if dead.isTheControllersNotices(a.ID) {
|
||||
dead.check() // the lease write's step removed it: the watchman may take the notices now
|
||||
dead.check() // the controller's consumer removed: the watchman may take the notices now
|
||||
return
|
||||
}
|
||||
w.Advised(a)
|
||||
@@ -137,17 +137,16 @@ func hear(conn *nats.Conn, w *watch.Watchman, dead *deadKeeper, logf func(string
|
||||
return conn.Flush()
|
||||
}
|
||||
|
||||
// deadKeeper keeps what consumers gave up on from the lease write (ADR 0564 §5): it counts DEAD_LETTERS into findings,
|
||||
// and takes DEAD_LETTER_NOTICES on its own consumer only once the lease names no controller and `controller` is gone.
|
||||
// deadKeeper keeps what consumers gave up on (ADR 0564 §5): it counts DEAD_LETTERS into findings, and takes
|
||||
// DEAD_LETTER_NOTICES on its own consumer once `controller` is gone (issue 1126: no lease is read).
|
||||
type deadKeeper struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
w *watch.Watchman
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
handedAway bool
|
||||
taking jetstream.ConsumeContext
|
||||
mu sync.Mutex
|
||||
taking jetstream.ConsumeContext
|
||||
}
|
||||
|
||||
// ages pulses each consumer's restricted letters by when they age out, read from DEAD_LETTERS' headers (at the start and
|
||||
@@ -198,38 +197,20 @@ func (d *deadKeeper) count() {
|
||||
d.w.DeadLettersHeld(held, deadletters.ConsumerInWords)
|
||||
}
|
||||
|
||||
// lease is the lease's change heard: handed away or not.
|
||||
func (d *deadKeeper) lease(handedAway bool) {
|
||||
d.mu.Lock()
|
||||
d.handedAway = handedAway
|
||||
d.mu.Unlock()
|
||||
d.check()
|
||||
}
|
||||
|
||||
func (d *deadKeeper) away() bool {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return d.handedAway
|
||||
}
|
||||
|
||||
// isTheControllersNotices says an advisory is about the controller's consumer on DEAD_LETTER_NOTICES once the lease is
|
||||
// handed away: its removal is the lease write's step, not a loss.
|
||||
// isTheControllersNotices says an advisory is about the controller's consumer on DEAD_LETTER_NOTICES: its removal is the
|
||||
// controller's end, not a loss.
|
||||
func (d *deadKeeper) isTheControllersNotices(id string) bool {
|
||||
return d.away() && id == buscompose.DeadLetterNoticesStream+"."+buscompose.ControllerName
|
||||
return id == buscompose.DeadLetterNoticesStream+"."+buscompose.ControllerName
|
||||
}
|
||||
|
||||
// check takes the notices, or says they wait, as Gate has it.
|
||||
func (d *deadKeeper) check() {
|
||||
away := d.away()
|
||||
stands := true
|
||||
if away {
|
||||
var err error
|
||||
if stands, err = deadletters.ControllerStands(d.ctx, d.js); err != nil {
|
||||
d.logf("whether the controller's consumer on %s stands cannot be read (%v): it is taken to stand", buscompose.DeadLetterNoticesStream, err)
|
||||
stands = true
|
||||
}
|
||||
stands, err := deadletters.ControllerStands(d.ctx, d.js)
|
||||
if err != nil {
|
||||
d.logf("whether the controller's consumer on %s stands cannot be read (%v): it is taken to stand", buscompose.DeadLetterNoticesStream, err)
|
||||
stands = true
|
||||
}
|
||||
take, waiting := deadletters.Gate(away, stands)
|
||||
take, waiting := deadletters.Gate(stands)
|
||||
d.w.NoticesWaiting(waiting)
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
@@ -242,7 +223,7 @@ func (d *deadKeeper) check() {
|
||||
return
|
||||
}
|
||||
d.taking = cc
|
||||
d.logf("the lease names no controller and its consumer on %s is gone: the watchman keeps the dead letters", buscompose.DeadLetterNoticesStream)
|
||||
d.logf("the controller's consumer on %s is gone: the watchman keeps the dead letters", buscompose.DeadLetterNoticesStream)
|
||||
}
|
||||
|
||||
// answer is the program's answer to the tools' ask on its socket.
|
||||
@@ -262,7 +243,7 @@ func answer(a socket.Ask, w *watch.Watchman, dead *deadKeeper, ask func([]byte)
|
||||
if strings.TrimSpace(a.Why) == "" {
|
||||
return nil, errors.New("a dead letter is delivered again or dropped only with why")
|
||||
}
|
||||
return nil, deadletters.Refusal(dead.away(), a.Act)
|
||||
return nil, deadletters.Refusal(a.Act)
|
||||
case a.Act != "":
|
||||
return nil, fmt.Errorf("%q is not deliver or drop", a.Act)
|
||||
case a.ID > 0:
|
||||
@@ -272,11 +253,7 @@ func answer(a socket.Ask, w *watch.Watchman, dead *deadKeeper, ask func([]byte)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keeper := "the controller (it holds its lease)"
|
||||
if dead.away() {
|
||||
keeper = "the watchman"
|
||||
}
|
||||
return map[string]any{"dead-letters": list, "total": total, "kept-by": keeper}, nil
|
||||
return map[string]any{"dead-letters": list, "total": total, "kept-by": "the watchman"}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("refused: the watchman's socket answers %q and %q", socket.AskDeadLetters, socket.AskResync)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
// Package deadletters is what a consumer gave up on, kept by the watchman from the lease write (novox/hq ADR 0564 §5,
|
||||
// Package deadletters is what a consumer gave up on, kept by the watchman since the controller's end (novox/hq ADR 0564 §5,
|
||||
// ADR 0264, ADR 0376, issue 1952 WM5): ported from mesh-controller internal/link/deadletters.go at e04846e9 with nothing
|
||||
// changed in what is kept, aged or withheld, on the jetstream API.
|
||||
//
|
||||
// DEAD_LETTER_NOTICES is a work queue that holds one consumer. The controller's consumer `controller` never expires, so
|
||||
// the watchman makes its own (Consumer) only when the lease names no controller and `controller` is gone (Gate); while
|
||||
// `controller` still stands after the lease write it says so as a finding. Removing `controller` is a step of the lease
|
||||
// write, with the bus module's tool. Before the lease write the watchman only lists and shows DEAD_LETTERS, which the
|
||||
// controller still fills, and refuses to deliver or drop, naming the controller's verb; from it, delivering and dropping
|
||||
// wait for the hand-act log's home (issue 1953), so they are refused naming that.
|
||||
// DEAD_LETTER_NOTICES is a work queue that holds one consumer. The ended controller's consumer `controller` never
|
||||
// expires, so the watchman makes its own (Consumer) only once `controller` is gone (Gate); while it stands the watchman says
|
||||
// so as a finding. Removing `controller` is a hand act with the bus module's tool (novox/hq ADR 0590 clause 6). Delivering
|
||||
// and dropping wait for the hand-act log's home (issue 1953), so they are refused naming that.
|
||||
package deadletters
|
||||
|
||||
import (
|
||||
@@ -305,27 +303,22 @@ func Named(ctx context.Context, js jetstream.JetStream, id uint64) (DeadLetter,
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Refusal is why the watchman delivers nothing again and drops nothing: before the lease write the controller keeps them
|
||||
// and its verb acts; from it the act waits for the hand-act log's home (issue 1953), where a person's act is recorded.
|
||||
// act is deliver or drop.
|
||||
func Refusal(handedAway bool, act string) error {
|
||||
// Refusal is why the watchman delivers nothing again and drops nothing: the act waits for the hand-act log's home (issue
|
||||
// 1953), where a person's act is recorded. act is deliver or drop.
|
||||
func Refusal(act string) error {
|
||||
done := map[string]string{"deliver": "delivered again", "drop": "dropped"}[act]
|
||||
if done == "" {
|
||||
return fmt.Errorf("%q is not deliver or drop", act)
|
||||
}
|
||||
if !handedAway {
|
||||
return fmt.Errorf("the controller still holds its lease and keeps the dead letters: %s it with "+
|
||||
"mesh-controller.dead-letters (novox/hq ADR 0564 §5); nothing is %s here", act, done)
|
||||
}
|
||||
return fmt.Errorf("a dead letter is %s only as a person's act recorded in the hand-act log, whose home is not "+
|
||||
"delivered yet (issue 1953): nothing is %s", done, done)
|
||||
}
|
||||
|
||||
// Gate is whether the watchman takes the notices itself: only when the lease names no controller (handedAway) and the
|
||||
// controller's consumer is gone. waiting says the lease is handed away while `controller` still stands, which is the
|
||||
// watchman's finding: each notice waits in the queue until the lease write's step removes it.
|
||||
func Gate(handedAway, controllerStands bool) (take, waiting bool) {
|
||||
return handedAway && !controllerStands, handedAway && controllerStands
|
||||
// Gate is whether the watchman takes the notices itself: once the controller's consumer is gone. waiting says `controller`
|
||||
// still stands, which is the watchman's finding: each notice waits in the queue until that consumer is removed (the
|
||||
// controller ended by its unassignment, issue 1126).
|
||||
func Gate(controllerStands bool) (take, waiting bool) {
|
||||
return !controllerStands, controllerStands
|
||||
}
|
||||
|
||||
// ControllerStands asks the bus whether the controller's consumer on DEAD_LETTER_NOTICES stands.
|
||||
|
||||
@@ -8,36 +8,32 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// 1952/WM5, ADR 0564 §5: the watchman makes no consumer while `controller` exists or the lease names a controller, makes
|
||||
// one once both are false, and says the wait meanwhile; it delivers and drops nothing, naming who does; and a kept message
|
||||
// reads as the controller's copies do, a restricted one's age and work queue kept.
|
||||
func TestTheWatchmanKeepsTheDeadLettersOnlyFromTheLeaseWrite(t *testing.T) {
|
||||
// 1952/WM5, ADR 0564 §5, 1126/A2/AC2: with no lease read at all (the controller ended by its unassignment), the watchman
|
||||
// makes no consumer while `controller` stands on DEAD_LETTER_NOTICES and says the wait, and makes its own once it is gone;
|
||||
// it delivers and drops nothing, naming why; and a kept message reads as the controller's copies do, a restricted one's
|
||||
// age and work queue kept.
|
||||
func TestTheWatchmanKeepsTheDeadLetters(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
handedAway, controllerStands, take, waiting bool
|
||||
controllerStands, take, waiting bool
|
||||
}{
|
||||
{false, true, false, false},
|
||||
{false, false, false, false},
|
||||
{true, true, false, true},
|
||||
{true, false, true, false},
|
||||
{true, false, true},
|
||||
{false, true, false},
|
||||
} {
|
||||
take, waiting := Gate(c.handedAway, c.controllerStands)
|
||||
take, waiting := Gate(c.controllerStands)
|
||||
if take != c.take || waiting != c.waiting {
|
||||
t.Errorf("Gate(%v, %v) = %v, %v", c.handedAway, c.controllerStands, take, waiting)
|
||||
t.Errorf("Gate(%v) = %v, %v", c.controllerStands, take, waiting)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
away bool
|
||||
act string
|
||||
says string
|
||||
}{
|
||||
{false, "deliver", "mesh-controller.dead-letters"},
|
||||
{false, "drop", "nothing is dropped"},
|
||||
{true, "drop", "issue 1953"},
|
||||
{true, "deliver", "delivered again"},
|
||||
{true, "send", "not deliver or drop"},
|
||||
{"drop", "issue 1953"},
|
||||
{"deliver", "delivered again"},
|
||||
{"send", "not deliver or drop"},
|
||||
} {
|
||||
if err := Refusal(c.away, c.act); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("Refusal(%v, %s) = %v, want it to say %q", c.away, c.act, err, c.says)
|
||||
if err := Refusal(c.act); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("Refusal(%s) = %v, want it to say %q", c.act, err, c.says)
|
||||
}
|
||||
}
|
||||
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
// The bus's advisories the watchman hears (S9, mesh-controller internal/link/advisories.go at e04846e9): a durable consumer
|
||||
// the mesh defines deleted is `consumer-lost`. A message given up on is said from DEAD_LETTERS, where it is kept, for as
|
||||
// long as it is kept (DeadLettersHeld), not from its advisory. The controller's own slow consumer and refused subjects end
|
||||
// long as it is kept (DeadLettersHeld), not from its advisory. The controller's own slow consumer and refused subjects ended
|
||||
// with it (ADR 0564 §4).
|
||||
|
||||
// Advisory subjects the watchman subscribes.
|
||||
@@ -48,7 +48,9 @@ func MeshNamed(stream, name string) bool {
|
||||
case strings.HasPrefix(stream, "SEAT_"):
|
||||
return strings.HasSuffix(name, "_worker")
|
||||
case name == "controller" || name == "controller-merges":
|
||||
return true
|
||||
// The ended controller's consumers are removed as hand acts (novox/hq ADR 0590 clause 6), never lost; its consumer on
|
||||
// DEAD_LETTER_NOTICES alone is read, whose removal tells the watchman to take the notices.
|
||||
return stream == "DEAD_LETTER_NOTICES" && name == "controller"
|
||||
case stream == "EVENTS":
|
||||
return strings.Contains(name, "_")
|
||||
}
|
||||
|
||||
@@ -8,15 +8,15 @@ import (
|
||||
)
|
||||
|
||||
// The dead letters' findings (novox/hq ADR 0564 §5, issue 330): a consumer DEAD_LETTERS holds a message for is said for as
|
||||
// long as it holds any, `bus.<stream>.<consumer>.max-deliveries`, and the notices waiting behind the controller's consumer
|
||||
// after the lease write are said as `bus.DEAD_LETTER_NOTICES.controller.notices-waiting`.
|
||||
// long as it holds any, `bus.<stream>.<consumer>.max-deliveries`, and the notices waiting behind the ended controller's
|
||||
// consumer are said as `bus.DEAD_LETTER_NOTICES.controller.notices-waiting`.
|
||||
|
||||
// DeadRecountEvery is how often DEAD_LETTERS is counted again while a consumer's finding stands: a letter delivered again
|
||||
// or dropped by the controller's verb, which the watchman does not hear, ends its finding within it. The expiry of the
|
||||
// watchman's own pulse, never a loop.
|
||||
const DeadRecountEvery = 5 * time.Minute
|
||||
|
||||
// KindNoticesWaiting is the finding of notices waiting while the controller's consumer stands after the lease write.
|
||||
// KindNoticesWaiting is the finding of notices waiting while the ended controller's consumer stands.
|
||||
const KindNoticesWaiting = "notices-waiting"
|
||||
|
||||
// DeadLettersHeld is DEAD_LETTERS' count per consumer, `<stream>.<consumer>`, read whole: each consumer held writes its
|
||||
@@ -73,16 +73,16 @@ func consumerMachine(stream, consumer string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// NoticesWaiting says whether the lease is handed away while the controller's consumer on DEAD_LETTER_NOTICES still
|
||||
// stands: every notice then waits in the queue, kept a week, until the lease write's step removes the consumer.
|
||||
// NoticesWaiting says whether the ended controller's consumer on DEAD_LETTER_NOTICES still stands: every notice then waits
|
||||
// in the queue, kept a week, until that consumer is removed (novox/hq ADR 0590 clause 6).
|
||||
func (w *Watchman) NoticesWaiting(waiting bool) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
f := Finding{Scope: ScopeBus, ID: "DEAD_LETTER_NOTICES.controller", Kind: KindNoticesWaiting, Severity: Warning, Source: "dead-letters",
|
||||
Summary: "the lease names no controller but the controller's consumer on DEAD_LETTER_NOTICES still stands, so every " +
|
||||
"notice of a message given up on waits in the queue, kept a week, and nothing keeps it in DEAD_LETTERS",
|
||||
Said: "the consumer controller on DEAD_LETTER_NOTICES stands after the lease write",
|
||||
Needs: "remove the consumer controller on DEAD_LETTER_NOTICES with the bus module's tool, the lease write's step."}
|
||||
Summary: "the controller is unassigned but its consumer on DEAD_LETTER_NOTICES still stands, so every notice of a " +
|
||||
"message given up on waits in the queue, kept a week, and nothing keeps it in DEAD_LETTERS",
|
||||
Said: "the consumer controller on DEAD_LETTER_NOTICES stands after the controller's end",
|
||||
Needs: "remove the consumer controller on DEAD_LETTER_NOTICES with the bus module's tool, recorded as a hand act (ADR 0590)."}
|
||||
if waiting {
|
||||
w.find(f)
|
||||
return
|
||||
|
||||
@@ -140,6 +140,18 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.KV_x.abc", []byte(`{"stream":"KV_x","consumer":"abc"}`), nil); ok {
|
||||
t.Fatal("a reader's own consumer is said")
|
||||
}
|
||||
// 1126/B9 (ADR 0590 clause 6): the ended controller's consumers are removed as hand acts, never lost; only its
|
||||
// consumer on DEAD_LETTER_NOTICES is still read, which tells the watchman to take the notices.
|
||||
for _, gone := range [][2]string{{"EVENTS", "controller"}, {"EVENTS", "controller-merges"}, {"CONTROL", "controller"}} {
|
||||
body := []byte(`{"stream":"` + gone[0] + `","consumer":"` + gone[1] + `"}`)
|
||||
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED."+gone[0]+"."+gone[1], body, func(s, c string) string { return c }); ok {
|
||||
t.Errorf("the removal of %s.%s is said as a consumer lost", gone[0], gone[1])
|
||||
}
|
||||
}
|
||||
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.DEAD_LETTER_NOTICES.controller",
|
||||
[]byte(`{"stream":"DEAD_LETTER_NOTICES","consumer":"controller"}`), func(s, c string) string { return c }); !ok {
|
||||
t.Error("the removal of the controller's consumer on DEAD_LETTER_NOTICES is not read")
|
||||
}
|
||||
w.Advised(a)
|
||||
w.Advised(a)
|
||||
k := "bus.EVENTS.ace_web.consumer-lost"
|
||||
@@ -229,7 +241,7 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
w.Provision("ace.web.oidc", enc(provisions.Waits, "x"), Put)
|
||||
w.Provision("ace.web.oidc", nil, Delete)
|
||||
}, map[string]string{}},
|
||||
{"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
{"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found, a login fallen back", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
report := func(h delivery.NodeHealth) []byte {
|
||||
b, err := delivery.EncodeNodeHealth(h)
|
||||
if err != nil {
|
||||
@@ -237,16 +249,18 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
}
|
||||
return b
|
||||
}
|
||||
w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node
|
||||
w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put) // the control node: the bus node
|
||||
w.NodeHealth("novox", report(delivery.NodeHealth{Node: "novox", Network: &delivery.NetworkHealth{State: "unhealthy", Parts: []delivery.NetworkPart{
|
||||
{Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"},
|
||||
{Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"},
|
||||
{Part: "route", State: "unhealthy", Reason: "no default route", Said: "y"}}},
|
||||
Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}},
|
||||
Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}},
|
||||
Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}}), Put)
|
||||
Units: &delivery.UnitsHealth{State: "degraded", Failed: []delivery.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}},
|
||||
Relogin: []delivery.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}},
|
||||
Found: []delivery.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}},
|
||||
Fallback: &delivery.LoginFallback{Node: "novox", From: "node.novox--b", Why: "refused twice", At: time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC)}}), Put)
|
||||
for key, sev := range map[string]string{"machine.novox.network": Urgent, "machine.novox.systemd-resolved.rewritten": Urgent,
|
||||
"machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning} {
|
||||
"machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning,
|
||||
"machine.novox.node-account-fallback": Warning} {
|
||||
if o.findings[key].Severity != sev {
|
||||
t.Errorf("%s = %+v", key, o.findings[key])
|
||||
}
|
||||
@@ -327,7 +341,16 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
w.Have("ace.mail", haveOf(t, delivery.PhaseRemoved, delivery.HealthUnknown, ""), Put)
|
||||
}, map[string]string{}},
|
||||
{"WM2 S1: a control node silent is urgent after half an hour", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.Have("novox.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
|
||||
// 1126 (ADR 0590): the control node is the bus node, the one whose have holds the bus module; a have of the
|
||||
// ended controller names none.
|
||||
w.Have("g14.mesh-controller", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
|
||||
w.Connect("node.g14")
|
||||
w.Disconnect("node.g14")
|
||||
expire(w, c, pulseKeyOf(t, presenceNS, "g14"), 120*time.Second)
|
||||
if _, ok := w.signals[signalKey("S1", "g14")]; ok {
|
||||
t.Fatalf("a node running the controller's have is held a control node")
|
||||
}
|
||||
w.Have("novox.nats", haveOf(t, delivery.PhaseRunning, delivery.HealthOK, ""), Put)
|
||||
w.Connect("node.novox")
|
||||
w.Disconnect("node.novox")
|
||||
expire(w, c, pulseKeyOf(t, presenceNS, "novox"), 120*time.Second)
|
||||
@@ -335,7 +358,7 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
expire(w, c, signalKey("S1", "novox"), ControlNodeUrgentAfter)
|
||||
}, map[string]string{"machine.novox.silent": Urgent}},
|
||||
}, map[string]string{"machine.novox.silent": Urgent, "machine.g14.silent": Warning}},
|
||||
{"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0}
|
||||
w.Closed(KindSlowConsumer, cl)
|
||||
|
||||
@@ -21,7 +21,9 @@ import (
|
||||
// said once there, never while its own network or its silence says it;
|
||||
// - machine-units `machine.<node>.units`: failed units no module places;
|
||||
// - relogin-needed `module.<module>.<node>.relogin-needed` and directory-used-as-found
|
||||
// `module.<module>.<node>.directory-used-as-found`.
|
||||
// `module.<module>.<node>.directory-used-as-found`;
|
||||
// - node-account-fallback `machine.<node>.node-account-fallback`: the engine fell back to its node's old login during a
|
||||
// rotation (novox/hq ADR 0583 §6), raised while its report carries the fall back and cleared once it no longer does.
|
||||
//
|
||||
// A failing part that points at other mesh machines (NetworkPart.Targets: the hub's node, each resolver's node) is held
|
||||
// there, as the controller holds it: when every machine it points at is down on the record (silent, its own network
|
||||
@@ -36,6 +38,7 @@ const (
|
||||
KindMachineUnits = "machine-units"
|
||||
KindReloginNeeded = "relogin-needed"
|
||||
KindUsedAsFound = "directory-used-as-found"
|
||||
KindAccountFallback = delivery.NodeAccountFallbackFinding
|
||||
)
|
||||
|
||||
// The parts and states a report names (mesh-control internal/link).
|
||||
@@ -101,6 +104,10 @@ func (w *Watchman) nodeFindings(node string) {
|
||||
want[f.Key()] = true
|
||||
w.find(f)
|
||||
}
|
||||
if f, raise := fallbackOf(node, h.Fallback); raise {
|
||||
want[f.Key()] = true
|
||||
w.find(f)
|
||||
}
|
||||
}
|
||||
for key := range w.findings {
|
||||
if !want[key] && w.ownsHealthKey(key, node) {
|
||||
@@ -249,6 +256,19 @@ func foundOf(node string, dirs []delivery.NodeDirectory) []Finding {
|
||||
return out
|
||||
}
|
||||
|
||||
// fallbackOf is a node's fall back to its old login during a rotation of its account (ADR 0583 §6), one finding.
|
||||
func fallbackOf(node string, f *delivery.LoginFallback) (Finding, bool) {
|
||||
if f == nil || f.Node != node {
|
||||
return Finding{}, false
|
||||
}
|
||||
said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why)
|
||||
return Finding{Scope: ScopeMachine, ID: node, Token: KindAccountFallback, Kind: KindAccountFallback, Machine: node, Severity: Warning,
|
||||
Source: "health-report", Since: f.At, Summary: said, Said: said,
|
||||
Headline: fmt.Sprintf("%s fell back to its old bus login", node),
|
||||
Needs: fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node),
|
||||
Resolved: fmt.Sprintf("%s no longer falls back to its old bus login", node)}, true
|
||||
}
|
||||
|
||||
func orModule(m string) string {
|
||||
if m == "" {
|
||||
return "_"
|
||||
|
||||
@@ -252,7 +252,7 @@ func (w *Watchman) stuckFinding(sk string, s delivery.Stuck) {
|
||||
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1",
|
||||
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound),
|
||||
Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since}
|
||||
// A control node, the machine running the controller, is urgent after half an hour silent, as the controller
|
||||
// A control node, the bus node (ControlNodeModule), is urgent after half an hour silent, as the controller
|
||||
// says it (its watchHeartbeats): the expiry of a pulse from its silence's start, never a loop.
|
||||
if w.control[node] {
|
||||
if age := w.now().Sub(s.Since); age > ControlNodeUrgentAfter {
|
||||
@@ -459,14 +459,14 @@ func ToolsNode(subject string) string {
|
||||
// ---- S21 and unhealthy, from the haves and the provisioner clients' reports ----
|
||||
|
||||
// haveFindings reads a have's change for the rows the haves feed: S21 (the machine's applies failing, applyCheck),
|
||||
// unhealthy (two writes in a row, urgent after four hours), and where the controller runs (S1's control node). A have
|
||||
// unhealthy (two writes in a row, urgent after four hours), and where the bus module runs (S1's control node). A have
|
||||
// removed or deleted ends its part of them.
|
||||
//
|
||||
// needs-operator is not observed from the haves: a module's part that waits for the operator is a provision in the state
|
||||
// `waits` of the seat mesh-provisioner (Provision, ADR 0570 §7).
|
||||
func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Have) {
|
||||
unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy")
|
||||
if module == ControllerModule {
|
||||
if module == ControlNodeModule {
|
||||
w.control[node] = h != nil && h.Phase != delivery.PhaseRemoved
|
||||
}
|
||||
if h == nil || h.Phase == delivery.PhaseRemoved {
|
||||
@@ -531,8 +531,10 @@ func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Hav
|
||||
}
|
||||
}
|
||||
|
||||
// ControllerModule is the module whose have names a control node.
|
||||
const ControllerModule = "mesh-controller"
|
||||
// ControlNodeModule is the module whose have names a control node: the bus module, so the control node is the bus node,
|
||||
// the one stably reachable node every part depends on (novox/hq ADR 0590, issue 1126: the controller that named it ended).
|
||||
// The same name the bus module's events are heard under (SlowConsumerSubject).
|
||||
const ControlNodeModule = "nats"
|
||||
|
||||
// Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its
|
||||
// consumer waiting for the operator, said as `module.<consumer>.<node>.needs-operator` naming each waiting provision and
|
||||
|
||||
@@ -90,7 +90,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-controller.lease",
|
||||
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps",
|
||||
"mesh-delivery-assigner.wants", "mesh-delivery-indexer.modules", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) {
|
||||
t.Errorf("reads = %v", reads)
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-controller.lease",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-host.health",
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Confirm ends an enrolment that came through (ADR 0587 §6): the node, dialled as its own login node.<node> after it
|
||||
// redeemed, calls it, and the runtime's proof of that caller is the confirmation (as ADR 0583 §3). Only then does the holder
|
||||
// say node-enrolled, with the key's whole content, and delete the key and its address key. The node's name is the caller's
|
||||
// own; an argument naming another node is refused. A confirm called again after a stop between the event and the deletes
|
||||
// finishes the deletes and says nothing twice (the bus keeps one message of one id).
|
||||
func (h *Holder) Confirm(caller, node string) (map[string]any, error) {
|
||||
own, ok := strings.CutPrefix(caller, nodePrefix)
|
||||
if !ok || CheckNode(own) != nil || strings.Contains(own, "--") {
|
||||
return nil, fmt.Errorf("only the node itself, dialled as node.<node>, confirms its enrolment, and the runtime proved the caller to be %q", caller)
|
||||
}
|
||||
if node != "" && node != own {
|
||||
return nil, fmt.Errorf("%s confirms its own enrolment only, not that of %s", caller, node)
|
||||
}
|
||||
row, found, err := h.Store.Get(NodeKey(own))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the enrolment of %s cannot be read: %w", own, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no enrolment of %s is open: nothing to confirm", own)
|
||||
}
|
||||
e, err := decodeEnrolment(row.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.Stage != StageRedeeming {
|
||||
return nil, fmt.Errorf("the enrolment of %s is %s: it is confirmed after redeem", own, e.Stage)
|
||||
}
|
||||
// The address key goes first (ADR 0589 §6): a confirm stopped midway never leaves one the catch-up would take for an
|
||||
// expiry and unplace a node that joined.
|
||||
if err := h.dropAddress(e); err != nil {
|
||||
return nil, fmt.Errorf("%w (call confirm again)", err)
|
||||
}
|
||||
if err := h.sayEvent(EventEnrolled, e, ""); err != nil {
|
||||
return nil, fmt.Errorf("node-enrolled of %s cannot be said, so nothing is deleted (call confirm again): %w", own, err)
|
||||
}
|
||||
if err := h.release(e); err != nil {
|
||||
return nil, fmt.Errorf("the enrolment of %s is confirmed and not closed (call confirm again): %w", own, err)
|
||||
}
|
||||
return map[string]any{"node": own, "address": e.Address, "says": "the node is enrolled; its enrolment is closed and the vault is asked to drop the token"}, nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// 774/S57/AC1: a confirm from any caller but node.<node> is refused; from node.<node> node-enrolled is said and both keys
|
||||
// are deleted; a stop between the event and the deletes is finished by calling confirm again.
|
||||
func confirmIsTheNodesOwnAndEndsTheEnrolment(t *testing.T) {
|
||||
for _, c := range []struct{ name, caller, node, want string }{
|
||||
{"the enrolment user", "enrol.ace", "", "dialled as node.<node>"},
|
||||
{"another node", "node.g14", "ace", "its own enrolment only"},
|
||||
{"the second login", "node.ace--b", "", "dialled as node.<node>"},
|
||||
{"no one", "unproven", "ace", "dialled as node.<node>"},
|
||||
} {
|
||||
r := newRig(t)
|
||||
r.seed("ace", "10.9.0.2", StageRedeeming)
|
||||
if _, err := r.h.Confirm(c.caller, c.node); err == nil {
|
||||
t.Errorf("%s: confirmed", c.name)
|
||||
} else {
|
||||
mustContain(t, err.Error(), c.want)
|
||||
}
|
||||
if len(r.said) != 0 || len(r.keys()) != 2 {
|
||||
t.Errorf("%s: said %v, keys %v", c.name, r.events(), r.keys())
|
||||
}
|
||||
}
|
||||
r := newRig(t)
|
||||
r.seed("ace", "10.9.0.2", StageLive)
|
||||
if _, err := r.h.Confirm("node.ace", ""); err == nil {
|
||||
t.Error("confirmed before redeem")
|
||||
}
|
||||
r.seed("ace", "10.9.0.2", StageRedeeming)
|
||||
r.st.failDelete[NodeKey("ace")] = true
|
||||
if _, err := r.h.Confirm("node.ace", ""); err == nil {
|
||||
t.Fatal("a delete that failed was taken for done")
|
||||
}
|
||||
if r.has(AddressKey("10.9.0.2")) || !r.has(NodeKey("ace")) || len(r.said) != 1 {
|
||||
t.Fatalf("keys %v, said %v", r.keys(), r.events())
|
||||
}
|
||||
if _, err := r.h.Confirm("node.ace", ""); err != nil || len(r.keys()) != 0 {
|
||||
t.Fatalf("%v, keys %v", err, r.keys())
|
||||
}
|
||||
if r.events()[0] != EventEnrolled || r.said[0].ID != r.said[1].ID {
|
||||
t.Errorf("events %v ids %s %s", r.events(), r.said[0].ID, r.said[1].ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Revoke ends an enrolment at any stage (ADR 0587 §5), by a person's call: at asked it withdraws the ask; at redeeming it also
|
||||
// undoes what redeem had not finished, the vault retiring a node account it already minted, and a node the yes added to
|
||||
// placement is taken off it again. It only takes access away, so it needs no Approve;
|
||||
// it is recorded in the hand-act log. enrolment-revoked carries the key's content and the why; the token is then dropped by
|
||||
// the durable consumer of that event, never before. A revoke that stopped is finished by calling it again.
|
||||
func (h *Holder) Revoke(caller, node, why string) (map[string]any, error) {
|
||||
if caller == "" || caller == stdio.Unproven {
|
||||
return nil, fmt.Errorf("the caller is not proven by the runtime, and a revoke is recorded with who made it")
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("a revoke says why: it is recorded as an act done by hand")
|
||||
}
|
||||
row, found, err := h.Store.Get(NodeKey(node))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the enrolment of %s cannot be read: %w", node, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no enrolment of %s is open: mesh-enrolment.tokens lists those that are", node)
|
||||
}
|
||||
e, err := decodeEnrolment(row.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.Stage == StageAsked && e.Ask != "" && h.Asker != nil {
|
||||
if err := h.Asker.Cancel(e.Ask); err != nil {
|
||||
h.logf("the ask %s of %s was not taken back at the router: %v", e.Ask, node, err)
|
||||
}
|
||||
}
|
||||
if e.Stage == StageRedeeming {
|
||||
if err := h.Vault.Retire(node); err != nil {
|
||||
return nil, fmt.Errorf("the vault did not retire the account minted for %s, so nothing is ended (call revoke again): %w", node, err)
|
||||
}
|
||||
}
|
||||
// A node that entered placement on the yes and never confirmed leaves it again (the operator's decision of 2026-11-02 01:05).
|
||||
if err := h.unplace(e, "revoked: "+why); err != nil {
|
||||
return nil, fmt.Errorf("%s is not taken off placement, so nothing is ended (call revoke again): %w", node, err)
|
||||
}
|
||||
out := map[string]any{"node": node, "was": e.Stage}
|
||||
if h.Act != nil {
|
||||
if err := h.Act("mesh-enrolment revoke of "+node+" at "+string(e.Stage), why+" (by "+caller+")", "enrolment-revoke"); err != nil {
|
||||
out["recorded"] = "NOT recorded in the hand-act log: " + err.Error()
|
||||
h.logf("the revoke of %s is not recorded in the hand-act log: %v", node, err)
|
||||
}
|
||||
}
|
||||
if err := h.sayEvent(EventRevoked, e, why); err != nil {
|
||||
return nil, fmt.Errorf("enrolment-revoked of %s cannot be said, so its keys stay (call revoke again): %w", node, err)
|
||||
}
|
||||
if err := h.release(e); err != nil {
|
||||
return nil, fmt.Errorf("the enrolment of %s is revoked and its keys are not all deleted (call revoke again): %w", node, err)
|
||||
}
|
||||
out["says"] = "the enrolment is ended; the vault is asked to drop the token and, for a redeem under way, the account was retired"
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Heard is the holder's watch of its own bucket (stdio.State.Follow): it keeps each node key's last content, and the bus's
|
||||
// expiry marker of a key at stage live says enrolment-expired, naming the node and the enrolment-issued event that carried its
|
||||
// content (ADR 0587 §5). redeem checks the expiry itself, so a marker heard late admits no machine.
|
||||
func (h *Holder) Heard(op, key string, value []byte) error {
|
||||
node, ok := strings.CutPrefix(key, "node.")
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.seen == nil {
|
||||
h.seen = map[string]Enrolment{}
|
||||
}
|
||||
switch op {
|
||||
case stdio.OpPut:
|
||||
if e, err := decodeEnrolment(value); err == nil {
|
||||
h.seen[node] = e
|
||||
}
|
||||
case stdio.OpDelete:
|
||||
delete(h.seen, node)
|
||||
case stdio.OpExpire:
|
||||
e, known := h.seen[node]
|
||||
delete(h.seen, node)
|
||||
if known {
|
||||
return h.endExpired(e)
|
||||
}
|
||||
// The key expired before the watch handed its content over: its address key still names its ask and issue (ADR 0589 §6).
|
||||
return h.reconcileLocked()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reconcile ends every enrolment whose key is gone while its address key still names its ask: one that expired while the
|
||||
// holder was down, or before its watch caught up, or whose end stopped midway (ADR 0589 §6). Run at start, after each
|
||||
// catch-up and on every expiry; a step that fails is tried again on the next pass.
|
||||
func (h *Holder) Reconcile() error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.reconcileLocked()
|
||||
}
|
||||
|
||||
func (h *Holder) reconcileLocked() error {
|
||||
keys, err := h.Store.Keys()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the enrolments cannot be listed: %w", err)
|
||||
}
|
||||
open := map[string]bool{}
|
||||
for _, k := range keys {
|
||||
if n, ok := strings.CutPrefix(k, "node."); ok {
|
||||
open[n] = true
|
||||
}
|
||||
}
|
||||
var errs []error
|
||||
for _, k := range keys {
|
||||
address, ok := strings.CutPrefix(k, "address.")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
row, found, err := h.Store.Get(k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
r, err := delivery.DecodeEnrolmentAddress(row.Value)
|
||||
// An address with no ask is reserved by an issue under way: its enrolment key is written next, then the ask.
|
||||
if err != nil || r.Ask == "" || open[r.Node] {
|
||||
continue
|
||||
}
|
||||
// Only an ask answered yes made a token and placed the node (ADR 0589 §6): one declined, or not answered yet, is no
|
||||
// expiry, and one not answered stays until its ask ends.
|
||||
if yes, err := h.answeredYes(r.Ask); err != nil {
|
||||
errs = append(errs, fmt.Errorf("the answer to %s cannot be read, so the address %s waits for the next pass: %w", r.Ask, address, err))
|
||||
continue
|
||||
} else if !yes {
|
||||
continue
|
||||
}
|
||||
h.logf("the enrolment of %s (ask %s) is gone and its address %s is still reserved: it expired unseen, and is ended now", r.Node, r.Ask, address)
|
||||
if err := h.endExpired(Enrolment{Node: r.Node, Address: address, Ask: r.Ask, Issued: r.Issued}); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// answeredYes reads the router's record of ask: whether the operator answered it yes.
|
||||
func (h *Holder) answeredYes(ask string) (bool, error) {
|
||||
_, w, err := h.whereIs(ask)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return w != nil && w.Option == "yes", nil
|
||||
}
|
||||
|
||||
// endExpired ends an enrolment whose key expired, in this order, each step safe to do again: the node, placed on the yes and
|
||||
// never confirmed, is taken out of placement on the warrant of its ask (only when placement holds it); enrolment-expired names
|
||||
// its issued event (said under one id, so said again is said once); the vault drops the token; the address key goes last, so
|
||||
// a step that failed is found and done on the next pass (ADR 0589 §6).
|
||||
func (h *Holder) endExpired(e Enrolment) error {
|
||||
if err := h.unplace(e, "expired"); err != nil {
|
||||
return fmt.Errorf("%s, whose enrolment expired, is not taken out of placement (tried again on the next pass): %w", e.Node, err)
|
||||
}
|
||||
body, err := delivery.EncodeEnrolmentExpiry(delivery.EnrolmentExpiry{Node: e.Node, IssuedID: delivery.EnrolmentIssuedID(e.Node, e.Issued)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := h.say(EventExpired, json.RawMessage(body), eventID(EventExpired, e)); err != nil {
|
||||
return fmt.Errorf("enrolment-expired of %s cannot be said (tried again on the next pass): %w", e.Node, err)
|
||||
}
|
||||
if err := h.Vault.Drop(e.Node); err != nil {
|
||||
return fmt.Errorf("the token of %s, whose enrolment expired, is not dropped (tried again on the next pass): %w", e.Node, err)
|
||||
}
|
||||
if row, found, err := h.Store.Get(AddressKey(e.Address)); err != nil {
|
||||
return fmt.Errorf("the address key %s cannot be read (tried again on the next pass): %w", e.Address, err)
|
||||
} else if found {
|
||||
if r, err := delivery.DecodeEnrolmentAddress(row.Value); err == nil && r.Node == e.Node && r.Ask == e.Ask {
|
||||
if err := h.Store.Delete(AddressKey(e.Address)); err != nil {
|
||||
return fmt.Errorf("the address key %s is not deleted (tried again on the next pass): %w", e.Address, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ended is the durable consumer's act on one of the holder's own ending events (ADR 0587 §5): the vault is asked to drop the
|
||||
// node's token and its hash, and the event is acknowledged only when the vault answers that both are gone, so a drop that
|
||||
// fails is delivered again by the bus. It never runs before an end: the consumer hears only the end events.
|
||||
func (h *Holder) Ended(event string, body []byte) error {
|
||||
var e struct {
|
||||
Node string `json:"node"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &e); err != nil || CheckNode(e.Node) != nil {
|
||||
h.logf("the event %s names no node and is not acted on: %v", event, err)
|
||||
return nil
|
||||
}
|
||||
if err := h.Vault.Drop(e.Node); err != nil {
|
||||
return fmt.Errorf("the vault did not drop the token of %s (the bus delivers %s again): %w", e.Node, event, err)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.ended = append(h.ended, json.RawMessage(`{"event":`+fmt.Sprintf("%q", event)+`,"content":`+string(body)+`}`))
|
||||
if len(h.ended) > 200 {
|
||||
h.ended = h.ended[len(h.ended)-200:]
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// 774/S58/AC1: a revoke at redeeming, whether the vault minted the account yet or not, takes both keys away, the vault retires
|
||||
// the account, the revoke is recorded and enrolment-revoked is said; one the vault failed to retire ends nothing and is
|
||||
// finished by calling it again. At asked it withdraws the ask; an expiry marker says enrolment-expired with the issued event.
|
||||
func revokeEndsAtAnyStageAndAnExpiryMarkerSaysExpired(t *testing.T) {
|
||||
r := newRig(t)
|
||||
r.seed("ace", "10.9.0.2", StageRedeeming)
|
||||
r.vault.fail["retire ace"] = true
|
||||
if _, err := r.h.Revoke("person.operator", "ace", "never confirmed"); err == nil || len(r.keys()) != 2 || len(r.said) != 0 {
|
||||
t.Fatalf("a failed retire ended it: %v, keys %v, events %v", err, r.keys(), r.events())
|
||||
}
|
||||
if _, err := r.h.Revoke("person.operator", "ace", "never confirmed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The node the yes placed, never confirmed, leaves placement on the enrolment's warrant before the end is said.
|
||||
if _, placed := r.rd["mesh-delivery-assigner.placement"]["ace.node"]; placed || len(r.added) == 0 ||
|
||||
!reflect.DeepEqual(r.added[len(r.added)-1].Change, delivery.PlacementProposalChange{Op: delivery.OpRemoveNode, Node: "ace"}) || r.added[len(r.added)-1].Warrant.Ask != "ask-ace" {
|
||||
t.Errorf("revoked and still placed: %+v", r.added)
|
||||
}
|
||||
if len(r.keys()) != 0 || !slices.Equal(r.events(), []string{EventRevoked}) || len(r.acts) != 1 || !slices.Contains(r.vault.calls, "retire ace") {
|
||||
t.Errorf("keys %v events %v acts %v vault %v", r.keys(), r.events(), r.acts, r.vault.calls)
|
||||
}
|
||||
mustContain(t, string(r.said[0].Body), "never confirmed")
|
||||
// At asked it withdraws the ask and the vault is not asked to retire anything.
|
||||
r.seed("g14", "10.9.0.3", StageAsked)
|
||||
if _, err := r.h.Revoke("person.operator", "g14", "changed our mind"); err != nil || !slices.Equal(r.asker.cancelled, []string{"ask-g14"}) || slices.Contains(r.vault.calls, "retire g14") {
|
||||
t.Errorf("%v, cancelled %v, vault %v", err, r.asker.cancelled, r.vault.calls)
|
||||
}
|
||||
if _, err := r.h.Revoke(stdio.Unproven, "ace", "x"); err == nil {
|
||||
t.Error("revoked for no one")
|
||||
}
|
||||
// The bus's marker of a key at live: the content the watch handed over is said with the issued event.
|
||||
r.seed("novox", "10.9.0.4", StageLive)
|
||||
row, _, _ := r.st.Get(NodeKey("novox"))
|
||||
_ = r.h.Heard(stdio.OpPut, NodeKey("novox"), row.Value)
|
||||
if err := r.h.Heard(stdio.OpExpire, NodeKey("novox"), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, placed := r.rd["mesh-delivery-assigner.placement"]["novox.node"]; placed {
|
||||
t.Error("expired and still placed")
|
||||
}
|
||||
last := r.said[len(r.said)-1]
|
||||
x, err := delivery.DecodeEnrolmentExpiry(last.Body)
|
||||
if err != nil || last.Event != EventExpired || x.Node != "novox" || !strings.HasPrefix(x.IssuedID, "enrolment-issued.novox.") ||
|
||||
!slices.Contains(r.vault.calls, "drop novox") || r.has(AddressKey("10.9.0.4")) {
|
||||
t.Errorf("said %s %s: %+v, %v; vault %v, keys %v", last.Event, last.Body, x, err, r.vault.calls, r.keys())
|
||||
}
|
||||
// ADR 0589 §6: an enrolment whose key expired unseen (before the watch handed it over, or while the holder was down) is
|
||||
// found by its address key, which names its ask and issue: it leaves placement on that ask's warrant, enrolment-expired
|
||||
// names its rebuilt issued event, the token is dropped and the address key goes last. A step that fails is done again on
|
||||
// the next pass, the address key kept until then.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
node string
|
||||
fail string
|
||||
passS func(r *rig) error
|
||||
}{
|
||||
{"expired unseen", "unseen", "", func(r *rig) error { return r.h.Heard(stdio.OpExpire, NodeKey("unseen"), nil) }},
|
||||
{"the holder was down, and its drop failed once", "down", "drop down", func(r *rig) error { return r.h.Reconcile() }},
|
||||
} {
|
||||
r.seed(c.node, "10.9.0.9", StageLive)
|
||||
r.asker.answer("ask-"+c.node, "yes")
|
||||
issued := r.now.Add(-time.Minute)
|
||||
_ = r.st.Delete(NodeKey(c.node)) // the bus took the key; the watch never handed its content over
|
||||
if c.fail != "" {
|
||||
r.vault.fail[c.fail] = true
|
||||
if err := c.passS(r); err == nil || !r.has(AddressKey("10.9.0.9")) {
|
||||
t.Errorf("%s: a failed drop ended the pass (%v) or took the address key", c.name, err)
|
||||
}
|
||||
}
|
||||
if err := c.passS(r); err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
last := r.said[len(r.said)-1]
|
||||
x, err := delivery.DecodeEnrolmentExpiry(last.Body)
|
||||
_, placed := r.rd["mesh-delivery-assigner.placement"][c.node+".node"]
|
||||
removed := r.added[len(r.added)-1]
|
||||
if err != nil || x.IssuedID != delivery.EnrolmentIssuedID(c.node, issued) || placed || removed.Change.Op != delivery.OpRemoveNode ||
|
||||
removed.Change.Node != c.node || removed.Warrant.Ask != "ask-"+c.node || !slices.Contains(r.vault.calls, "drop "+c.node) || r.has(AddressKey("10.9.0.9")) {
|
||||
t.Errorf("%s: said %s (%v), placed %v, last placement call %+v, vault %v, keys %v", c.name, last.Body, err, placed, removed, r.vault.calls, r.keys())
|
||||
}
|
||||
}
|
||||
// ADR 0589 §6, the negative rows: an address key the catch-up finds with its enrolment key gone is no expiry when its ask
|
||||
// was declined or is not answered yet, and none is left behind by a confirm, a decline or a refused add, which delete it
|
||||
// first. In none of them is the node unplaced.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
end func(r *rig, node string)
|
||||
key bool // the address key stays (an ask not answered yet stays until its ask ends)
|
||||
}{
|
||||
{"after a confirm stopped before the enrolment key went", func(r *rig, node string) {
|
||||
e, _ := r.enrolment(node)
|
||||
e.Stage, e.KeysDigest = StageRedeeming, delivery.KeysDigest("pk", "ov", "se", "sv")
|
||||
_, _ = r.st.Put(NodeKey(node), e, 0)
|
||||
r.asker.answer("ask-"+node, "yes")
|
||||
r.st.failDelete[NodeKey(node)] = true // the confirm stops after the address key, before the enrolment key
|
||||
_, _ = r.h.Confirm(NodeUser(node), "")
|
||||
_ = r.st.Delete(NodeKey(node)) // and the key is gone by the next pass
|
||||
}, false},
|
||||
{"after a decline", func(r *rig, node string) {
|
||||
_ = r.h.Warranted(asks.Option{ID: "no"}, asks.Warrant{Ask: "ask-" + node})
|
||||
r.asker.answer("ask-"+node, "no")
|
||||
}, false},
|
||||
{"after a refused add", func(r *rig, node string) {
|
||||
delete(r.rd["mesh-delivery-assigner.placement"], node+".node") // the add is tried, and refused
|
||||
r.placeErr = errors.New("mesh-placement did not take the node: the proposal is refused: change 1: an address taken")
|
||||
_ = r.h.Warranted(yes, asks.Warrant{Ask: "ask-" + node})
|
||||
r.asker.answer("ask-"+node, "yes")
|
||||
r.placeErr = nil
|
||||
if !slices.Contains(r.events(), EventRefused) {
|
||||
r.t.Errorf("a refused add did not end the enrolment: %v", r.events())
|
||||
}
|
||||
}, false},
|
||||
{"a declined ask whose enrolment key is gone", func(r *rig, node string) {
|
||||
r.asker.answer("ask-"+node, "no")
|
||||
_ = r.st.Delete(NodeKey(node))
|
||||
}, true},
|
||||
{"an ask not answered yet", func(r *rig, node string) { _ = r.st.Delete(NodeKey(node)) }, true},
|
||||
} {
|
||||
r := newRig(t)
|
||||
stage := StageAsked
|
||||
if strings.HasPrefix(c.name, "after a confirm") {
|
||||
stage = StageLive
|
||||
}
|
||||
r.seed("joiner", "10.9.0.11", stage)
|
||||
r.rd["mesh-delivery-assigner.placement"]["joiner.node"] = `{"v":1,"node":"joiner","decided":{"address":"10.9.0.11"},"modules":[]}`
|
||||
c.end(r, "joiner")
|
||||
removes := 0
|
||||
for _, a := range r.added {
|
||||
if a.Change.Op == delivery.OpRemoveNode {
|
||||
removes++
|
||||
}
|
||||
}
|
||||
if err := r.h.Reconcile(); err != nil {
|
||||
t.Errorf("%s: %v", c.name, err)
|
||||
}
|
||||
after := 0
|
||||
for _, a := range r.added {
|
||||
if a.Change.Op == delivery.OpRemoveNode {
|
||||
after++
|
||||
}
|
||||
}
|
||||
if after != removes || r.has(AddressKey("10.9.0.11")) != c.key || slices.Contains(r.events(), EventExpired) {
|
||||
t.Errorf("%s: the catch-up unplaced (%d to %d), address key %v, events %v", c.name, removes, after, r.has(AddressKey("10.9.0.11")), r.events())
|
||||
}
|
||||
}
|
||||
// An address reserved by an issue under way (no ask yet) is never taken for an expired enrolment.
|
||||
_, _ = r.st.Put(AddressKey("10.9.0.10"), Reservation{Node: "busy"}, 0)
|
||||
if err := r.h.Reconcile(); err != nil || !r.has(AddressKey("10.9.0.10")) {
|
||||
t.Errorf("a reservation under way was ended: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S58/AC2 and AC3: the drop is asked of the vault only from an ending event, acknowledged only once the vault answers, and
|
||||
// asked again when its first answer fails; and neither an event nor the bucket ever holds the token, sealed or not.
|
||||
func theDropFollowsOnlyAnEndAndNothingHoldsTheToken(t *testing.T) {
|
||||
r := newRig(t)
|
||||
a, _ := r.h.Issue("person.operator", "ace", "a new machine", tunnelKey, 0)
|
||||
_ = r.h.Warranted(yes, askWarrant(a.Ask))
|
||||
if _, err := r.h.Tokens("ace", false); err != nil { // relays the sealed token
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slices.ContainsFunc(r.vault.calls, func(c string) bool { return strings.HasPrefix(c, "drop") }) {
|
||||
t.Fatalf("the vault was asked to drop a token before an end: %v", r.vault.calls)
|
||||
}
|
||||
if _, err := r.h.Revoke("person.operator", "ace", "lost the machine"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
end := r.said[len(r.said)-1]
|
||||
r.vault.fail["drop ace"] = true
|
||||
if err := r.h.Ended(end.Event, end.Body); err == nil {
|
||||
t.Fatal("an event was acknowledged though the vault did not answer")
|
||||
}
|
||||
if err := r.h.Ended(end.Event, end.Body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := strings.Count(strings.Join(r.vault.calls, ","), "drop ace"); n != 2 {
|
||||
t.Errorf("the drop was asked %d times", n)
|
||||
}
|
||||
for _, s := range r.said {
|
||||
if strings.Contains(string(s.Body), "sealed") || strings.Contains(string(s.Body), "token") {
|
||||
t.Errorf("the event %s holds a token: %s", s.Event, s.Body)
|
||||
}
|
||||
}
|
||||
for k, row := range r.st.rows {
|
||||
if strings.Contains(string(row.Value), "sealed") {
|
||||
t.Errorf("the key %s holds a token", k)
|
||||
}
|
||||
}
|
||||
tk, _ := r.h.Tokens("", true)
|
||||
if raw, _ := json.Marshal(tk); !strings.Contains(string(raw), "enrolment-revoked") {
|
||||
t.Errorf("history %s", raw)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user